From b2f07de1848ecf13cfeda07db73f9dfb0aa35fea Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 01:25:13 +0200 Subject: [PATCH 01/45] reformat code --- wo/cli/main.py | 4 +++- wo/cli/plugins/stack.py | 3 +-- wo/cli/plugins/stack_migrate.py | 4 ++-- wo/core/aptget.py | 16 +++++++++------- wo/core/cron.py | 29 ++++++++++++++--------------- wo/core/download.py | 3 ++- wo/core/exc.py | 1 + wo/core/fileutils.py | 4 ++-- wo/core/mysql.py | 17 +++++++++++------ wo/core/variables.py | 7 ++++--- 10 files changed, 49 insertions(+), 39 deletions(-) diff --git a/wo/cli/main.py b/wo/cli/main.py index e8d9812..230257d 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -86,7 +86,8 @@ def main(): # if not root...kick out if not os.geteuid() == 0: - print("\nNon-privileged users cant use WordOps. Switch to root or invoke sudo.\n") + print("\nNon-privileged users cant use WordOps. " + "Switch to root or invoke sudo.\n") app.close(1) # Setup the application @@ -130,5 +131,6 @@ def get_test_app(**kw): app = WOApp(**kw) return app + if __name__ == '__main__': main() diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 00ea021..38160c0 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1,7 +1,7 @@ """Stack Plugin for WordOps""" from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook +from cement.core import handler from wo.cli.plugins.site_functions import * from wo.core.variables import WOVariables from wo.core.aptget import WOAptGet @@ -20,7 +20,6 @@ from wo.core.variables import WOVariables import random import string import configparser -import time import shutil import os import pwd diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 058d076..38bb394 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -100,8 +100,8 @@ class WOStackMigrateController(CementBaseController): Log.error( self, "Remote MySQL server in use, skipping local install") - if WOShellExec.cmd_exec(self, "mysqladmin ping") and (not - WOAptGet.is_installed(self, 'mariadb-server')): + if (WOShellExec.cmd_exec(self, "mysqladmin ping") and + (not WOAptGet.is_installed(self, 'mariadb-server'))): Log.info(self, "If your database size is big, " "migration may take some time.") diff --git a/wo/core/aptget.py b/wo/core/aptget.py index 7c274cf..bacaef4 100644 --- a/wo/core/aptget.py +++ b/wo/core/aptget.py @@ -36,7 +36,8 @@ class WOAptGet(): for single_error in error_list: if "NO_PUBKEY" in single_error: key = single_error.rsplit(None, 1)[-1] - WORepo.add_key(self, key, keyserver="hkp://pgp.mit.edu") + WORepo.add_key( + self, key, keyserver="hkp://pgp.mit.edu") proc = subprocess.Popen('apt-get update', shell=True, @@ -47,7 +48,8 @@ class WOAptGet(): if proc.returncode == 0: return True else: - Log.info(self, Log.FAIL + "Whoops, something went wrong...") + Log.info(self, Log.FAIL + + "Whoops, something went wrong...") Log.error(self, "Check the WordOps log for more details " "`tail /var/log/wo/wordops.log` and please try again...") @@ -60,7 +62,7 @@ class WOAptGet(): """ try: check_update = subprocess.Popen(['apt-get upgrade -s | grep ' - '\"^Inst\" | wc -l'], + '\"^Inst\" | wc -l'], stdout=subprocess.PIPE, shell=True).communicate()[0] if check_update == b'0\n': @@ -190,13 +192,13 @@ class WOAptGet(): apt_cache = apt.cache.Cache() apt_cache.open() if (package_name.strip() in apt_cache and - apt_cache[package_name.strip()].is_installed): + apt_cache[package_name.strip()].is_installed): # apt_cache.close() return True # apt_cache.close() return False - def download_only(self,package_name,repo_url=None,repo_key=None): + def download_only(self, package_name, repo_url=None, repo_key=None): """ Similar to `apt-get install --download-only PACKAGE_NAME` """ @@ -221,11 +223,11 @@ class WOAptGet(): if proc.returncode == 0: return True else: - Log.error(self,"Error in fetching dpkg package.\nReverting changes ..",False) + Log.error( + self, "Error in fetching dpkg package.\nReverting changes ..", False) if repo_url is not None: WORepo.remove(self, repo_url=repo_url) return False except Exception as e: Log.error(self, "Error while downloading packages, " "apt-get exited with error") - diff --git a/wo/core/cron.py b/wo/core/cron.py index 2392ad4..abd3bd6 100644 --- a/wo/core/cron.py +++ b/wo/core/cron.py @@ -5,28 +5,27 @@ from wo.core.logging import Log Set CRON on LINUX system. """ + class WOCron(): - def setcron_weekly(self,cmd,comment='Cron set by WordOps',user='root',min=0,hour=12): + def setcron_weekly(self, cmd, comment='Cron set by WordOps', user='root', min=0, hour=12): if not WOShellExec.cmd_exec(self, "crontab -l | grep -q \'{0}\'".format(cmd)): WOShellExec.cmd_exec(self, "/bin/bash -c \"crontab -l " - "2> /dev/null | {{ cat; echo -e" - " \\\"" - "\\n0 0 * * 0 " - "{0}".format(cmd) + - " # {0}".format(comment)+ - "\\\"; } | crontab -\"") + "2> /dev/null | {{ cat; echo -e" + " \\\"" + "\\n0 0 * * 0 " + "{0}".format(cmd) + + " # {0}".format(comment) + + "\\\"; } | crontab -\"") Log.debug(self, "Cron set") - - - def remove_cron(self,cmd): + def remove_cron(self, cmd): if WOShellExec.cmd_exec(self, "crontab -l | grep -q \'{0}\'".format(cmd)): if not WOShellExec.cmd_exec(self, "/bin/bash -c " - "\"crontab " - "-l | sed '/{0}/d'" - "| crontab -\"" - .format(cmd)): - Log.error(self, "Failed to remove crontab entry",False) + "\"crontab " + "-l | sed '/{0}/d'" + "| crontab -\"" + .format(cmd)): + Log.error(self, "Failed to remove crontab entry", False) else: Log.debug(self, "Cron not found") diff --git a/wo/core/download.py b/wo/core/download.py index ef94d4f..36d3d7a 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -22,7 +22,8 @@ class WODownload(): if not os.path.exists(directory): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') - req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + req = urllib.request.Request( + url, headers={'User-Agent': 'Mozilla/5.0'}) with urllib.request.urlopen(req) as response, open(filename, 'wb') as out_file: out_file.write(response.read()) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" diff --git a/wo/core/exc.py b/wo/core/exc.py index 67e968f..02c03dc 100644 --- a/wo/core/exc.py +++ b/wo/core/exc.py @@ -3,6 +3,7 @@ class WOError(Exception): """Generic errors.""" + def __init__(self, msg): Exception.__init__(self) self.msg = msg diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index a6032b8..a7a4c29 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -21,14 +21,14 @@ class WOFileUtils(): Log.info(self, "Removing {0:65}".format(file), end=' ') os.remove(file) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + - Log.OKBLUE + "]")) + Log.OKBLUE + "]")) Log.debug(self, 'file Removed') if os.path.isdir(file): try: Log.info(self, "Removing {0:65}".format(file), end=' ') shutil.rmtree(file) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + - Log.OKBLUE + "]")) + Log.OKBLUE + "]")) except shutil.Error as e: Log.debug(self, "{err}".format(err=str(e.reason))) Log.error(self, 'Unable to Remove file ') diff --git a/wo/core/mysql.py b/wo/core/mysql.py index 92a1c5e..6b1f115 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -28,10 +28,12 @@ class WOMysql(): """Method for MySQL connection""" def connect(self): - """Makes connection with MySQL server""" + # Makes connection with MySQL server try: if os.path.exists('/etc/mysql/conf.d/my.cnf'): - connection = pymysql.connect(read_default_file='/etc/mysql/conf.d/my.cnf') + connection = \ + pymysql.connect(read_default_file='/etc/mysql/' + 'conf.d/my.cnf') else: connection = pymysql.connect(read_default_file='~/.my.cnf') return connection @@ -45,9 +47,11 @@ class WOMysql(): def dbConnection(self, db_name): try: if os.path.exists('/etc/mysql/conf.d/my.cnf'): - connection = pymysql.connect(db=db_name,read_default_file='/etc/mysql/conf.d/my.cnf') + connection = pymysql.connect( + db=db_name, read_default_file='/etc/mysql/conf.d/my.cnf') else: - connection = pymysql.connect(db=db_name,read_default_file='~/.my.cnf') + connection = pymysql.connect( + db=db_name, read_default_file='~/.my.cnf') return connection except DatabaseError as e: @@ -58,12 +62,13 @@ class WOMysql(): except pymysql.err.InternalError as e: Log.debug(self, str(e)) raise MySQLConnectionError - except Exception as e : + except Exception as e: Log.debug(self, "[Error]Setting up database: \'" + str(e) + "\'") raise MySQLConnectionError def execute(self, statement, errormsg='', log=True): - """Get login details from /etc/mysql/conf.d/my.cnf & Execute MySQL query""" + # Get login details from /etc/mysql/conf.d/my.cnf + # & Execute MySQL query connection = WOMysql.connect(self) log and Log.debug(self, "Exceuting MySQL Statement : {0}" .format(statement)) diff --git a/wo/core/variables.py b/wo/core/variables.py index 74c4cec..054ef53 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -3,7 +3,6 @@ import platform import socket import configparser import os -import sys import psutil import datetime @@ -149,11 +148,13 @@ class WOVariables(): # MySQL repo and packages if wo_platform_distro == 'ubuntu': - wo_mysql_repo = ("deb [arch=amd64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/" + wo_mysql_repo = ("deb [arch=amd64,ppc64el] " + "http://sfo1.mirrors.digitalocean.com/mariadb/repo/" "10.3/ubuntu {codename} main" .format(codename=wo_platform_codename)) elif wo_platform_distro == 'debian': - wo_mysql_repo = ("deb [arch=amd64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/" + wo_mysql_repo = ("deb [arch=amd64,ppc64el] " + "http://sfo1.mirrors.digitalocean.com/mariadb/repo/" "10.3/debian {codename} main" .format(codename=wo_platform_codename)) From 739f1bc0f877e8105e58f6699cafc4bde5f933be Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 01:34:33 +0200 Subject: [PATCH 02/45] fix hook --- wo/cli/plugins/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 38160c0..eab3f5c 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1,7 +1,7 @@ """Stack Plugin for WordOps""" from cement.core.controller import CementBaseController, expose -from cement.core import handler +from cement.core import handler, hook from wo.cli.plugins.site_functions import * from wo.core.variables import WOVariables from wo.core.aptget import WOAptGet From 4ac9a5f4fec102111018fcfd797b82fd4b13b29a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 01:38:14 +0200 Subject: [PATCH 03/45] add fail2ban --- smtp-cli.pl | 1074 ------------------------------ wo/cli/plugins/stack.py | 28 + wo/cli/plugins/stack_services.py | 43 +- 3 files changed, 67 insertions(+), 1078 deletions(-) delete mode 100644 smtp-cli.pl diff --git a/smtp-cli.pl b/smtp-cli.pl deleted file mode 100644 index 0eeb426..0000000 --- a/smtp-cli.pl +++ /dev/null @@ -1,1074 +0,0 @@ -#!/usr/bin/perl - -# -# Command line SMTP client with SSL, STARTTLS, SMTP-AUTH and IPv6 support. -# Michal Ludvig, 2003-2018 -# See http://smtp-cli.logix.cz for details -# and https://github.com/mludvig/smtp-cli for code. -# Thanks to all contributors for ideas and fixes! -# - -my $version = "3.9"; - -# -# ChangeLog: -# * Version 3.9 (2018-04-06) -# - Don't attempt to use IO::Socket::INET6 with --ipv4. -# -# * Version 3.8 (2017-07-05) -# - New parameter --local-addr -# - Support body and attachment reading from non-regular files -# - Various protocol fixes -# -# * Version 3.7 (2014-11-21) -# - Support STDIN input with --body-plain=- or --body-html=- -# -# * Version 3.6 (2013-07-11) -# - Improved compatibility with perl < 5.10 and perl >= 5.18 -# - Added support for more chars in user-part of email address. -# -# * Version 3.5 (2013-05-08) -# - Improved compliance with SMTP RFC 5321 -# - New parameter --text-encoding -# -# * Version 3.4 (2013-02-05) -# - Ok, ok, support both File::Type and File::LibMagic -# -# * Version 3.3 (2012-07-30) -# - Moved from File::Type to File::LibMagic -# (File::Type is no longer maintained and not available -# in EPEL for RHEL 6) -# -# * Version 3.2 (2012-06-26) -# - Fixed syntax error -# -# * Version 3.1 (2012-06-25) -# - New --add-header, --replace-header and --remove-header options. -# - Improved compatibility with new IO::Socket::SSL releases. -# -# * Version 3.0 (2012-01-24) -# - Support for server SSL verification agains CA root cert. -# - Use "Content-Disposition: attachment" for all attachments -# unless --attach-inline was used. -# - No longer default to --server=localhost -# - Support for --charset= affecting all text/* parts. -# - Ensure "To: undisclosed-recipients:;" if sending only to Bcc. -# -# * Version 2.9 (2011-09-02) -# - Fixed problem when using IPv6 addresses with --server. -# For example with --server 2001:db8::123 it was connecting -# to server 2001:db8:: port 123. Fixed now. -# -# * Version 2.8 (2011-01-05) -# - Added --ssl to support for SSMTP (SMTP over SSL). This is -# turned on by default when --port=465. -# -# * Version 2.7 (2010-09-08) -# - Added support for Cc header (--cc=...) -# - Addressess (From, To, Cc) can now contain a "display name", -# for example --from="Michal Ludvig " -# - Support for --mail-from and --rcpt-to addresses independent -# on --from, --to, --cc and --bcc -# - Fixed warnings in Perl 5.12 -# -# * Version 2.6 (2009-08-05) -# - Message building fixed for plaintext+attachment case. -# - Auto-enable AUTH as soon as --user parameter is used. -# (previously --enable-auth or --auth-plain had to be used -# together with --user, that was confusing). -# - New --print-only parameter for displaying the composed -# MIME message without sending. -# - All(?) non-standard modules are now optional. -# - Displays local and remote address on successfull connect. -# -# * Version 2.5 (2009-07-21) -# - IPv6 support provided the required modules are -# available. -# -# * Version 2.1 (2008-12-08) -# - Make the MIME modules optional. Simply disable -# the required functionality if they're not available. -# -# * Version 2.0 (2008-11-18) -# - Support for message building through MIME::Lite, -# including attachments, multipart, etc. -# -# * Version 1.1 (2006-08-26) -# - STARTTLS and AUTH support -# -# * Version 1.0 -# - First public version -# -# This program is licensed under GNU Public License v3 (GPLv3) -# - -## Require Perl 5.8 or higher -> we need open(.., .., \$variable) construct -require 5.008; - -use strict; -use IO::Socket::INET; -use MIME::Base64 qw(encode_base64 decode_base64); -use Getopt::Long; -use Socket qw(:DEFAULT :crlf); - -my @valid_encodings = ("7bit", "8bit", "binary", "base64", "quoted-printable"); - -my ($user, $pass, $host, $port, $addr_family, $localaddr, - $use_login, $use_plain, $use_cram_md5, - $ehlo_ok, $auth_ok, $starttls_ok, $ssl, $verbose, - $hello_host, $datasrc, - $mail_from, @rcpt_to, $from, @to, @cc, @bcc, - $missing_modules_ok, $missing_modules_count, - $subject, $body_plain, $body_html, $charset, $text_encoding, $print_only, - @attachments, @attachments_inline, - @add_headers, @replace_headers, @remove_headers, - $ssl_ca_file, $ssl_ca_path, - $sock, $built_message); - -$host = undef; -$port = 'smtp(25)'; -$addr_family = AF_UNSPEC; -$localaddr = undef; -$hello_host = 'localhost'; -$verbose = 0; -$use_login = 0; -$use_plain = 0; -$use_cram_md5 = 0; -$starttls_ok = 1; -$ssl = undef; -$auth_ok = 0; -$ehlo_ok = 1; -$missing_modules_ok = 0; -$missing_modules_count = 0; -$charset = undef; -$text_encoding = "quoted-printable"; -$print_only = 0; - -# Get command line options. -GetOptions ( - 'host|server=s' => \$host, - 'port=i' => \$port, - '4|ipv4' => sub { $addr_family = AF_INET; }, - '6|ipv6' => sub { $addr_family = AF_INET6; }, - 'local-addr=s' => \$localaddr, - 'user=s' => \$user, 'password=s' => \$pass, - 'auth-login' => \$use_login, - 'auth-plain' => \$use_plain, - 'auth-cram-md5' => \$use_cram_md5, - 'disable-ehlo' => sub { $ehlo_ok = 0; }, - 'force-ehlo' => sub { $ehlo_ok = 2; }, - 'hello-host|ehlo-host|helo-host=s' => \$hello_host, - 'auth|enable-auth' => \$auth_ok, - 'disable-starttls|disable-tls|disable-ssl' => - sub { $starttls_ok = 0; }, - 'ssl' => sub { $ssl = 1 }, - 'disable-ssl' => sub { $ssl = 0 }, - 'mail-from=s' => \$mail_from, - 'rcpt-to=s' => \@rcpt_to, - 'from=s' => \$from, - 'to=s' => \@to, - 'cc=s' => \@cc, - 'bcc=s' => \@bcc, - 'data=s' => \$datasrc, - 'subject=s' => \$subject, - 'body|body-plain=s' => \$body_plain, - 'body-html=s' => \$body_html, - 'charset=s' => \$charset, - 'text-encoding=s' => \$text_encoding, - 'attachment|attach=s' => \@attachments, - 'attachment-inline|attach-inline=s' => \@attachments_inline, - 'add-header=s' => \@add_headers, - 'replace-header=s' => \@replace_headers, - 'remove-header=s' => \@remove_headers, - 'print-only' => \$print_only, - 'missing-modules-ok' => \$missing_modules_ok, - 'ssl-ca-file=s' => \$ssl_ca_file, - 'ssl-ca-path=s' => \$ssl_ca_path, - 'v|verbose+' => \$verbose, - 'version' => sub { &version() }, - 'help' => sub { &usage() } ); - -#### Try to load optional modules - -## IO::Socket::SSL and Net::SSLeay are optional -my $have_ssl = eval { require IO::Socket::SSL; require Net::SSLeay; 1; }; -if (not $have_ssl and not $missing_modules_ok) { - warn("!!! IO::Socket::SSL and/or Net::SSLeay modules are not found\n"); - warn("!!! These modules are required for SSL and STARTTLS support\n"); - $missing_modules_count += 2; -} - -## IO::Socket::INET6 and Socket6 are optional -my $socket6 = eval { require IO::Socket::INET6; require Socket6; 1; }; -if (not $socket6 and not ($addr_family == AF_INET)) { - if ($addr_family == AF_INET6) { - die("!!! IO::Socket::INET6 and Socket6 modules are not found\nIPv6 support is not available\n"); - } - if (not $missing_modules_ok) { - warn("!!! IO::Socket::INET6 -- optional module not found\n"); - warn("!!! Socket6 -- optional module not found\n"); - warn("!!! These modules are required for IPv6 support\n\n"); - $missing_modules_count += 2; - } -} - -## MIME::Lite dependency is optional -my $mime_lite = eval { require MIME::Lite; 1; }; -if (not $mime_lite and not $missing_modules_ok) { - warn("!!! MIME::Lite -- optional module not found\n"); - warn("!!! Used for composing messages from --subject, --body, --attachment, etc.\n\n"); - $missing_modules_count++; -} - -## File::LibMagic dependency is optional -my $file_libmagic = eval { require File::LibMagic; File::LibMagic->new(); }; - -## File::Type dependency is optional -## Not needed if File::LibMagic is available -my $file_type = eval { require File::Type; File::Type->new(); }; - -if (not $file_libmagic and not $file_type and not $missing_modules_ok) { - warn("!!! Neither File::LibMagic nor File::Type module found.\n"); - warn("!!! Used for guessing MIME types of attachments. Optional.\n\n"); - $missing_modules_count++; -} - -## Term::ReadKey dependency is optional -my $have_term_readkey = eval { require Term::ReadKey; 1; }; -if (not $have_term_readkey and not $missing_modules_ok) { - warn("!!! Term::ReadKey -- optional module not found\n"); - warn("!!! Used for hidden reading SMTP password from the terminal\n\n"); - $missing_modules_count++; -} - -my $have_hmac_md5 = eval { require Digest::HMAC_MD5; 1; }; -if (not $have_hmac_md5 and not $missing_modules_ok) { - if ($use_cram_md5) { - die("!!! CRAM-MD5 authentication is not available because Digest::HMAC_MD5 module is missing\n"); - } - warn("!!! Digest::HMAC_MD5 -- optional module missing\n"); - warn("!!! Used for CRAM-MD5 authentication method\n"); - $missing_modules_count++; -} - -## Advise about --missing-modules-ok parameter -if ($missing_modules_count) { - warn("!!! Use --missing-modules-ok if you don't need the above listed modules\n"); - warn("!!! and don't want to see this message again.\n\n"); -} - -## Make sure we've got a server name to connect to -if (not defined($host)) { - if (not $print_only) { - die("Error: Specify the SMTP server with --server=hostname[:port]\n"); - } else { - # We're printing to stdout only, let's assign just about any - # hostname to satisfy the next few tests. - $host = "localhost"; - } -} - -## Make sure the --text-encoding value is valid -if (not grep(/^$text_encoding$/, @valid_encodings)) -{ - die ("The --text-encoding value is invalid: $text_encoding\nMust be one of: " . join(', ', @valid_encodings) . "\n"); -} - -## Accept hostname with port number as host:port -## Either it's a hostname:port or 1.2.3.4:port or [2001:db8::1]:port. -## Don't parse 2001:db8::1 as $host=2001:db8:: and $port=1! -if (($host =~ /^([^:]+):([:alnum:]+)$/) or - ($host =~ /^\[([[:xdigit:]:]+)\]:([:alnum:]+)$/)) -{ - $host = $1; - $port = $2; -} - -## Automatically start in SSL mode if port == 465 (SSMTP) -if (not defined($ssl)) { - $ssl = ($port == 465); -} - -# Extract $mail_from address from $from -if (not defined($mail_from) and defined($from)) { - $mail_from = &find_email_addr($from) or - die ("The --from string does not contain a valid email address: $from\n"); -} - -# Extract @rcpt_to list from @to, @cc and @bcc -if (not @rcpt_to) { - foreach my $rcpt (@to, @cc, @bcc) { - my $rcpt_addr = &find_email_addr($rcpt); - if (not defined($rcpt_addr)) { - warn("No valid email address found in: $rcpt\n"); - next; - } - push(@rcpt_to, $rcpt_addr); - } -} - -# Ensure "To: undisclosed-recipients:;" when sending only to Bcc's -if (not @to and not @cc) { - push(@to, "undisclosed-recipients:;"); -} - -# Build the MIME message if required -if (defined($subject) or defined($body_plain) or defined($body_html) or - @attachments or @attachments_inline) { - if (not $mime_lite) { - die("Module MIME::Lite is not available. Unable to build the message, sorry.\n". - "Use --data and provide a complete email payload including headers instead.\n"); - } - if (defined($datasrc)) { - die("Requested building a message and at the same time used --data parameter.\n". - "That's not possible, sorry.\n"); - } - if (defined($body_plain)) { - if (-e $body_plain) { - local $/=undef; - open(FILE, $body_plain); - $body_plain = ; - close(FILE); - } elsif ($body_plain eq "-") { - local $/=undef; - $body_plain = ; - } - } - if (defined($body_html)) { - if (-e $body_html) { - local $/=undef; - open(FILE, $body_html); - $body_html = ; - close(FILE); - } elsif ($body_html eq "-") { - local $/=undef; - $body_html = ; - } - } - my $message = &build_message(); - - open(BUILT_MESSAGE, "+>", \$built_message); - $datasrc = "///built_message"; - if ($print_only) { - $message->print(); - exit(0); - } else { - $message->print(\*BUILT_MESSAGE); - } - seek(BUILT_MESSAGE, 0, 0); -} - -# Username was given -> enable AUTH -if ($user) - { $auth_ok = 1; } - -# If at least one --auth-* option was given, enable AUTH. -if ($use_login + $use_plain + $use_cram_md5 > 0) - { $auth_ok = 1; } - -# If --enable-auth was given, enable all AUTH methods. -elsif ($auth_ok && ($use_login + $use_plain + $use_cram_md5 == 0)) -{ - $use_login = 1; - $use_plain = 1; - $use_cram_md5 = 1 if ($have_hmac_md5); -} - -# Exit if user haven't specified username for AUTH. -if ($auth_ok && !defined ($user)) - { die ("SMTP AUTH support requested without --user\n"); } - -# Ask for password if it wasn't supplied on the command line. -if ($auth_ok && defined ($user) && !defined ($pass)) -{ - if ($have_term_readkey) { - # Set echo off. - Term::ReadKey::ReadMode (2); - } else { - warn ("Module Term::ReadKey not available - password WILL NOT be hidden!!!\n"); - } - printf ("Enter password for %s@%s : ", $user, $host); - $pass = <>; - if ($have_term_readkey) { - # Restore echo. - Term::ReadKey::ReadMode (0); - printf ("\n"); - } - exit if (! defined ($pass)); - chop ($pass); -} - -# Connect to the SMTP server. -my %connect_args = ( - PeerAddr => $host, - PeerPort => $port, - Proto => 'tcp', - Timeout => 5); - -if (defined($localaddr)) { - $connect_args{'LocalAddr'} = $localaddr; -} - -if ($addr_family == AF_INET) { - # If the user requested --ipv4 don't even bother with INET6 module - # (although it should work some users reported problems) - $sock = IO::Socket::INET->new(%connect_args) or die ("Connect failed: $@\n"); -} else { - # Either --ipv6 or no preference - do the best we can - $connect_args{'Domain'} = $addr_family; - $sock = IO::Socket::INET6->new(%connect_args) or die ("Connect failed: $@\n"); -} - -if ($verbose >= 1) { - my $addr_fmt = "%s"; - $addr_fmt = "[%s]" if ($sock->sockhost() =~ /:/); ## IPv6 connection - - printf ("Connection from $addr_fmt:%s to $addr_fmt:%s\n", - $sock->sockhost(), $sock->sockport(), - $sock->peerhost(), $sock->peerport()); -} - -if ($ssl) { - printf ("Starting SMTP/SSL...\n") if ($verbose >= 1); - &socket_to_ssl($sock); -} - -my ($code, $text); -my (%features); - -# Wait for the welcome message of the server. -($code, $text) = &get_line ($sock); -die ("Unknown welcome string: '$code $text'\n") if ($code != 220); -$ehlo_ok-- if ($text !~ /ESMTP/); - -# Send EHLO -&say_hello ($sock, $ehlo_ok, $hello_host, \%features) or exit (1); - -# Run the SMTP session -my $exitcode = &run_smtp (); - -# Good bye... -&send_line ($sock, "QUIT\n"); -($code, $text) = &get_line ($sock); -die ("Unknown QUIT response '$code'.\n") if ($code != 221); - -exit $exitcode; - -# This is the main SMTP "engine". -sub run_smtp -{ - # See if we could start encryption - if ((defined ($features{'STARTTLS'}) || defined ($features{'TLS'})) && $starttls_ok && !$have_ssl) - { - warn ("Module IO::Socket::SSL is missing - STARTTLS support disabled.\n"); - warn ("Use --disable-starttls or install the modules to avoid this warning.\n"); - undef ($features{'STARTTLS'}); - undef ($features{'TLS'}); - } - - if ((defined ($features{'STARTTLS'}) || defined ($features{'TLS'})) && $starttls_ok) - { - printf ("Starting TLS...\n") if ($verbose >= 1); - - &send_line ($sock, "STARTTLS\n"); - ($code, $text) = &get_line ($sock); - die ("Unknown STARTTLS response '$code'.\n") if ($code != 220); - - &socket_to_ssl($sock); - - # Send EHLO again (required by the SMTP standard). - &say_hello ($sock, $ehlo_ok, $hello_host, \%features) or return 0; - } - - # See if we should authenticate ourself - if (defined ($features{'AUTH'}) && $auth_ok) - { - printf ("AUTH method (%s): ", $features{'AUTH'}) if ($verbose >= 1); - - ## Try DIGEST-MD5 first - # Actually we won't. It never worked reliably here. - # After all DIGEST-MD5 is on a way to deprecation - # see this thread: http://www.imc.org/ietf-sasl/mail-archive/msg02996.html - - # Instead use CRAM-MD5 if supported by the server - if ($features{'AUTH'} =~ /CRAM-MD5/i && $use_cram_md5) - { - printf ("using CRAM-MD5\n") if ($verbose >= 1); - &send_line ($sock, "AUTH CRAM-MD5\n"); - ($code, $text) = &get_line ($sock); - if ($code != 334) - { die ("AUTH CRAM-MD5 failed: $code $text\n"); } - - my $response = &encode_cram_md5 ($text, $user, $pass); - &send_line ($sock, "%s\n", $response); - ($code, $text) = &get_line ($sock); - if ($code != 235) - { die ("AUTH CRAM-MD5 failed: $code $text\n"); } - } - # Eventually try LOGIN method - elsif ($features{'AUTH'} =~ /LOGIN/i && $use_login) - { - printf ("using LOGIN\n") if ($verbose >= 1); - &send_line ($sock, "AUTH LOGIN\n"); - ($code, $text) = &get_line ($sock); - if ($code != 334) - { die ("AUTH LOGIN failed: $code $text\n"); } - - &send_line ($sock, "%s\n", encode_base64 ($user, "")); - - ($code, $text) = &get_line ($sock); - if ($code != 334) - { die ("AUTH LOGIN failed: $code $text\n"); } - - &send_line ($sock, "%s\n", encode_base64 ($pass, "")); - - ($code, $text) = &get_line ($sock); - if ($code != 235) - { die ("AUTH LOGIN failed: $code $text\n"); } - } - # Or finally PLAIN if nothing else was supported. - elsif ($features{'AUTH'} =~ /PLAIN/i && $use_plain) - { - printf ("using PLAIN\n") if ($verbose >= 1); - &send_line ($sock, "AUTH PLAIN %s\n", - encode_base64 ("$user\0$user\0$pass", "")); - ($code, $text) = &get_line ($sock); - if ($code != 235) - { die ("AUTH PLAIN failed: $code $text\n"); } - } - # Complain otherwise. - else - { - warn ("No supported authentication method\n". - "advertised by the server.\n"); - return 1; - } - - printf ("Authentication of $user\@$host succeeded\n") if ($verbose >= 1); - } - - # We can do a relay-test now if a recipient was set. - if ($#rcpt_to >= 0) - { - if (!defined ($mail_from)) - { - warn ("From: address not set. Using empty one.\n"); - $mail_from = ""; - } - &send_line ($sock, "MAIL FROM:<%s>\n", $mail_from); - ($code, $text) = &get_line ($sock); - if ($code != 250) - { - warn ("MAIL FROM <$mail_from> failed: '$code $text'\n"); - return 1; - } - - my $i; - for ($i=0; $i <= $#rcpt_to; $i++) - { - &send_line ($sock, "RCPT TO:<%s>\n", $rcpt_to[$i]); - ($code, $text) = &get_line ($sock); - if ($code != 250) - { - warn ("RCPT TO <".$rcpt_to[$i]."> ". - "failed: '$code $text'\n"); - return 0; - } - } - } - - # Wow, we should even send something! - if (defined ($datasrc)) - { - if ($datasrc eq "///built_message") - { - *MAIL = *BUILT_MESSAGE; - } - elsif ($datasrc eq "-") - { - *MAIL = *STDIN; - } - elsif (!open (MAIL, $datasrc)) - { - warn ("Can't open file '$datasrc'\n"); - return 0; - } - - &send_line ($sock, "DATA\n"); - ($code, $text) = &get_line ($sock); - if ($code != 354) - { - warn ("DATA failed: '$code $text'\n"); - return 0; - } - - while () - { - my $line = $_; - # RFC 5321 section 4.5.2 - leading dot must be doubled - $line =~ s/^\./\.\./; - # RFC 5321 section 2.3.8 - ensure CR-LF line ending - $line =~ s/[\r\n]+$/$CRLF/; - $sock->print ($line); - } - - close (MAIL); - - $sock->printf ("$CRLF.$CRLF"); - - ($code, $text) = &get_line ($sock); - if ($code != 250) - { - warn ("DATA not send: '$code $text'\n"); - return 0; - } - } - - # Perfect. Everything succeeded! - return 1; -} - -# Get one line of response from the server. -sub get_one_line ($) -{ - my $sock = shift; - my ($code, $sep, $text) = ($sock->getline() =~ /(\d+)(.)([^\r]*)/); - my $more; - $more = ($sep eq "-"); - if ($verbose) - { printf ("[%d] '%s'\n", $code, $text); } - return ($code, $text, $more); -} - -# Get concatenated lines of response from the server. -sub get_line ($) -{ - my $sock = shift; - my ($code, $text, $more) = &get_one_line ($sock); - while ($more) { - my ($code2, $line); - ($code2, $line, $more) = &get_one_line ($sock); - $text .= " $line"; - die ("Error code changed from $code to $code2. That's illegal.\n") if ($code ne $code2); - } - return ($code, $text); -} - -# Send one line back to the server -sub send_line ($@) -{ - my $socket = shift; - my @args = @_; - - if ($verbose) - { printf ("> "); printf (@args); } - $args[0] =~ s/\n/$CRLF/g; - $socket->printf (@args); -} - -sub socket_to_ssl($) -{ - if (!$have_ssl) { - die ("SSL/TLS support is not available due to missing modules. Sorry.\n"); - } - - # Do Net::SSLeay initialization - Net::SSLeay::load_error_strings(); - Net::SSLeay::SSLeay_add_ssl_algorithms(); - Net::SSLeay::randomize(); - - if (! IO::Socket::SSL->start_SSL($sock, { - SSL_ca_file => $ssl_ca_file, - SSL_ca_path => $ssl_ca_path, - SSL_verify_mode => (defined($ssl_ca_file) or defined($ssl_ca_path)) ? 0x01 : 0x00, - })) - { - die ("SSL/TLS: ".IO::Socket::SSL::errstr()."\n"); - } - - if ($verbose >= 1) - { - printf ("Using cipher: %s\n", $sock->get_cipher ()); - printf ("%s", $sock->dump_peer_certificate()); - } -} - -# Helper function to encode CRAM-MD5 challenge -sub encode_cram_md5 ($$$) -{ - my ($ticket64, $username, $password) = @_; - my $ticket = decode_base64($ticket64) or - die ("Unable to decode Base64 encoded string '$ticket64'\n"); - - print "Decoded CRAM-MD5 challenge: $ticket\n" if ($verbose > 1); - my $password_md5 = Digest::HMAC_MD5::hmac_md5_hex($ticket, $password); - return encode_base64 ("$username $password_md5", ""); -} - -# Store all server's ESMTP features to a hash. -sub say_hello ($$$$) -{ - my ($sock, $ehlo_ok, $hello_host, $featref) = @_; - my ($feat, $param); - my $hello_cmd = $ehlo_ok > 0 ? "EHLO" : "HELO"; - - &send_line ($sock, "$hello_cmd $hello_host\n"); - my ($code, $text, $more) = &get_one_line ($sock); - - if ($code != 250) - { - warn ("$hello_cmd failed: '$code $text'\n"); - return 0; - } - - # Empty the hash - %{$featref} = (); - - ($feat, $param) = ($text =~ /^(\w+)[= ]*(.*)$/); - $featref->{$feat} = $param; - - # Load all features presented by the server into the hash - while ($more == 1) - { - ($code, $text, $more) = &get_one_line ($sock); - ($feat, $param) = ($text =~ /^(\w+)[= ]*(.*)$/); - $featref->{$feat} = $param; - } - - return 1; -} - -sub find_email_addr($) -{ - my $addr = shift; - if ($addr =~ /([A-Z0-9._%=#+-]+@(?:[A-Z0-9-]+\.)+[A-Z]+)\b/i) { - return $1; - } - return undef; -} - -sub guess_mime_type($) -{ - my $filename = shift; - if (defined($file_libmagic)) { - ## Use File::LibMagic if possible - return $file_libmagic->checktype_filename($filename); - } elsif (defined($file_type)) { - ## Use File::Type if possible - return $file_type->mime_type($filename); - } else { - ## Module File::LibMagic is not available - ## Still recognise some common extensions - return "image/jpeg" if ($filename =~ /\.jpe?g/i); - return "image/gif" if ($filename =~ /\.gif/i); - return "image/png" if ($filename =~ /\.png/i); - return "text/plain" if ($filename =~ /\.txt/i); - return "application/zip" if ($filename =~ /\.zip/i); - return "application/x-gzip" if ($filename =~ /\.t?gz/i); - return "application/x-bzip" if ($filename =~ /\.t?bz2?/i); - } - return "application/octet-stream"; -} - -sub basename($) -{ - my $path = shift; - my @parts = split(/\//, $path); - return $parts[$#parts]; -} - -sub prepare_attachment($) -{ - my $attachment = shift; - my ($path, $mime_type); - - if (-e $attachment) { - $path = $attachment; - $mime_type = guess_mime_type($attachment); - } elsif ($attachment =~ /(.*)@([^@]*)$/ and -e $1) { - $path = $1; - $mime_type = $2; - } - return ($path, $mime_type); -} - -sub attach_attachments($$@) -{ - my $message = shift; - my $disposition = shift; - my @attachments = @_; - - foreach my $attachment (@attachments) { - my ($path, $mime_type) = prepare_attachment($attachment); - if (not defined($path)) { - warn("$attachment: File not found. Ignoring.\n"); - next; - } - $message->attach( - Type => $mime_type, - Path => $path, - Id => basename($path), - Disposition => $disposition, - ); - } -} - -sub safe_attach($$) -{ - my ($message, $part) = @_; - ## Remove some headers when $part is becoming a subpart of $message - $part->delete("Date"); - $part->delete("X-Mailer"); - $part->attr("MIME-Version" => undef); - $message->attach($part); - return $message; -} - -sub mime_message($$) -{ - my ($type, $data) = @_; - - ## Set QP encoding for text/* types, let MIME::Lite decide for all other types. - my $encoding = $type =~ /^text\// ? $text_encoding : undef; - my $message = MIME::Lite->new( - Type => $type, - Encoding=> $encoding, - Data => $data); - $message->attr('content-type.charset' => $charset) if (($type =~ /^text\//i) and defined($charset)); - return $message; -} - -sub build_message -{ - my ($part_plain, $part_html, $part_body, $message); - - if (@attachments_inline) { - if (not defined($body_html)) { - die("Inline attachments (--attach-inline) must be used with --body-html\n"); - } - $part_html = MIME::Lite->new(Type => 'multipart/related'); - $part_html->attach(Type => 'text/html', Data => $body_html); - attach_attachments($part_html, "inline", @attachments_inline); - $message = $part_html; - # undefine $body_html to prevent confusion in the next if() - undef($body_html); - } - - if (defined($body_html)) { - $part_html = mime_message('text/html', $body_html); - $message = $part_html; - } - - if (defined($body_plain)) { - $part_plain = mime_message('text/plain', $body_plain); - $message = $part_plain; - } - - if (defined($part_plain) and defined($part_html)) { - $part_body = mime_message("multipart/alternative", undef); - safe_attach($part_body, $part_plain); - safe_attach($part_body, $part_html); - $message = $part_body; - } - - if (@attachments) { - if (defined($message)) { - # We already have some plaintext and/or html content built - # => make it the first part of multipart/mixed - my $message_body = $message; - $message = mime_message("multipart/mixed", undef); - safe_attach($message, $message_body); - attach_attachments($message, "attachment", @attachments); - } elsif ($#attachments == 0) { - # Only one single attachment - let it be the body - my ($path, $mime_type) = prepare_attachment($attachments[0]); - if (not defined($path)) { - die($attachments[0].": File not found. No other message parts defined. Aborting.\n"); - } - $message = MIME::Lite->new( - Type => $mime_type, - Path => $path); - } else { - # Message consisting only of attachments - $message = mime_message("multipart/mixed", undef); - attach_attachments($message, "attachment", @attachments); - } - } - - # Last resort - empty plaintext message - if (!defined($message)) { - $message = mime_message("TEXT", ""); - } - - $message->replace("From" => $from); - $message->replace("To" => join(", ", @to)); - $message->replace("Cc" => join(", ", @cc)); - $message->replace("Subject" => $subject); - $message->replace("X-Mailer" => "smtp-cli $version, see http://smtp-cli.logix.cz"); - $message->replace("Message-ID" => "<".time()."-".int(rand(999999))."\@smtp-cli>"); - - for my $header (@add_headers) { - my ($hdr, $val) = ($header =~ /^([^:]+):\s*(.*)$/); - die("Not a valid header format: ${header}\n") if (not $hdr or not $val); - $message->add($hdr => $val); - } - for my $header (@replace_headers) { - my ($hdr, $val) = ($header =~ /^([^:]+):\s*(.*)$/); - die("Not a valid header format: ${header}\n") if (not $hdr or not $val); - $message->replace($hdr => $val); - } - for my $header (@remove_headers) { - my ($hdr) = ($header =~ /^([^:\s]+)/); - $message->replace($header => ""); - } - - return $message; -} - -sub version () -{ - print "smtp-cli version $version\n"; - exit (0); -} - -sub usage () -{ - printf ( -"Simple SMTP client written in Perl that supports advanced -features like STARTTLS and SMTP-AUTH and IPv6. It can also -create messages from components (files, text snippets) and -attach files. - -Version: smtp-cli v$version - -Author: Michal Ludvig (c) 2003-2017 - http://smtp-cli.logix.cz - -Usage: smtp-cli [--options] - - --server=[:] - Host name or IP address of the SMTP server. - May include the port after colon, alternatively - use --port. - --port= Port where the SMTP server is listening. - (default: 25) - -4 or --ipv4 Use standard IP (IPv4) protocol. - -6 or --ipv6 Use IPv6 protocol. For hosts that have - both IPv6 and IPv4 addresses the IPv6 - connection is tried first. - --local-addr=
Specify local address (by default the OS chooses) - - --hello-host= String to use in the EHLO/HELO command. - --disable-ehlo Don't use ESMTP EHLO command, only HELO. - --force-ehlo Use EHLO even if server doesn't say ESMTP. - - Transport encryption (TLS) - --disable-starttls Don't use encryption even if the remote - host offers it. - --ssl Start in SMTP/SSL mode (aka SSMTP). - Default when --port=465 - --disable-ssl Don't start SSMTP even if --port=465 - --ssl-ca-file= - Verify the server's SSL certificate against - a trusted CA root certificate file. - --ssl-ca-path= Similar to --ssl-ca-file but will look for - the appropriate root certificate file in - the given directory. The certificates must - must be stored one per file with hash-links - generated by, for example, c_rehash script - from OpenSSL. - - Authentication options (AUTH) - --user= Username for SMTP authentication. - --pass= Corresponding password. - --auth-login Enable only AUTH LOGIN method. - --auth-plain Enable only AUTH PLAIN method. - --auth-cram-md5 Enable only AUTH CRAM-MD5 method. - --auth Enable all supported methods. This is - normally not needed, --user enables - everything as well. - - Sender / recipient - --from=\"Display Name \" - Sender's name address (or address only). - --to=\"Display Name \" - --cc=\"Display Name \" - --bcc=\"Display Name \" - Message recipients. Each parameter can be - used multiple times. - The --bcc addresses won't apprear in - the composed message. - - SMTP Envelope sender / recipient - (rarely needed, use --from, --to, --cc and --bcc instead) - --mail-from=
Address to use in MAIL FROM command. - Use --from instead, unless you want - a different address in the envelope and - in the headers. - --rcpt-to=
Address to use in RCPT TO command. Can be - used multiple times. Normally not needed, - use --to, --cc and --bcc instead. - If set the --to, --cc and --bcc will only - be used for composing the message body and - not for delivering the messages. - - Send a complete RFC822-compliant email message: - --data= Name of file to send after DATA command. - With \"--data=-\" the script will read - standard input (useful e.g. for pipes). - - Alternatively build email a message from provided components: - --subject= Subject of the message - --body-plain= - --body-html= - Plaintext and/or HTML body of the message - If both are provided the message is sent - as multipart. - --charset= Character set used for Subject and Body, - for example UTF-8, ISO-8859-2, KOI8-R, etc. - --text-encoding= - Enforce Content-Transfer-Encoding for text - parts of the email, including body and - attachments. Must be one of: - ".join(", ", @valid_encodings)." - The default is: quoted-printable - --attach=[\@] - Attach a given filename. - MIME-Type of the attachment is guessed - by default guessed but can optionally - be specified after '\@' delimiter. - For instance: --attach mail.log\@text/plain - Parameter can be used multiple times. - --attach-inline=[\@] - Attach a given filename (typically a picture) - as a 'related' part to the above 'body-html'. - Refer to these pictures as - in the 'body-html' contents. - See --attach for details about MIME-Type. - Can be used multiple times. - --add-header=\"Header: value\" - --replace-header=\"Header: value\" - --remove-header=\"Header\" - Add, Replace or Remove pretty much any header - in the email. For example to set a different - Mailer use --replace-header=\"X-Mailer: Blah\", - to remove it altogether --remove-header=X-Mailer - or to add a completely custom header use - --add-header=\"X-Something: foo bar\". - --print-only Dump the composed MIME message to standard - output. This is useful mainly for debugging - or in the case you need to run the message - through some filter before sending. - - Other options - --verbose[=] Be more verbose, print the SMTP session. - --missing-modules-ok Don't complain about missing optional modules. - --version Print: smtp-cli version $version - --help Guess what is this option for ;-) - -PayPal donations: http://smtp-cli.logix.cz/donate - Thanks in advance for your support! - -"); - exit (0); -} diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index eab3f5c..1f2b330 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -71,6 +71,8 @@ class WOStackController(CementBaseController): dict(help='Install WordOps dashboard', action='store_true')), (['--adminer'], dict(help='Install Adminer stack', action='store_true')), + (['--fail2ban'], + dict(help='Install Fail2ban stack', action='store_true')), (['--utils'], dict(help='Install Utils stack', action='store_true')), (['--redis'], @@ -1014,6 +1016,23 @@ class WOStackController(CementBaseController): WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") WOService.reload_service(self, 'mysql') + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + wo_fail2ban = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=wo_fail2ban) + wo_fail2ban.close() + + Log.debug(self, "Setting up fail2ban wp filter") + wo_fail2ban = open('/etc/fail2ban/filter.d/wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=wo_fail2ban) + wo_fail2ban.close() + if len(packages): if any('/usr/local/bin/wp' == x[1] for x in packages): Log.debug(self, "Setting Privileges" @@ -1441,6 +1460,15 @@ class WOStackController(CementBaseController): Log.debug(self, "WP-CLI is already installed") Log.info(self, "WP-CLI is already installed") + # fail2ban + if self.app.pargs.fail2ban: + Log.debug(self, "Setting apt_packages variable for Fail2ban") + if not WOAptGet.is_installed(self, 'fail2ban'): + apt_packages = apt_packages + WOVariables.wo_fail2ban + else: + Log.debug(self, "Fail2ban already installed") + Log.info(self, "Fail2ban already installed") + # PHPMYADMIN if self.app.pargs.phpmyadmin: Log.debug(self, "Setting packages variable for phpMyAdmin ") diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 8b81b86..872e518 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -26,7 +26,9 @@ class WOStackStatusController(CementBaseController): self.app.pargs.php73 or self.app.pargs.mysql or self.app.pargs.memcached or - self.app.pargs.redis): + self.app.pargs.redis or + self.app.pargs.fail2ban or + self.app.pargs.netdata): self.app.pargs.nginx = True self.app.pargs.php = True self.app.pargs.mysql = True @@ -78,6 +80,12 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "Redis server is not installed") + if self.app.pargs.fail2ban: + if WOAptGet.is_installed(self, 'fail2ban'): + services = services + ['fail2ban-client'] + else: + Log.info(self, "fail2ban is not installed") + for service in services: Log.debug(self, "Starting service: {0}".format(service)) WOService.start_service(self, service) @@ -143,6 +151,12 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "Redis server is not installed") + if self.app.pargs.fail2ban: + if WOAptGet.is_installed(self, 'fail2ban'): + services = services + ['fail2ban-client'] + else: + Log.info(self, "fail2ban is not installed") + for service in services: Log.debug(self, "Stopping service: {0}".format(service)) WOService.stop_service(self, service) @@ -155,7 +169,8 @@ class WOStackStatusController(CementBaseController): self.app.pargs.php73 or self.app.pargs.mysql or self.app.pargs.memcached or - self.app.pargs.redis): + self.app.pargs.redis or + self.app.pargs.fail2ban): self.app.pargs.nginx = True self.app.pargs.php = True self.app.pargs.mysql = True @@ -209,6 +224,12 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "Redis server is not installed") + if self.app.pargs.fail2ban: + if WOAptGet.is_installed(self, 'fail2ban'): + services = services + ['fail2ban-client'] + else: + Log.info(self, "fail2ban is not installed") + for service in services: Log.debug(self, "Restarting service: {0}".format(service)) WOService.restart_service(self, service) @@ -221,7 +242,8 @@ class WOStackStatusController(CementBaseController): self.app.pargs.php73 or self.app.pargs.mysql or self.app.pargs.memcached or - self.app.pargs.redis): + self.app.pargs.redis or + self.app.pargs.fail2ban): self.app.pargs.nginx = True self.app.pargs.php = True self.app.pargs.mysql = True @@ -274,6 +296,12 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "Redis server is not installed") + if self.app.pargs.fail2ban: + if WOAptGet.is_installed(self, 'fail2ban'): + services = services + ['fail2ban-client'] + else: + Log.info(self, "fail2ban is not installed") + for service in services: if WOService.get_service_status(self, service): Log.info(self, "{0:10}: {1}".format(service, "Running")) @@ -286,7 +314,8 @@ class WOStackStatusController(CementBaseController): self.app.pargs.php73 or self.app.pargs.mysql or self.app.pargs.memcached or - self.app.pargs.redis): + self.app.pargs.redis or + self.app.pargs.fail2ban): self.app.pargs.nginx = True self.app.pargs.php = True self.app.pargs.mysql = True @@ -340,6 +369,12 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "Redis server is not installed") + if self.app.pargs.fail2ban: + if WOAptGet.is_installed(self, 'fail2ban'): + services = services + ['fail2ban-client'] + else: + Log.info(self, "fail2ban is not installed") + for service in services: Log.debug(self, "Reloading service: {0}".format(service)) WOService.reload_service(self, service) From 5536a66a9d816b8191fb81308de3139216c82158 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 10:47:48 +0200 Subject: [PATCH 04/45] update ssl config * remove ssl on; for Nginx 1.16.0 * add ca certifcate * enable OSCP stapling --- wo/cli/plugins/site_functions.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 818e23e..c18581c 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1287,6 +1287,7 @@ def setupLetsEncrypt(self, wo_domain_name): "--key-file {0}/{1}/key.pem " "--fullchain-file " "{0}/{1}/fullchain.pem " + "--ca-file {0}/{1}/ca.pem " "--reloadcmd " "\"service nginx restart\" " .format(WOVariables.wo_ssl_live, @@ -1300,9 +1301,10 @@ def setupLetsEncrypt(self, wo_domain_name): encoding='utf-8', mode='w') sslconf.write("listen 443 ssl http2;\n" "listen [::]:443 ssl http2;\n" - "ssl on;\n" "ssl_certificate {0}/{1}/fullchain.pem;\n" "ssl_certificate_key {0}/{1}/key.pem;\n" + "ssl_trusted_certificate {0}/{1}/ca.pem;\n" + "ssl_stapling_verify on;\n" .format(WOVariables.wo_ssl_live, wo_domain_name)) sslconf.close() updateSiteInfo(self, wo_domain_name, ssl=True) @@ -1368,6 +1370,7 @@ def setupLetsEncryptSubdomain(self, wo_domain_name): "--key-file {0}/{1}/key.pem " "--fullchain-file " "{0}/{1}/fullchain.pem " + "--ca-file {0}/{1}/ca.pem " "--reloadcmd " "\"service nginx restart\" " .format(WOVariables.wo_ssl_live, @@ -1382,9 +1385,10 @@ def setupLetsEncryptSubdomain(self, wo_domain_name): encoding='utf-8', mode='w') sslconf.write("listen 443 ssl http2;\n" "listen [::]:443 ssl http2;\n" - "ssl on;\n" "ssl_certificate {0}/{1}/fullchain.pem;\n" "ssl_certificate_key {0}/{1}/key.pem;\n" + "ssl_trusted_certificate {0}/{1}/ca.pem;\n" + "ssl_stapling_verify on;\n" .format(WOVariables.wo_ssl_live, wo_domain_name)) sslconf.close() updateSiteInfo(self, wo_domain_name, ssl=True) @@ -1548,6 +1552,7 @@ def archivedCertificateHandle(self, domain): "--key-file {0}/{1}/key.pem " "--fullchain-file " "{0}/{1}/fullchain.pem " + "--ca-file {0}/{1}/ca.pem " "--reloadcmd " "\"service nginx restart\" " .format(WOVariables.wo_ssl_live, @@ -1567,10 +1572,11 @@ def archivedCertificateHandle(self, domain): encoding='utf-8', mode='w') sslconf.write("listen 443 ssl http2;\n" "listen [::]:443 ssl http2;\n" - "ssl on;\n" "ssl_certificate " "{0}/{1}/fullchain.pem;\n" "ssl_certificate_key {0}/{1}/key.pem;\n" + "ssl_trusted_certificate {0}/{1}/ca.pem;\n" + "ssl_stapling_verify on;\n" .format(WOVariables.wo_ssl_live, domain)) sslconf.close() @@ -1614,6 +1620,8 @@ def archivedCertificateHandle(self, domain): "--key-file {0}/{1}/key.pem " "--fullchain-file " "{0}/{1}/fullchain.pem " + "ssl_trusted_certificate " + "{0}/{1}/ca.pem;\n" "--reloadcmd " "\"service nginx restart\" " .format(WOVariables.wo_ssl_live, domain)) From 4ba8007a065ba494962f64b00444833b67a0f671 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 11:01:26 +0200 Subject: [PATCH 05/45] update version to prepare release --- install | 26 +++++++++++++------------- setup.py | 2 +- wo/core/variables.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/install b/install index 1ea406b..59f8d3e 100755 --- a/install +++ b/install @@ -7,10 +7,10 @@ # Copyright (c) 2019 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- -# Version 3.9.5 - 2019-04-22 +# Version 3.9.5 - 2019-04-25 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" -readonly wo_version_new="3.9.4.6" +readonly wo_version_new="3.9.5" # CONTENTS # --- # 1. VARIABLES AND DECLARATIONS @@ -380,15 +380,6 @@ wo_install_acme_sh() { # enable auto-upgrade /etc/letsencrypt/acme.sh --config-home /etc/letsencrypt/config --upgrade --auto-upgrade - # Let's Encrypt .well-known folder setup - if [ ! -d /var/www/html/.well-known/acme-challenge ]; then - mkdir -p /var/www/html/.well-known/acme-challenge - chown -R www-data:www-data /var/www/html /var/www/html/.well-known - chmod 750 /var/www/html /var/www/html/.well-known - else - chmod 750 /var/www/html /var/www/html/.well-known - fi - } >> "$wo_install_log" 2>&1 fi if [ -d "$HOME/.acme.sh" ]; then @@ -406,6 +397,14 @@ wo_install_acme_sh() { } >> "$wo_install_log" 2>&1 fi + # Let's Encrypt .well-known folder setup + if [ ! -d /var/www/html/.well-known/acme-challenge ]; then + mkdir -p /var/www/html/.well-known/acme-challenge + chown -R www-data:www-data /var/www/html /var/www/html/.well-known + chmod 750 /var/www/html /var/www/html/.well-known + else + chmod 750 /var/www/html /var/www/html/.well-known + fi } # Clone Github repository if it doesn't exist @@ -642,8 +641,9 @@ wo_clean_ee() { # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then - wo -v 2>&1 | grep $wo_version_new - if [[ $? -ne 0 ]]; then + if ! { + wo -v 2>&1 | grep $wo_version_new + }; then read -p "Update WordOps to $wo_version_new (y/n): " wo_ans if [ "$wo_ans" = "y" ] || [ "$wo_ans" = "Y" ]; then wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log diff --git a/setup.py b/setup.py index 39ecda6..82c3cde 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ if not os.path.isfile('/root/.gitconfig'): shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') setup(name='wo', - version='3.9.4', + version='3.9.5', description=long_description, long_description=long_description, classifiers=[], diff --git a/wo/core/variables.py b/wo/core/variables.py index 054ef53..68fab32 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -11,7 +11,7 @@ class WOVariables(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.4" + wo_version = "3.9.5" # WordOps packages versions wo_wp_cli = "2.1.0" wo_adminer = "4.7.1" From b892e171b8e257a4a25ece88b8744e5f4435f0cc Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 12:05:48 +0200 Subject: [PATCH 06/45] add python3-mysqldb for netdata --- wo/cli/plugins/stack.py | 4 ++++ wo/core/variables.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 1f2b330..5b29844 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1522,6 +1522,10 @@ class WOStackController(CementBaseController): 'kickstart-static64.sh', '/tmp/kickstart.sh', 'Netdata']] + if not WOAptGet.is_installed(self, 'python3-mysqldb'): + Log.debug( + self, "Setting apt_packages variable for Fail2ban") + apt_packages = apt_packages + WOVariables.wo_netdata # WordOps Dashboard if self.app.pargs.dashboard: diff --git a/wo/core/variables.py b/wo/core/variables.py index 68fab32..6b2556f 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -162,6 +162,8 @@ class WOVariables(): wo_fail2ban = "fail2ban" + wo_netdata = "python3-mysqldb" + # Redis repo details if wo_platform_distro == 'ubuntu': wo_redis_repo = ("ppa:chris-lea/redis-server") From 77aa92137c5dfe7f50e0c35689b10ea12699c467 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Apr 2019 13:02:11 +0200 Subject: [PATCH 07/45] move python3-mysqldb to mysql packages --- wo/cli/plugins/stack.py | 4 ---- wo/core/variables.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 5b29844..1f2b330 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1522,10 +1522,6 @@ class WOStackController(CementBaseController): 'kickstart-static64.sh', '/tmp/kickstart.sh', 'Netdata']] - if not WOAptGet.is_installed(self, 'python3-mysqldb'): - Log.debug( - self, "Setting apt_packages variable for Fail2ban") - apt_packages = apt_packages + WOVariables.wo_netdata # WordOps Dashboard if self.app.pargs.dashboard: diff --git a/wo/core/variables.py b/wo/core/variables.py index 6b2556f..fd12917 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -158,12 +158,10 @@ class WOVariables(): "10.3/debian {codename} main" .format(codename=wo_platform_codename)) - wo_mysql = ["mariadb-server", "percona-toolkit"] + wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb"] wo_fail2ban = "fail2ban" - wo_netdata = "python3-mysqldb" - # Redis repo details if wo_platform_distro == 'ubuntu': wo_redis_repo = ("ppa:chris-lea/redis-server") From 2f9b598113274b3bf462e92ca759a81f7e5b4a22 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 26 Apr 2019 21:30:46 +0200 Subject: [PATCH 08/45] Add fail2ban conf --- config/bash_completion.d/wo_auto.rc | 2 +- docs/wo.8 | 2 +- install | 15 +- wo/cli/plugins/update.py | 7 +- wo/cli/templates/fail2ban-forbidden.mustache | 4 + wo/cli/templates/fail2ban.mustache | 2 +- wo/cli/templates/fastcgi.mustache | 4 +- wo/cli/templates/sysctl.mustache | 266 +++++++++++++++++++ 8 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 wo/cli/templates/fail2ban-forbidden.mustache create mode 100644 wo/cli/templates/sysctl.mustache diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index 6a6e3a9..ead7ffc 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -74,7 +74,7 @@ _wo_complete() # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE "install" | "purge" | "remove" ) COMPREPLY=( $(compgen \ - -W "--web --admin --nginx --php --php73 --mysql --wpcli --phpmyadmin --adminer --utils --all --redis --phpredisadmin --composer --netdata --fail2ban" \ + -W "--web --admin --nginx --php --php73 --mysql --wpcli --phpmyadmin --adminer --utils --all --redis --phpredisadmin --composer --netdata --fail2ban --dashboard" \ -- $cur) ) ;; "upgrade" ) diff --git a/docs/wo.8 b/docs/wo.8 index 5c62929..f0d2891 100644 --- a/docs/wo.8 +++ b/docs/wo.8 @@ -5,7 +5,7 @@ .SH SYNOPSIS wo [ --version | --help | info | stack | site | debug | update | clean | import_slow_log | log | secure | sync | maintenance] .TP -wo stack [ install | remove | purge | migrate | upgrade] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils ] +wo stack [ install | remove | purge | migrate | upgrade] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils | --dashboard | --netdata ] .TP wo stack [ status | start | stop | reload | restart ] [--all | --nginx | --php | --php73 |--mysql | --web | --memcached | --redis] .TP diff --git a/install b/install index 59f8d3e..e7e6cf7 100755 --- a/install +++ b/install @@ -7,7 +7,7 @@ # Copyright (c) 2019 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- -# Version 3.9.5 - 2019-04-25 +# Version 3.9.5 - 2019-04-26 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" readonly wo_version_new="3.9.5" @@ -216,9 +216,6 @@ wo_sync_db() { # Copy the EasyEngine database cp /var/lib/ee/ee.db /var/lib/wo/dbase-ee.db - # Set the migration variable for the closing text - migration=1 - ### # Clean WO installation ### @@ -327,9 +324,9 @@ wo_sync_db() { secure_wo_db() { # The owner is root - chown -R root:root /var/lib/wo/ + chown -R root:root /var/lib/wo # Only allow access by root, block others - chmod -R 600 /var/lib/wo/ + chmod -R 600 /var/lib/wo } @@ -635,6 +632,12 @@ wo_clean_ee() { rm -f /usr/local/bin/ee /etc/bash_completion.d/ee_auto.rc /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee >> /var/log/wo/install.log 2>&1 } +# wo_tweak_kernel() { +# if [ ! -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf ]; then + +# fi +# } + ### # 4 - WO MAIN SETUP ### diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 4f589de..fa955bc 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -1,9 +1,10 @@ -from cement.core.controller import CementBaseController, expose +import os +import time + from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose from wo.core.download import WODownload from wo.core.logging import Log -import time -import os def wo_update_hook(app): diff --git a/wo/cli/templates/fail2ban-forbidden.mustache b/wo/cli/templates/fail2ban-forbidden.mustache new file mode 100644 index 0000000..6f708bf --- /dev/null +++ b/wo/cli/templates/fail2ban-forbidden.mustache @@ -0,0 +1,4 @@ +[Definition] +failregex = ^ \[error\] \d+#\d+: .* forbidden .*, client: , .*$ + +ignoreregex = diff --git a/wo/cli/templates/fail2ban.mustache b/wo/cli/templates/fail2ban.mustache index 10937f6..f7eb611 100644 --- a/wo/cli/templates/fail2ban.mustache +++ b/wo/cli/templates/fail2ban.mustache @@ -21,4 +21,4 @@ port = http,https logpath = /var/log/nginx/*error*.log findtime = 60 bantime = 6000 -maxretry = 3 \ No newline at end of file +maxretry = 5 \ No newline at end of file diff --git a/wo/cli/templates/fastcgi.mustache b/wo/cli/templates/fastcgi.mustache index 7300bb0..72cd66b 100644 --- a/wo/cli/templates/fastcgi.mustache +++ b/wo/cli/templates/fastcgi.mustache @@ -1,4 +1,4 @@ -# FastCGI cache settings - WO v3.9.5.1 +# FastCGI cache settings - WO v3.9.5 fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=60m max_size=256M; fastcgi_cache_key "$scheme$request_method$host$request_uri"; fastcgi_cache_use_stale error timeout invalid_header updating http_500 http_503; @@ -17,3 +17,5 @@ fastcgi_buffer_size 32k; fastcgi_param SERVER_NAME $http_host; fastcgi_ignore_headers Cache-Control Expires Set-Cookie; fastcgi_keep_conn on; +# only available with Nginx 1.15.6 and earlier +fastcgi_socket_keepalive on; diff --git a/wo/cli/templates/sysctl.mustache b/wo/cli/templates/sysctl.mustache new file mode 100644 index 0000000..cc2c332 --- /dev/null +++ b/wo/cli/templates/sysctl.mustache @@ -0,0 +1,266 @@ +# Kernel sysctl configuration file for Linux +# +# Version 1.16 - 2019-10-25 +# Michiel Klaver - IT Professional +# Modified by VirtuBox +# +# Instructions available on https://github.com/VirtuBox/ubuntu-nginx-web-server +# +# Sources : +# https://klaver.it/linux/sysctl.conf +# https://easyengine.io/tutorials/linux/sysctl-conf/ +# +# +# Credits: +# +# http://www.enigma.id.au/linux_tuning.txt +# http://www.securityfocus.com/infocus/1729 +# http://fasterdata.es.net/TCP-tuning/linux.html +# http://fedorahosted.org/ktune/browser/sysctl.ktune +# http://www.cymru.com/Documents/ip-stack-tuning.html +# http://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt +# http://www.frozentux.net/ipsysctl-tutorial/chunkyhtml/index.html +# http://knol.google.com/k/linux-performance-tuning-and-measurement +# http://www.cyberciti.biz/faq/linux-kernel-tuning-virtual-memory-subsystem/ +# http://www.redbooks.ibm.com/abstracts/REDP4285.html +# http://www.speedguide.net/read_articles.php?id=121 +# http://lartc.org/howto/lartc.kernel.obscure.html +# http://en.wikipedia.org/wiki/Sysctl +# +# Usage +# wget -O /etc/sysctl.d/60-ubuntu-nginx-web-server.conf https://virtubox.github.io/ubuntu-nginx-web-server/files/etc/sysctl.d/60-ubuntu-nginx-web-server.conf +# +# sysctl -e -p /etc/sysctl.d/60-ubuntu-nginx-web-server.conf +# For binary values, 0 is disabled, 1 is enabled. See sysctl(8) and sysctl.conf(5) for more details. +# + +### +### GENERAL SYSTEM SECURITY OPTIONS ### +### + +# Controls the System Request debugging functionality of the kernel +kernel.sysrq = 0 + +# Controls whether core dumps will append the PID to the core filename. +# Useful for debugging multi-threaded applications. +kernel.core_uses_pid = 1 + +#Allow for more PIDs +kernel.pid_max = 65535 + +# The contents of /proc//maps and smaps files are only visible to +# readers that are allowed to ptrace() the process +kernel.maps_protect = 1 + +#Enable ExecShield protection +kernel.exec-shield = 1 +kernel.randomize_va_space = 2 + +# Controls the maximum size of a message, in bytes +kernel.msgmnb = 65535 + +# Controls the default maxmimum size of a mesage queue +kernel.msgmax = 65535 + +# Restrict core dumps +fs.suid_dumpable = 0 + +# Hide exposed kernel pointers +kernel.kptr_restrict = 1 + +### +### IMPROVE SYSTEM MEMORY MANAGEMENT ### +### + +# Increase size of file handles and inode cache +fs.file-max = 209708 + +# Do less swapping +vm.swappiness = 10 +vm.dirty_ratio = 30 +vm.dirty_background_ratio = 5 + +# specifies the minimum virtual address that a process is allowed to mmap +vm.mmap_min_addr = 4096 + +# 50% overcommitment of available memory +vm.overcommit_ratio = 50 + +# allow memory overcommit required for redis +vm.overcommit_memory = 1 + +# Set maximum amount of memory allocated to shm to 256MB +kernel.shmmax = 268435456 +kernel.shmall = 268435456 + +# Keep at least 64MB of free RAM space available +vm.min_free_kbytes = 65535 + +### +### GENERAL NETWORK SECURITY OPTIONS ### +### + +#Prevent SYN attack, enable SYNcookies (they will kick-in when the max_syn_backlog reached) +net.ipv4.tcp_syncookies = 1 +net.ipv4.tcp_syn_retries = 2 +net.ipv4.tcp_synack_retries = 2 +net.ipv4.tcp_max_syn_backlog = 4096 + +# Disables IP source routing +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.default.send_redirects = 0 +net.ipv4.conf.all.accept_source_route = 0 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv6.conf.all.accept_source_route = 0 +net.ipv6.conf.default.accept_source_route = 0 + +# Enable IP spoofing protection, turn on source route verification +net.ipv4.conf.all.rp_filter = 1 +net.ipv4.conf.default.rp_filter = 1 + +# Disable ICMP Redirect Acceptance +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.default.accept_redirects = 0 +net.ipv4.conf.all.secure_redirects = 0 +net.ipv4.conf.default.secure_redirects = 0 +net.ipv6.conf.all.accept_redirects = 0 +net.ipv6.conf.default.accept_redirects = 0 + +# Enable Log Spoofed Packets, Source Routed Packets, Redirect Packets +net.ipv4.conf.all.log_martians = 1 +net.ipv4.conf.default.log_martians = 1 + +# Decrease the time default value for tcp_fin_timeout connection +net.ipv4.tcp_fin_timeout = 7 + +# Decrease the time default value for connections to keep alive +net.ipv4.tcp_keepalive_time = 300 +net.ipv4.tcp_keepalive_probes = 5 +net.ipv4.tcp_keepalive_intvl = 15 + +# Don't relay bootp +net.ipv4.conf.all.bootp_relay = 0 + +# Don't proxy arp for anyone +net.ipv4.conf.all.proxy_arp = 0 + +# Turn on the tcp_timestamps, accurate timestamp make TCP congestion control algorithms work better +net.ipv4.tcp_timestamps = 1 + +# Don't ignore directed pings +net.ipv4.icmp_echo_ignore_all = 0 + +# Enable ignoring broadcasts request +net.ipv4.icmp_echo_ignore_broadcasts = 1 + +# Enable bad error message Protection +net.ipv4.icmp_ignore_bogus_error_responses = 1 + +# Allowed local port range +net.ipv4.ip_local_port_range = 16384 65535 + +# Enable a fix for RFC1337 - time-wait assassination hazards in TCP +net.ipv4.tcp_rfc1337 = 1 + +# Do not auto-configure IPv6 +net.ipv6.conf.all.autoconf=0 +net.ipv6.conf.all.accept_ra=0 +net.ipv6.conf.default.autoconf=0 +net.ipv6.conf.default.accept_ra=0 +net.ipv6.conf.all.accept_ra_defrtr = 0 +net.ipv6.conf.default.accept_ra_defrtr = 0 +net.ipv6.conf.all.accept_ra_pinfo = 0 +net.ipv6.conf.default.accept_ra_pinfo = 0 + +### +### TUNING NETWORK PERFORMANCE ### +### + +# For servers with tcp-heavy workloads, enable 'fq' queue management scheduler (kernel > 3.12) +net.core.default_qdisc = fq + +# Turn on the tcp_window_scaling +net.ipv4.tcp_window_scaling = 1 + +# Increase the read-buffer space allocatable +net.ipv4.tcp_rmem = 8192 87380 16777216 +net.ipv4.udp_rmem_min = 16384 +net.core.rmem_default = 262144 +net.core.rmem_max = 16777216 + +# Increase the write-buffer-space allocatable +net.ipv4.tcp_wmem = 8192 65536 16777216 +net.ipv4.udp_wmem_min = 16384 +net.core.wmem_default = 262144 +net.core.wmem_max = 16777216 + +# Increase number of incoming connections +net.core.somaxconn = 32768 + +# Increase number of incoming connections backlog +net.core.netdev_max_backlog = 16384 +net.core.dev_weight = 64 + +# Increase the maximum amount of option memory buffers +net.core.optmem_max = 65535 + +# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks +net.ipv4.tcp_max_tw_buckets = 1440000 + +# try to reuse time-wait connections, but don't recycle them (recycle can break clients behind NAT) +net.ipv4.tcp_tw_recycle = 0 +net.ipv4.tcp_tw_reuse = 1 + +# Limit number of orphans, each orphan can eat up to 16M (max wmem) of unswappable memory +net.ipv4.tcp_max_orphans = 16384 +net.ipv4.tcp_orphan_retries = 0 + +# Limit the maximum memory used to reassemble IP fragments (CVE-2018-5391) +net.ipv4.ipfrag_low_thresh = 196608 +net.ipv6.ip6frag_low_thresh = 196608 +net.ipv4.ipfrag_high_thresh = 262144 +net.ipv6.ip6frag_high_thresh = 262144 + + +# don't cache ssthresh from previous connection +net.ipv4.tcp_no_metrics_save = 1 +net.ipv4.tcp_moderate_rcvbuf = 1 + +# Increase size of RPC datagram queue length +net.unix.max_dgram_qlen = 50 + +# Don't allow the arp table to become bigger than this +net.ipv4.neigh.default.gc_thresh3 = 2048 + +# Tell the gc when to become aggressive with arp table cleaning. +# Adjust this based on size of the LAN. 1024 is suitable for most /24 networks +net.ipv4.neigh.default.gc_thresh2 = 1024 + +# Adjust where the gc will leave arp table alone - set to 32. +net.ipv4.neigh.default.gc_thresh1 = 32 + +# Adjust to arp table gc to clean-up more often +net.ipv4.neigh.default.gc_interval = 30 + +# Increase TCP queue length +net.ipv4.neigh.default.proxy_qlen = 96 +net.ipv4.neigh.default.unres_qlen = 6 + +# Enable Explicit Congestion Notification (RFC 3168), disable it if it doesn't work for you +net.ipv4.tcp_ecn = 1 +net.ipv4.tcp_reordering = 3 + +# How many times to retry killing an alive TCP connection +net.ipv4.tcp_retries2 = 15 +net.ipv4.tcp_retries1 = 3 + +# Avoid falling back to slow start after a connection goes idle +# keeps our cwnd large with the keep alive connections (kernel > 3.6) +net.ipv4.tcp_slow_start_after_idle = 0 + +# Allow the TCP fastopen flag to be used, beware some firewalls do not like TFO! (kernel > 3.7) +net.ipv4.tcp_fastopen = 3 + +# This will enusre that immediatly subsequent connections use the new values +net.ipv4.route.flush = 1 +net.ipv6.route.flush = 1 From b0671c0e7492d09f40967aa58305b320e4f4be06 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sat, 27 Apr 2019 03:02:26 +0200 Subject: [PATCH 09/45] Update wordops version automatically --- wo/cli/plugins/stack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 1f2b330..d9e56f3 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -378,10 +378,12 @@ class WOStackController(CementBaseController): self.app.render((data), 'locations.mustache', out=wo_nginx) wo_nginx.close() + if not os.path.isfile("/etc/nginx/common/release"): with open("/etc/nginx/common/release", "a") as release_file: - release_file.write("v3.9.5") + release_file.write("v{0}" + .format(WOVariables.wo_version)) release_file.close() # Nginx-Plus does not have nginx From 490f2c80dfac8a9aa68629b79877a83c1a4937f6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 28 Apr 2019 21:28:13 +0200 Subject: [PATCH 10/45] compress mysqldump with pigz during site update --- wo/cli/plugins/site_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index c18581c..807716d 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -697,7 +697,7 @@ def sitebackup(self, data): if data['wo_db_name']: Log.info(self, 'Backing up database \t\t', end='') try: - if not WOShellExec.cmd_exec(self, "mysqldump {0} > {1}/{0}.sql" + if not WOShellExec.cmd_exec(self, "mysqldump --single-transaction {0} | pigz -9 -p\"$(nproc)\" > {1}/{0}.gz" .format(data['wo_db_name'], backup_path)): Log.info(self, From 8a29383ea8e0f14926a2a5a4413015db51c1089f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 28 Apr 2019 22:04:18 +0200 Subject: [PATCH 11/45] move return ssl --- wo/cli/plugins/site_functions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 807716d..cb20254 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1147,7 +1147,7 @@ def detSitePar(opts): def generate_random(): wo_random10 = (''.join(random.sample(string.ascii_uppercase + string.ascii_lowercase + - string.digits, 16))) + string.digits, 24))) return wo_random10 @@ -1408,8 +1408,6 @@ def setupLetsEncryptSubdomain(self, wo_domain_name): "\n to allow it to verify the site automatically.") # letsencrypt cert renewal - - def renewLetsEncrypt(self, wo_domain_name): ssl = WOShellExec.cmd_exec( @@ -1642,4 +1640,4 @@ def archivedCertificateHandle(self, domain): '/var/www/{0}/conf/nginx/ssl.conf.bak' .format(domain)) - return ssl + return ssl From 264d014c5d071b61854f1ae86ad20eae0e8a2f17 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 29 Apr 2019 00:45:02 +0200 Subject: [PATCH 12/45] reformat code * add security stack for futur usage * fix pep8 formatting issues --- CHANGELOG.md | 2 ++ README.md | 3 ++- install | 2 +- setup.py | 2 +- wo/cli/plugins/site_functions.py | 4 +++- wo/cli/plugins/stack.py | 2 +- wo/core/variables.py | 10 +++++----- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ead24bb..360a141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MySQL monitoring with Netdata - WordOps-dashboard on 22222 - Extplorer filemanager +- Enable OSCP Stapling with Let's Encrypt +- Compress database backup with pigz before updating sites #### Changed diff --git a/README.md b/README.md index e379646..8c028b7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ build MIT Commits -GitHub release +GitHub release +

diff --git a/install b/install index e7e6cf7..1dbc233 100755 --- a/install +++ b/install @@ -178,7 +178,7 @@ wo_install_dep() { if [ "$wo_linux_distro" == "Ubuntu" ]; then DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confold" -y install build-essential curl gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 else - wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg + [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confold" -y install build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-dev ca-certificates sqlite3 git tar software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 fi diff --git a/setup.py b/setup.py index 82c3cde..c302674 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ except Exception as e: print("Your informations will ONLY be stored locally") wo_user = input("Enter your name: ") - while wo_user is "": + while wo_user == "": print("Unfortunately, this can't be left blank") wo_user = input("Enter your name: ") diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index cb20254..693fbab 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -697,7 +697,9 @@ def sitebackup(self, data): if data['wo_db_name']: Log.info(self, 'Backing up database \t\t', end='') try: - if not WOShellExec.cmd_exec(self, "mysqldump --single-transaction {0} | pigz -9 -p\"$(nproc)\" > {1}/{0}.gz" + if not WOShellExec.cmd_exec(self, "mysqldump --single-transaction " + "{0} | pigz -9 -p\"$(nproc)\" " + "> {1}/{0}.gz" .format(data['wo_db_name'], backup_path)): Log.info(self, diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index d9e56f3..e4b2459 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1159,7 +1159,7 @@ class WOStackController(CementBaseController): WOFileUtils.searchreplace(self, "{0}22222/htdocs/index.php" .format(WOVariables.wo_webroot), "eth0", - "{0}".format(WOVariables.wo_wan_interface)) + "{0}".format(WOVariables.wo_wan)) Log.debug(self, "Setting Privileges to " "{0}22222/htdocs" diff --git a/wo/core/variables.py b/wo/core/variables.py index fd12917..1fae267 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -22,11 +22,11 @@ class WOVariables(): wo_wpcli_path = '/usr/local/bin/wp ' # get wan network interface name - wo_wan_interface = os.popen("ip -4 route get 8.8.8.8 | " - "grep -oP \"dev [^[:space:]]+ \" " - "| cut -d ' ' -f 2").read() - if wo_wan_interface == '': - wo_wan_interface = 'eth0' + wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " + "grep -oP \"dev [^[:space:]]+ \" " + "| cut -d ' ' -f 2").read() + if wo_wan == '': + wo_wan = 'eth0' # Current date and time of System wo_date = datetime.datetime.now().strftime('%d%b%Y%H%M%S') From 0508592d2a27497b8b741b6bddc60bb9db8f8ff5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 29 Apr 2019 00:54:37 +0200 Subject: [PATCH 13/45] fix wo_wan variable --- wo/cli/plugins/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index e4b2459..732fbea 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1155,7 +1155,7 @@ class WOStackController(CementBaseController): WOExtract.extract(self, '/tmp/wo-dashboard.tar.gz', '{0}22222/htdocs' .format(WOVariables.wo_webroot)) - if WOVariables.wo_wan_interface != 'eth0': + if WOVariables.wo_wan != 'eth0': WOFileUtils.searchreplace(self, "{0}22222/htdocs/index.php" .format(WOVariables.wo_webroot), "eth0", From 9de9ba8637d0ffd3fac7313d3f4f61ae29f52c73 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 29 Apr 2019 02:06:32 +0200 Subject: [PATCH 14/45] reformat code according to pep8 --- wo/cli/plugins/debug.py | 88 ++++++++++++++++----------------- wo/cli/plugins/log.py | 3 +- wo/cli/plugins/stack_upgrade.py | 3 +- wo/core/cron.py | 9 ++-- wo/core/mysql.py | 2 +- wo/core/services.py | 3 +- 6 files changed, 57 insertions(+), 51 deletions(-) diff --git a/wo/cli/plugins/debug.py b/wo/cli/plugins/debug.py index af1cfda..51178ae 100644 --- a/wo/cli/plugins/debug.py +++ b/wo/cli/plugins/debug.py @@ -191,22 +191,22 @@ class WODebugController(CementBaseController): nc.savef('/etc/nginx/conf.d/upstream.conf') # Enable xdebug - WOFileUtils.searchreplace(self, "/etc/{0}/mods-available/".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php/7.2") + + WOFileUtils.searchreplace(self, "/etc/{0}/" + "mods-available/".format("php/7.2") + "xdebug.ini", ";zend_extension", "zend_extension") # Fix slow log is not enabled default in PHP5.6 config = configparser.ConfigParser() - config.read('/etc/{0}/fpm/pool.d/debug.conf'.format( - "php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")) - config['debug']['slowlog'] = '/var/log/{0}/slow.log'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5") + config.read('/etc/{0}/fpm/pool.d/debug.conf'.format("php/7.2")) + config['debug']['slowlog'] = '/var/log/{0}/slow.log'.format( + "php/7.2") config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/{0}/fpm/pool.d/debug.conf'.format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5"), + with open('/etc/{0}/fpm/pool.d/debug.conf'.format("php/7.2"), encoding='utf-8', mode='w') as confifile: Log.debug(self, "Writting debug.conf configuration into " - "/etc/{0}/fpm/pool.d/debug.conf".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")) + "/etc/{0}/fpm/pool.d/debug.conf".format("php/7.2")) config.write(confifile) self.trigger_php = True @@ -214,8 +214,7 @@ class WODebugController(CementBaseController): else: Log.info(self, "PHP debug is already enabled") - self.msg = self.msg + ['/var/log/{0}/slow.log'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5")] + self.msg = self.msg + ['/var/log/{0}/slow.log'.format("php/7.2")] # PHP global debug stop elif (self.app.pargs.php == 'off' and not self.app.pargs.site_name): @@ -231,7 +230,8 @@ class WODebugController(CementBaseController): nc.savef('/etc/nginx/conf.d/upstream.conf') # Disable xdebug - WOFileUtils.searchreplace(self, "/etc/{0}/mods-available/".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5") + + WOFileUtils.searchreplace(self, "/etc/{0}/" + "mods-available/".format("php/7.2") + "xdebug.ini", "zend_extension", ";zend_extension") @@ -247,43 +247,41 @@ class WODebugController(CementBaseController): # PHP5-FPM start global debug if (self.app.pargs.fpm == 'on' and not self.app.pargs.site_name): if not WOShellExec.cmd_exec(self, "grep \"log_level = debug\" " - "/etc/{0}/fpm/php-fpm.conf".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")): + "/etc/{0}/" + "fpm/php-fpm.conf".format("php/7.2")): Log.info(self, "Setting up PHP5-FPM log_level = debug") config = configparser.ConfigParser() - config.read('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5")) + config.read('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2")) config.remove_option('global', 'include') config['global']['log_level'] = 'debug' - config['global']['include'] = '/etc/{0}/fpm/pool.d/*.conf'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5") - with open('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5"), + config['global']['include'] = '/etc/{0}/fpm/pool.d/*.conf'.format( + "php/7.2") + with open('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2"), encoding='utf-8', mode='w') as configfile: Log.debug(self, "Writting php5-FPM configuration into " - "/etc/{0}/fpm/php-fpm.conf".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")) + "/etc/{0}/fpm/php-fpm.conf".format("php/7.2")) config.write(configfile) self.trigger_php = True else: Log.info(self, "PHP5-FPM log_level = debug already setup") - self.msg = self.msg + ['/var/log/{0}/fpm.log'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5")] + self.msg = self.msg + ['/var/log/{0}/fpm.log'.format("php/7.2")] # PHP5-FPM stop global debug elif (self.app.pargs.fpm == 'off' and not self.app.pargs.site_name): if WOShellExec.cmd_exec(self, "grep \"log_level = debug\" " - "/etc/{0}/fpm/php-fpm.conf".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")): + "/etc/{0}/fpm/php-fpm.conf".format("php/7.2")): Log.info(self, "Disabling PHP5-FPM log_level = debug") config = configparser.ConfigParser() - config.read('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5")) + config.read('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2")) config.remove_option('global', 'include') config['global']['log_level'] = 'notice' - config['global']['include'] = '/etc/{0}/fpm/pool.d/*.conf'.format("php/7.2" if ( - WOVariables.wo_platform_distro == 'ubuntu') else "php5") - with open('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5"), + config['global']['include'] = '/etc/{0}/fpm/pool.d/*.conf'.format( + "php/7.2") + with open('/etc/{0}/fpm/php-fpm.conf'.format("php/7.2"), encoding='utf-8', mode='w') as configfile: Log.debug(self, "writting php5 configuration into " - "/etc/{0}/fpm/php-fpm.conf".format("php/7.2" if (WOVariables.wo_platform_distro == 'ubuntu') else "php5")) + "/etc/{0}/fpm/php-fpm.conf".format("php/7.2")) config.write(configfile) self.trigger_php = True @@ -335,7 +333,8 @@ class WODebugController(CementBaseController): # PHP global debug stop elif (self.app.pargs.php73 == 'off' and not self.app.pargs.site_name): - if WOShellExec.cmd_exec(self, " sed -n \"/upstream php72 {/,/}/p\" " + if WOShellExec.cmd_exec(self, " sed -n \"/upstream " + "php72 {/,/}/p\" " "/etc/nginx/conf.d/upstream.conf " "| grep 9172"): Log.info(self, "Disabling PHP 7.2 debug") @@ -399,7 +398,8 @@ class WODebugController(CementBaseController): config.write(configfile) self.trigger_php = True else: - Log.info(self, "PHP7.3-FPM log_level = debug already disabled") + Log.info(self, "PHP7.3-FPM log_level " + "= debug already disabled") @expose(hide=True) def debug_mysql(self): @@ -536,8 +536,8 @@ class WODebugController(CementBaseController): self.msg = self.msg + ['/var/log/nginx/*.error.log'] # Stop Nginx rewrite debug globally - elif (self.app.pargs.rewrite == 'off' - and not self.app.pargs.site_name): + elif (self.app.pargs.rewrite == 'off' and + not self.app.pargs.site_name): if WOShellExec.cmd_exec(self, "grep \"rewrite_log on;\" " "/etc/nginx/nginx.conf"): Log.info(self, "Disabling Nginx rewrite logs") @@ -644,10 +644,10 @@ class WODebugController(CementBaseController): (not self.app.pargs.fpm73) and (not self.app.pargs.mysql) and (not self.app.pargs.wp) and (not self.app.pargs.rewrite) and (not self.app.pargs.all) and (not self.app.pargs.site_name) and - (not self.app.pargs.import_slow_log) - and (not self.app.pargs.interval)): + (not self.app.pargs.import_slow_log) and + (not self.app.pargs.interval)): if self.app.pargs.stop or self.app.pargs.start: - print("--start/stop option is deprecated since ee v3.0.5") + print("--start/stop option is deprecated since wo v3.0.5") self.app.args.print_help() else: self.app.args.print_help() @@ -778,8 +778,8 @@ class WODebugController(CementBaseController): if len(self.msg) > 0: if not self.app.pargs.interactive: disp_msg = ' '.join(self.msg) - Log.info(self, "Use following command to check debug logs:\n" - + Log.ENDC + "tail -f {0}".format(disp_msg)) + Log.info(self, "Use following command to check debug logs:\n" + + Log.ENDC + "tail -f {0}".format(disp_msg)) else: signal.signal(signal.SIGINT, self.signal_handler) watch_list = [] @@ -797,18 +797,18 @@ class WODebugController(CementBaseController): # Get Anemometer user name and password Log.info(self, "Importing MySQL slow log to Anemometer") host = os.popen("grep -e \"\'host\'\" {0}22222/htdocs/" - .format(WOVariables.wo_webroot) - + "db/anemometer/conf/config.inc.php " + .format(WOVariables.wo_webroot) + + "db/anemometer/conf/config.inc.php " "| head -1 | cut -d\\\' -f4 | " "tr -d '\n'").read() user = os.popen("grep -e \"\'user\'\" {0}22222/htdocs/" - .format(WOVariables.wo_webroot) - + "db/anemometer/conf/config.inc.php " + .format(WOVariables.wo_webroot) + + "db/anemometer/conf/config.inc.php " "| head -1 | cut -d\\\' -f4 | " "tr -d '\n'").read() password = os.popen("grep -e \"\'password\'\" {0}22222/" - .format(WOVariables.wo_webroot) - + "htdocs/db/anemometer/conf" + .format(WOVariables.wo_webroot) + + "htdocs/db/anemometer/conf" "/config.inc.php " "| head -1 | cut -d\\\' -f4 | " "tr -d '\n'").read() @@ -836,9 +836,9 @@ class WODebugController(CementBaseController): " so not imported slow logs") else: Log.error(self, "Anemometer is not installed." + - Log.ENDC + "\n Install Anemometer with:" - + Log.BOLD + "\n `wo stack install --utils`" - + Log.ENDC) + Log.ENDC + "\n Install Anemometer with:" + + Log.BOLD + "\n `wo stack install --utils`" + + Log.ENDC) def load(app): diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index 5f7436f..5b08189 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -444,7 +444,8 @@ class WOLogMailController(CementBaseController): (['--nginx'], dict(help='Mail Nginx Error logs file', action='store_true')), (['--php'], - dict(help='Mail PHP 7.2 Error logs file', action='store_true')), + dict(help='Mail PHP 7.2 Error logs file', + action='store_true')), (['--fpm'], dict(help='Mail PHP 7.2-fpm slow logs file', action='store_true')), diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0e844aa..43250f8 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -107,7 +107,8 @@ class WOStackUpgradeController(CementBaseController): if self.app.pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): if not WOAptGet.is_installed(self, 'php7.3-fpm'): - apt_packages = apt_packages + WOVariables.wo_php + WOVariables.wo_php_extra + apt_packages = apt_packages + WOVariables.wo_php + \ + WOVariables.wo_php_extra else: apt_packages = apt_packages + WOVariables.wo_php else: diff --git a/wo/core/cron.py b/wo/core/cron.py index abd3bd6..02e9e81 100644 --- a/wo/core/cron.py +++ b/wo/core/cron.py @@ -7,8 +7,10 @@ Set CRON on LINUX system. class WOCron(): - def setcron_weekly(self, cmd, comment='Cron set by WordOps', user='root', min=0, hour=12): - if not WOShellExec.cmd_exec(self, "crontab -l | grep -q \'{0}\'".format(cmd)): + def setcron_weekly(self, cmd, comment='Cron set by WordOps', user='root', + min=0, hour=12): + if not WOShellExec.cmd_exec(self, "crontab -l " + "| grep -q \'{0}\'".format(cmd)): WOShellExec.cmd_exec(self, "/bin/bash -c \"crontab -l " "2> /dev/null | {{ cat; echo -e" @@ -20,7 +22,8 @@ class WOCron(): Log.debug(self, "Cron set") def remove_cron(self, cmd): - if WOShellExec.cmd_exec(self, "crontab -l | grep -q \'{0}\'".format(cmd)): + if WOShellExec.cmd_exec(self, "crontab -l " + "| grep -q \'{0}\'".format(cmd)): if not WOShellExec.cmd_exec(self, "/bin/bash -c " "\"crontab " "-l | sed '/{0}/d'" diff --git a/wo/core/mysql.py b/wo/core/mysql.py index 6b1f115..af2d638 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -112,7 +112,7 @@ class WOMysql(): " --single-transaction".format(dbs), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - p2 = subprocess.Popen("gzip -c > /var/wo-mysqlbackup/{0}{1}.s" + p2 = subprocess.Popen("pigz -c > /var/wo-mysqlbackup/{0}{1}.s" "ql.gz".format(dbs, WOVariables.wo_date), stdin=p1.stdout, shell=True) diff --git a/wo/core/services.py b/wo/core/services.py index c028880..a6e4c04 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -116,7 +116,8 @@ class WOService(): try: is_exist = subprocess.getstatusoutput('which {0}' .format(service_name)) - if is_exist[0] == 0 or service_name in ['php7.2-fpm', 'php7.3-fpm']: + if is_exist[0] == 0 or service_name in ['php7.2-fpm', + 'php7.3-fpm']: retcode = subprocess.getstatusoutput('service {0} status' .format(service_name)) if retcode[0] == 0: From 7cf6d440865aa0b6095a34e9a7388273f8439424 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 29 Apr 2019 13:13:44 +0200 Subject: [PATCH 15/45] Add support for Ubuntu 19.04 Disco --- CHANGELOG.md | 1 + README.md | 17 ++++++++++------- install | 4 ++-- wo/cli/plugins/stack.py | 31 ++++++++++++++++--------------- wo/core/variables.py | 5 ++++- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 360a141..8df5576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Extplorer filemanager - Enable OSCP Stapling with Let's Encrypt - Compress database backup with pigz before updating sites +- Add support for Ubuntu 19.04 disco #### Changed diff --git a/README.md b/README.md index 8c028b7..17df59f 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,14 @@

-build +build MIT Commits -GitHub release - +GitHub release +
+Badge Twitter +Badge Slack +

@@ -27,10 +30,9 @@ License

- WordOps site • -Documentation • -Community forum • -Slack + WordOps.net • +Documentation • +Community forum

@@ -53,6 +55,7 @@ - Ubuntu 16.04 LTS (Xenial) - Ubuntu 18.04 LTS (Bionic) +- Ubuntu 19.04 (Disco) - Debian 8 (Jessie) - Debian 9 (Stretch) diff --git a/install b/install index 1dbc233..1eb57b4 100755 --- a/install +++ b/install @@ -139,9 +139,9 @@ if [ -z "$wo_force_install" ]; then wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." exit 100 else - check_wo_linux_distro=$(lsb_release -sc | grep -E "trusty|xenial|bionic|jessie|stretch") + check_wo_linux_distro=$(lsb_release -sc | grep -E "trusty|xenial|bionic|disco|jessie|stretch") if [ -z "$check_wo_linux_distro" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04, Debian 8.x and Debian 9.x" + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04/19.04 LTS, Debian 8.x and Debian 9.x" exit 100 fi fi diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 732fbea..8eafe59 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -92,21 +92,22 @@ class WOStackController(CementBaseController): """Pre settings to do before installation packages""" if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - Log.info(self, "Adding repository for MySQL, please wait...") - mysql_pref = ("Package: *\nPin: origin " - "sfo1.mirrors.digitalocean.com" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'MariaDB.pref', 'w') as mysql_pref_file: - mysql_pref_file.write(mysql_pref) - WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVariables.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - WORepo.add_key(self, '0xF1656F24C74CD1D8', - keyserver="keyserver.ubuntu.com") - chars = ''.join(random.sample(string.ascii_letters, 16)) + if (WOVariables.wo_platform_codename != 'disco'): + Log.info(self, "Adding repository for MySQL, please wait...") + mysql_pref = ("Package: *\nPin: origin " + "sfo1.mirrors.digitalocean.com" + "\nPin-Priority: 1000\n") + with open('/etc/apt/preferences.d/' + 'MariaDB.pref', 'w') as mysql_pref_file: + mysql_pref_file.write(mysql_pref) + WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) + Log.debug(self, 'Adding key for {0}' + .format(WOVariables.wo_mysql_repo)) + WORepo.add_key(self, '0xcbcb082a1bb943db', + keyserver="keyserver.ubuntu.com") + WORepo.add_key(self, '0xF1656F24C74CD1D8', + keyserver="keyserver.ubuntu.com") + chars = ''.join(random.sample(string.ascii_letters, 24)) Log.debug(self, "Pre-seeding MySQL") Log.debug(self, "echo \"mariadb-server-10.3 " "mysql-server/root_password " diff --git a/wo/core/variables.py b/wo/core/variables.py index 1fae267..293d79a 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -13,7 +13,7 @@ class WOVariables(): # WordOps version wo_version = "3.9.5" # WordOps packages versions - wo_wp_cli = "2.1.0" + wo_wp_cli = "2.2.0" wo_adminer = "4.7.1" # Get WPCLI path @@ -102,6 +102,9 @@ class WOVariables(): elif wo_platform_codename == 'bionic': wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" "/virtubox:/WordOps/xUbuntu_18.04/ /") + elif wo_platform_codename == 'disco': + wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" + "/virtubox:/WordOps/xUbuntu_19.04/ /") elif wo_platform_codename == 'jessie': wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" "/virtubox:/WordOps/Debian_8.0/ /") From e16c61113b1b66f565e627c4f4c26c0d9b2f89f2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 29 Apr 2019 16:09:18 +0200 Subject: [PATCH 16/45] fix proxy webroot --- wo/cli/plugins/site.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 86ecb10..a21c198 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -147,10 +147,6 @@ class WOSiteController(CementBaseController): wo_db_user = siteinfo.db_user wo_db_pass = siteinfo.db_password wo_db_host = siteinfo.db_host - if sitetype == "proxy": - access_log = "/var/log/nginx/{0}.access.log".format(wo_domain) - error_log = "/var/log/nginx/{0}.error.log".format(wo_domain) - wo_site_webroot = '' php_version = siteinfo.php_version From cb84089e51877f5807ccca0ea66b3d49a0956a62 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 14:39:55 +0200 Subject: [PATCH 17/45] extract phpmemcachedadmin in the proper folder --- wo/cli/plugins/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 8eafe59..1cd236a 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1200,7 +1200,7 @@ class WOStackController(CementBaseController): " {0}22222/htdocs/cache/memcached " .format(WOVariables.wo_webroot)) WOExtract.extract(self, '/tmp/memcached.tar.gz', - '{0}22222/htdocs/cache/memcached' + '{0}22222/htdocs/cache/memcached/' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " "{0}22222/htdocs/cache/memcached file" From 9a55d394f120dbca54d877ca50a6d47c31ff7a7e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 16:05:02 +0200 Subject: [PATCH 18/45] Add certificate removal --- wo/cli/plugins/site_functions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 693fbab..f8fa3d9 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1221,6 +1221,19 @@ def removeNginxConf(self, domain): .format(domain)) +def removeAcmeConf(self, domain): + if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' + .format(domain)): + Log.debug(self, "Removing Acme configuration") + WOFileUtils.rm(self, '/etc/letsencrypt/renewal/{0}_ecc' + .format(domain)) + WOFileUtils.rm(self, '/etc/letsencrypt/live/{0}' + .format(domain)) + WOGit.add(self, ["/etc/letsencrypt"], + msg="Deleted {0} " + .format(domain)) + + def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', dbhost=''): """ From f8f73fd28ab3bbad7f48f839802032def5a1c160 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 16:06:16 +0200 Subject: [PATCH 19/45] add acme removal action --- wo/cli/plugins/site_functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index f8fa3d9..e23ee87 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1245,6 +1245,10 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', if os.path.isfile('/etc/nginx/sites-available/{0}' .format(domain)): removeNginxConf(self, domain) + if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' + .format(domain)): + removeAcmeConf(self, domain) + if webroot: deleteWebRoot(self, webroot) @@ -1423,6 +1427,8 @@ def setupLetsEncryptSubdomain(self, wo_domain_name): "\n to allow it to verify the site automatically.") # letsencrypt cert renewal + + def renewLetsEncrypt(self, wo_domain_name): ssl = WOShellExec.cmd_exec( From 98cd8f349d729a854a9842023b9320eabb94a344 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 16:55:14 +0200 Subject: [PATCH 20/45] Fix acme.sh migration and backup cert before update/upgrade --- install | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/install b/install index 1eb57b4..37bfa17 100755 --- a/install +++ b/install @@ -354,7 +354,7 @@ wo_update_wp_cli() { wo_install_acme_sh() { # check if acme.sh is already installed - if [ ! -d /opt/acme.sh ]; then + if [ ! -x /etc/letsencrypt/acme.sh ]; then { # clone the git repository git clone https://github.com/Neilpang/acme.sh.git /opt/acme.sh -q @@ -388,7 +388,6 @@ wo_install_acme_sh() { --exclude="dnsapi" \ --exclude="http.header" \ --exclude="ca" \ - --del \ "$HOME/.acme.sh/" \ /etc/letsencrypt/renewal/ @@ -621,11 +620,11 @@ wo_git_init() { } wo_backup_ee() { - tar -I pigz -cf "$EE_BACKUP_FILE" /etc/nginx /usr/local/bin/ee /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee >> /var/log/wo/install.log 2>&1 + tar -I pigz -cf "$EE_BACKUP_FILE" /etc/nginx /usr/local/bin/ee /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee /etc/letsencrypt >> /var/log/wo/install.log 2>&1 } wo_backup_wo() { - tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx/ /usr/local/lib/python3.6/dist-packages/wo-*.egg /etc/wo /var/lib/wo >> /var/log/wo/install.log 2>&1 + tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx/ /usr/local/lib/python3.6/dist-packages/wo-*.egg /etc/wo /var/lib/wo /etc/letsencrypt >> /var/log/wo/install.log 2>&1 } wo_clean_ee() { From 573b1a0665d8078c6a2205aa3d1b5c6555ed8116 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 18:44:35 +0200 Subject: [PATCH 21/45] Several fix and improvements * fix acme.sh migration with previous install * add support for raspbian (testing) * fix memcache folder in WordOps dashboard * improve install code quality --- install | 54 ++++++++++++++++++++++++++++------------- wo/cli/plugins/stack.py | 34 +++++++++++++++++--------- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/install b/install index 37bfa17..a534f34 100755 --- a/install +++ b/install @@ -133,15 +133,15 @@ fi # 1 - Checking linux distro ### if [ -z "$wo_force_install" ]; then - if [ "$wo_linux_distro" != "Ubuntu" ] && [ "$wo_linux_distro" != "Debian" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu and Debian at the moment." + if [ "$wo_linux_distro" != "Ubuntu" ] && [ "$wo_linux_distro" != "Debian" ] && [ "$wo_linux_distro" != "Raspbian" ]; then + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu, Debian & Raspbian at the moment." wo_lib_echo_fail "If you are feeling adventurous, you are free to fork WordOps to support" wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." exit 100 else check_wo_linux_distro=$(lsb_release -sc | grep -E "trusty|xenial|bionic|disco|jessie|stretch") if [ -z "$check_wo_linux_distro" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04/19.04 LTS, Debian 8.x and Debian 9.x" + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04/19.04 LTS, Debian 8.x, Debian 9.x and Raspbian 9.x" exit 100 fi fi @@ -357,7 +357,11 @@ wo_install_acme_sh() { if [ ! -x /etc/letsencrypt/acme.sh ]; then { # clone the git repository - git clone https://github.com/Neilpang/acme.sh.git /opt/acme.sh -q + if [ -d /opt/acme.sh/.git ]; then + git -C /opt/acme.sh pull origin master + else + git clone https://github.com/Neilpang/acme.sh.git /opt/acme.sh -q + fi cd /opt/acme.sh || exit 1 # create conf directories [ ! -d /etc/letsencrypt/config ] && { @@ -379,8 +383,11 @@ wo_install_acme_sh() { } >> "$wo_install_log" 2>&1 fi - if [ -d "$HOME/.acme.sh" ]; then + if [ -x "$HOME/.acme.sh/acme.sh" ]; then { + # backup acme.sh folder + /bin/tar -I pigz -cf /var/lib/wo-backup/acme.sh.tar.gz "$HOME/.acme.sh" + # rsync previous certificates to new acme.sh location /usr/bin/rsync -rltgoDpz --exclude="account.conf" \ --exclude="acme.sh" \ --exclude="acme.sh.env" \ @@ -390,6 +397,11 @@ wo_install_acme_sh() { --exclude="ca" \ "$HOME/.acme.sh/" \ /etc/letsencrypt/renewal/ + # remove previous acme.sh folder + rm -rf "$HOME/.acme.sh" + # create acme.sh.env file inlcuded in .bashrc to avoid error when logging in + mkdir -p "$HOME/.acme.sh" + echo '' > "$HOME/.acme.sh/acme.sh.env" } >> "$wo_install_log" 2>&1 fi @@ -546,7 +558,7 @@ wo_update_latest() { if [ -f /etc/ImageMagick/policy.xml ]; then if [ ! -f /etc/ImageMagick/patch.txt ]; then - echo -e "\t\n\t\n\t\n\t\n\t" >> /etc/ImageMagick/patch.txt + echo -e "\t\n\t\n\t\n\t\n\t" >> /etc/ImageMagick/patch.txt sed -i '//r /etc/ImageMagick/patch.txt' /etc/ImageMagick/policy.xml fi fi @@ -576,10 +588,10 @@ wo_update_latest() { # Fix Redis-server security issue # http://redis.io/topics/security if [ -f /etc/redis/redis.conf ]; then - grep -0 -v "#" /etc/redis/redis.conf | grep 'bind' >> /dev/null 2>&1 + CHECK_REDIS_BIND=$(grep -0 -v "#" /etc/redis/redis.conf | grep 'bind' >> /dev/null 2>&1) - if [ "$?" -ne 0 ]; then - sed -i '$ a bind 127.0.0.1' /etc/redis/redis.conf & + if [ -z "$CHECK_REDIS_BIND" ]; then + echo 'bind 127.0.0.1 ::1' >> /etc/redis/redis.conf service redis-server restart > /dev/null 2>&1 @@ -593,9 +605,9 @@ wo_git_init() { # Nginx under git version control [ -d /etc/nginx ] && { cd /etc/nginx || exit 1 - if [ ! -d /etc/nginx/.git ]; then + [ ! -d /etc/nginx/.git ] && { git init - fi + } git add -A . git commit -am "Updated Nginx" } >> /var/log/wo/install.log 2>&1 @@ -611,20 +623,20 @@ wo_git_init() { # PHP under git version control [ -d /etc/php ] && { cd /etc/php || exit 1 - if [ ! -d /etc/php/.git ]; then + [ ! -d /etc/php/.git ] && { git init - fi + } git add -A . git commit -am "Updated PHP" } >> /var/log/wo/install.log 2>&1 } wo_backup_ee() { - tar -I pigz -cf "$EE_BACKUP_FILE" /etc/nginx /usr/local/bin/ee /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee /etc/letsencrypt >> /var/log/wo/install.log 2>&1 + /bin/tar -I pigz -cf "$EE_BACKUP_FILE" /etc/nginx /usr/local/bin/ee /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee /etc/letsencrypt >> /var/log/wo/install.log 2>&1 } wo_backup_wo() { - tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx/ /usr/local/lib/python3.6/dist-packages/wo-*.egg /etc/wo /var/lib/wo /etc/letsencrypt >> /var/log/wo/install.log 2>&1 + /bin/tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx /usr/local/lib/python3.6/dist-packages/wo-*.egg /etc/wo /var/lib/wo /etc/letsencrypt >> /var/log/wo/install.log 2>&1 } wo_clean_ee() { @@ -655,7 +667,11 @@ if [ -x /usr/local/bin/wo ]; then wo_lib_echo "Syncing WO database" | tee -ai $wo_install_log secure_wo_db | tee -ai $wo_install_log wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log - wo_install >> wo_install_log 2>&1 + if [ -f "$HOME/.gitconfig" ]; then + wo_install >> wo_install_log 2>&1 + else + wo_install | tee -ai $wo_install_log + fi if [ -x "$(command -v nginx)" ]; then wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log wo_upgrade_nginx | tee -ai $wo_install_log @@ -685,7 +701,11 @@ else wo_sync_db | tee -ai $wo_install_log secure_wo_db | tee -ai $wo_install_log wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log - wo_install >> wo_install_log 2>&1 + if [ -f "$HOME/.gitconfig" ]; then + wo_install >> wo_install_log 2>&1 + else + wo_install | tee -ai $wo_install_log + fi if [ -x "$(command -v nginx)" ]; then wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log wo_upgrade_nginx | tee -ai $wo_install_log diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 1cd236a..f14a24a 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -92,7 +92,9 @@ class WOStackController(CementBaseController): """Pre settings to do before installation packages""" if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - if (WOVariables.wo_platform_codename != 'disco'): + # add mariadb repository excepted on raspbian and ubuntu 19.04 + if (not WOVariables.wo_platform_codename == 'disco') and + (not WOVariables.wo_platform_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") mysql_pref = ("Package: *\nPin: origin " "sfo1.mirrors.digitalocean.com" @@ -107,7 +109,9 @@ class WOStackController(CementBaseController): keyserver="keyserver.ubuntu.com") WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver="keyserver.ubuntu.com") + # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) + # configure MySQL non-interactive install Log.debug(self, "Pre-seeding MySQL") Log.debug(self, "echo \"mariadb-server-10.3 " "mysql-server/root_password " @@ -136,7 +140,7 @@ class WOStackController(CementBaseController): log=False) except CommandExecutionError as e: Log.error("Failed to initialize MySQL package") - + # generate my.cnf root credentials mysql_config = """ [client] user = root @@ -153,12 +157,14 @@ class WOStackController(CementBaseController): Log.debug(self, 'Setting my.cnf permission') WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) + # add nginx repository if set(WOVariables.wo_nginx).issubset(set(apt_packages)): Log.info(self, "Adding repository for NGINX, please wait...") WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') WORepo.add_key(self, WOVariables.wo_nginx_key) + # add php repository if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or set(WOVariables.wo_php).issubset(set(apt_packages))): if (WOVariables.wo_platform_distro == 'ubuntu'): @@ -172,7 +178,7 @@ class WOStackController(CementBaseController): WORepo.add(self, repo_url=WOVariables.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') WORepo.add_key(self, WOVariables.wo_php_key) - + # add redis repository if set(WOVariables.wo_redis).issubset(set(apt_packages)): Log.info(self, "Adding repository for Redis, please wait...") if WOVariables.wo_platform_distro == 'ubuntu': @@ -644,7 +650,7 @@ class WOStackController(CementBaseController): "/var/run/php/php73-fpm.sock;\n}\n" "upstream debug73 {\nserver " "127.0.0.1:9173;\n}\n") - + # create nginx configuration for redis if set(WOVariables.wo_redis).issubset(set(apt_packages)): if (os.path.isfile("/etc/nginx/nginx.conf") and not os.path.isfile("/etc/nginx/common/" @@ -994,6 +1000,7 @@ class WOStackController(CementBaseController): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.3-fpm') + # create mysql config if it doesn't exist if set(WOVariables.wo_mysql).issubset(set(apt_packages)): if not os.path.isfile("/etc/mysql/my.cnf"): config = ("[mysqld]\nwait_timeout = 30\n" @@ -1019,6 +1026,7 @@ class WOStackController(CementBaseController): WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") WOService.reload_service(self, 'mysql') + # create fail2ban configuration files if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): data = dict() @@ -1197,13 +1205,13 @@ class WOStackController(CementBaseController): if any('/tmp/memcached.tar.gz' == x[1] for x in packages): Log.debug(self, "Extracting memcached.tar.gz to location" - " {0}22222/htdocs/cache/memcached " + " {0}22222/htdocs/cache/memcache " .format(WOVariables.wo_webroot)) WOExtract.extract(self, '/tmp/memcached.tar.gz', - '{0}22222/htdocs/cache/memcached/' + '{0}22222/htdocs/cache/memcache/' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/cache/memcached file" + "{0}22222/htdocs/cache/memcache file" .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222' .format(WOVariables.wo_webroot), @@ -1309,7 +1317,7 @@ class WOStackController(CementBaseController): if any('/usr/bin/pt-query-advisor' == x[1] for x in packages): WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) - # ph + # phpredisadmin if any('/tmp/pra.tar.gz' == x[1] for x in packages): if not os.path.exists('{0}22222/htdocs/cache/redis' @@ -1479,8 +1487,10 @@ class WOStackController(CementBaseController): packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", "/tmp/pma.tar.gz", "phpMyAdmin"], - ["https://getcomposer.org/installer", - "/tmp/composer-install", "Composer"]] + ["https://getcomposer.org/" + "installer", + "/tmp/composer-install", + "Composer"]] else: packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", @@ -1759,7 +1769,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot), '{0}22222/htdocs/cache/nginx/' 'clean.php'.format(WOVariables.wo_webroot), - '{0}22222/htdocs/cache/memcached' + '{0}22222/htdocs/cache/memcache' .format(WOVariables.wo_webroot), '/usr/bin/pt-query-advisor', '{0}22222/htdocs/db/anemometer' @@ -1891,7 +1901,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot), '{0}22222/htdocs/cache/nginx/' 'clean.php'.format(WOVariables.wo_webroot), - '{0}22222/htdocs/cache/memcached' + '{0}22222/htdocs/cache/memcache' .format(WOVariables.wo_webroot), '/usr/bin/pt-query-advisor', '{0}22222/htdocs/db/anemometer' From 3e454e66f248e7a11361b80d83782c610745207a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 18:50:30 +0200 Subject: [PATCH 22/45] remove "len" usage in stack --- wo/cli/plugins/stack.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index f14a24a..2512bf2 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -190,7 +190,7 @@ class WOStackController(CementBaseController): @expose(hide=True) def post_pref(self, apt_packages, packages): """Post activity after installation of packages""" - if len(apt_packages): + if (apt_packages): if set(WOVariables.wo_nginx).issubset(set(apt_packages)): if set(["nginx"]).issubset(set(apt_packages)): @@ -1044,7 +1044,7 @@ class WOStackController(CementBaseController): out=wo_fail2ban) wo_fail2ban.close() - if len(packages): + if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): Log.debug(self, "Setting Privileges" " to /usr/local/bin/wp file ") @@ -1601,16 +1601,16 @@ class WOStackController(CementBaseController): except Exception as e: pass - if len(apt_packages) or len(packages): + if (apt_packages) or (packages): Log.debug(self, "Calling pre_pref") self.pre_pref(apt_packages) - if len(apt_packages): + if (apt_packages): WOSwap.add(self) Log.info(self, "Updating apt-cache, please wait...") WOAptGet.update(self) Log.info(self, "Installing packages, please wait...") WOAptGet.install(self, apt_packages) - if len(packages): + if (packages): Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) Log.debug(self, "Calling post_pref") @@ -1657,7 +1657,7 @@ class WOStackController(CementBaseController): "/etc/redis/redis.conf") WOService.restart_service(self, 'redis-server') if disp_msg: - if len(self.msg): + if (self.msg): for msg in self.msg: Log.info(self, Log.ENDC + msg) Log.info(self, "Successfully installed packages") @@ -1775,7 +1775,7 @@ class WOStackController(CementBaseController): '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)] - if len(packages) or len(apt_packages): + if (packages) or (apt_packages): wo_prompt = input('Are you sure you to want to' ' remove from server.' '\nPackage configuration will remain' @@ -1789,11 +1789,11 @@ class WOStackController(CementBaseController): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') - if len(packages): + if (packages): WOFileUtils.remove(self, packages) WOAptGet.auto_remove(self) - if len(apt_packages): + if (apt_packages): Log.debug(self, "Removing apt_packages") Log.info(self, "Removing packages, please wait...") WOAptGet.remove(self, apt_packages) @@ -1908,7 +1908,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot) ] - if len(packages) or len(apt_packages): + if (packages) or (apt_packages): wo_prompt = input('Are you sure you to want to purge ' 'from server ' 'along with their configuration' @@ -1921,12 +1921,12 @@ class WOStackController(CementBaseController): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') - if len(apt_packages): + if (apt_packages): Log.info(self, "Purging packages, please wait...") WOAptGet.remove(self, apt_packages, purge=True) WOAptGet.auto_remove(self) - if len(packages): + if (packages): WOFileUtils.remove(self, packages) WOAptGet.auto_remove(self) From 8c21974f0e9a5c074d6ad47aa2ba3cedc6db7654 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 18:51:42 +0200 Subject: [PATCH 23/45] fix identation --- wo/cli/plugins/stack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 2512bf2..715d0a7 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -93,8 +93,8 @@ class WOStackController(CementBaseController): if set(WOVariables.wo_mysql).issubset(set(apt_packages)): # add mariadb repository excepted on raspbian and ubuntu 19.04 - if (not WOVariables.wo_platform_codename == 'disco') and - (not WOVariables.wo_platform_distro == 'raspbian'): + if ((not WOVariables.wo_platform_codename == 'disco') and + (not WOVariables.wo_platform_distro == 'raspbian')): Log.info(self, "Adding repository for MySQL, please wait...") mysql_pref = ("Package: *\nPin: origin " "sfo1.mirrors.digitalocean.com" From 09e764cba7330f9bcc848857141d2c5429c00712 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 19:00:14 +0200 Subject: [PATCH 24/45] improve code quality according to codacy --- wo/cli/plugins/info.py | 20 ++++++++++---------- wo/cli/plugins/stack.py | 1 - 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index d20d5a6..2d57e31 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -41,7 +41,7 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_nginx(self): """Display Nginx information""" - version = os.popen("nginx -v 2>&1 | awk -F '/' '{print $2}' | " + version = os.popen("/usr/sbin/nginx -v 2>&1 | awk -F '/' '{print $2}' | " "awk -F ' ' '{print $1}' | tr '\n' ' '").read() allow = os.popen("grep ^allow /etc/nginx/common/acl.conf | " "cut -d' ' -f2 | cut -d';' -f1 | tr '\n' ' '").read() @@ -66,7 +66,7 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_php(self): """Display PHP information""" - version = os.popen("php7.2 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" + version = os.popen("/usr/bin/php7.2 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" " cut -d'+' -f1 | tr -d '\n'").read config = configparser.ConfigParser() config.read('/etc/{0}/fpm/php.ini'.format("php/7.2")) @@ -140,7 +140,7 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_php73(self): """Display PHP information""" - version = os.popen("php7.3 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" + version = os.popen("/usr/bin/php7.3 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" " cut -d'+' -f1 | tr -d '\n'").read config = configparser.ConfigParser() config.read('/etc/php/7.3/fpm/php.ini') @@ -214,23 +214,23 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_mysql(self): """Display MySQL information""" - version = os.popen("mysql -V | awk '{print($5)}' | cut -d ',' " + version = os.popen("/usr/bin/mysql -V | awk '{print($5)}' | cut -d ',' " "-f1 | tr -d '\n'").read() host = "localhost" - port = os.popen("mysql -e \"show variables\" | grep ^port | awk " + port = os.popen("/usr/bin/mysql -e \"show variables\" | grep ^port | awk " "'{print($2)}' | tr -d '\n'").read() - wait_timeout = os.popen("mysql -e \"show variables\" | grep " + wait_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " "^wait_timeout | awk '{print($2)}' | " "tr -d '\n'").read() - interactive_timeout = os.popen("mysql -e \"show variables\" | grep " + interactive_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " "^interactive_timeout | awk " "'{print($2)}' | tr -d '\n'").read() - max_used_connections = os.popen("mysql -e \"show global status\" | " + max_used_connections = os.popen("/usr/bin/mysql -e \"show global status\" | " "grep Max_used_connections | awk " "'{print($2)}' | tr -d '\n'").read() - datadir = os.popen("mysql -e \"show variables\" | grep datadir | awk" + datadir = os.popen("/usr/bin/mysql -e \"show variables\" | grep datadir | awk" " '{print($2)}' | tr -d '\n'").read() - socket = os.popen("mysql -e \"show variables\" | grep \"^socket\" | " + socket = os.popen("/usr/bin/mysql -e \"show variables\" | grep \"^socket\" | " "awk '{print($2)}' | tr -d '\n'").read() data = dict(version=version, host=host, port=port, wait_timeout=wait_timeout, diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 715d0a7..f0dbfc3 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -16,7 +16,6 @@ from wo.core.git import WOGit from wo.core.checkfqdn import check_fqdn from pynginxconfig import NginxConfig from wo.core.services import WOService -from wo.core.variables import WOVariables import random import string import configparser From b1e6f22e319634d37540613bffe10f2bfa90f892 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 19:05:54 +0200 Subject: [PATCH 25/45] [skip travis] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8df5576..d9365a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enable OSCP Stapling with Let's Encrypt - Compress database backup with pigz before updating sites - Add support for Ubuntu 19.04 disco +- Add support for Raspbian +- backup letsencrypt certificate before upgrade #### Changed @@ -70,6 +72,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - phpredisadmin setup - --hsts flag with basic html site - hsts flag on site not secure with letsencrypt +- fix import of previous acme.sh certificate ### v3.9.4 - 2019-03-15 From b6542bbb34e2edc9c6ae952a98a5ec42e31d6ae4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 30 Apr 2019 19:55:55 +0200 Subject: [PATCH 26/45] improve code quality according to codacy --- wo/cli/plugins/info.py | 27 ++++++++++++++++++--------- wo/core/apt_repo.py | 16 ++++++++-------- wo/core/aptget.py | 24 ++++++++++++++++-------- wo/core/mysql.py | 10 ++++++---- wo/core/sslutils.py | 6 ++++-- wo/core/variables.py | 5 +++-- 6 files changed, 55 insertions(+), 33 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 2d57e31..9e071d7 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -41,7 +41,8 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_nginx(self): """Display Nginx information""" - version = os.popen("/usr/sbin/nginx -v 2>&1 | awk -F '/' '{print $2}' | " + version = os.popen("/usr/sbin/nginx -v 2>&1 | " + "awk -F '/' '{print $2}' | " "awk -F ' ' '{print $1}' | tr '\n' ' '").read() allow = os.popen("grep ^allow /etc/nginx/common/acl.conf | " "cut -d' ' -f2 | cut -d';' -f1 | tr '\n' ' '").read() @@ -66,7 +67,8 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_php(self): """Display PHP information""" - version = os.popen("/usr/bin/php7.2 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" + version = os.popen("/usr/bin/php7.2 -v 2>/dev/null | " + "head -n1 | cut -d' ' -f2 |" " cut -d'+' -f1 | tr -d '\n'").read config = configparser.ConfigParser() config.read('/etc/{0}/fpm/php.ini'.format("php/7.2")) @@ -140,7 +142,8 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_php73(self): """Display PHP information""" - version = os.popen("/usr/bin/php7.3 -v 2>/dev/null | head -n1 | cut -d' ' -f2 |" + version = os.popen("/usr/bin/php7.3 -v 2>/dev/null | " + "head -n1 | cut -d' ' -f2 |" " cut -d'+' -f1 | tr -d '\n'").read config = configparser.ConfigParser() config.read('/etc/php/7.3/fpm/php.ini') @@ -214,23 +217,29 @@ class WOInfoController(CementBaseController): @expose(hide=True) def info_mysql(self): """Display MySQL information""" - version = os.popen("/usr/bin/mysql -V | awk '{print($5)}' | cut -d ',' " + version = os.popen("/usr/bin/mysql -V | awk '{print($5)}' | " + "cut -d ',' " "-f1 | tr -d '\n'").read() host = "localhost" - port = os.popen("/usr/bin/mysql -e \"show variables\" | grep ^port | awk " + port = os.popen("/usr/bin/mysql -e \"show variables\" | " + "grep ^port | awk " "'{print($2)}' | tr -d '\n'").read() wait_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " "^wait_timeout | awk '{print($2)}' | " "tr -d '\n'").read() - interactive_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " + interactive_timeout = os.popen("/usr/bin/mysql -e " + "\"show variables\" | grep " "^interactive_timeout | awk " "'{print($2)}' | tr -d '\n'").read() - max_used_connections = os.popen("/usr/bin/mysql -e \"show global status\" | " + max_used_connections = os.popen("/usr/bin/mysql -e " + "\"show global status\" | " "grep Max_used_connections | awk " "'{print($2)}' | tr -d '\n'").read() - datadir = os.popen("/usr/bin/mysql -e \"show variables\" | grep datadir | awk" + datadir = os.popen("/usr/bin/mysql -e \"show variables\" | " + "grep datadir | awk" " '{print($2)}' | tr -d '\n'").read() - socket = os.popen("/usr/bin/mysql -e \"show variables\" | grep \"^socket\" | " + socket = os.popen("/usr/bin/mysql -e \"show variables\" | " + "grep \"^socket\" | " "awk '{print($2)}' | tr -d '\n'").read() data = dict(version=version, host=host, port=port, wait_timeout=wait_timeout, diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index 4c89dae..cfd1f22 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -23,8 +23,8 @@ class WORepo(): """ if repo_url is not None: - repo_file_path = ("/etc/apt/sources.list.d/" - + WOVariables().wo_repo_file) + repo_file_path = ("/etc/apt/sources.list.d/" + + WOVariables().wo_repo_file) try: if not os.path.isfile(repo_file_path): with open(repo_file_path, @@ -62,8 +62,8 @@ class WORepo(): "--remove '{ppa_name}'" .format(ppa_name=ppa)) elif repo_url: - repo_file_path = ("/etc/apt/sources.list.d/" - + WOVariables().wo_repo_file) + repo_file_path = ("/etc/apt/sources.list.d/" + + WOVariables().wo_repo_file) try: repofile = open(repo_file_path, "w+") @@ -84,8 +84,8 @@ class WORepo(): """ WOShellExec.cmd_exec(self, "gpg --keyserver {serv}" .format(serv=(keyserver or - "hkp://keyserver.ubuntu.com")) - + " --recv-keys {key}".format(key=keyids)) + "hkp://keyserver.ubuntu.com")) + + " --recv-keys {key}".format(key=keyids)) WOShellExec.cmd_exec(self, "gpg -a --export --armor {0}" - .format(keyids) - + " | apt-key add - ") + .format(keyids) + + " | apt-key add - ") diff --git a/wo/core/aptget.py b/wo/core/aptget.py index bacaef4..a480202 100644 --- a/wo/core/aptget.py +++ b/wo/core/aptget.py @@ -51,7 +51,8 @@ class WOAptGet(): Log.info(self, Log.FAIL + "Whoops, something went wrong...") Log.error(self, "Check the WordOps log for more details " - "`tail /var/log/wo/wordops.log` and please try again...") + "`tail /var/log/wo/wordops.log` " + "and please try again...") except Exception as e: Log.error(self, "apt-get update exited with error") @@ -98,7 +99,8 @@ class WOAptGet(): Log.info(self, Log.FAIL + "Oops Something went " "wrong!!") Log.error(self, "Check the WordOps log for more details " - "`tail /var/log/wo/wordops.log` and please try again...") + "`tail /var/log/wo/wordops.log` " + "and please try again...") except Exception as e: Log.error(self, "Error while installing packages, " "apt-get exited with error") @@ -124,13 +126,15 @@ class WOAptGet(): Log.info(self, Log.FAIL + "Oops Something went " "wrong!!") Log.error(self, "Check the WordOps log for more details " - "`tail /var/log/wo/wordops.log` and please try again...") + "`tail /var/log/wo/wordops.log` " + "and please try again...") except Exception as e: Log.info(self, Log.FAIL + "Oops Something went " "wrong!!") Log.error(self, "Check the WordOps log for more details " - "`tail /var/log/wo/wordops.log` and please try again...") + "`tail /var/log/wo/wordops.log` " + "and please try again...") def remove(self, packages, auto=False, purge=False): all_packages = ' '.join(packages) @@ -153,7 +157,8 @@ class WOAptGet(): Log.info(self, Log.FAIL + "Oops Something went " "wrong!!") Log.error(self, "Check the WordOps log for more details " - "`tail /var/log/wo/wordops.log` and please try again...") + "`tail /var/log/wo/wordops.log` " + "and please try again...") except Exception as e: Log.error(self, "Error while installing packages, " @@ -186,7 +191,8 @@ class WOAptGet(): def is_installed(self, package_name): """ - Checks if package is available in cache and is installed or not + Checks if package is available in cache " + "and is installed or not returns True if installed otherwise returns False """ apt_cache = apt.cache.Cache() @@ -209,7 +215,8 @@ class WOAptGet(): WORepo.add(self, repo_url=repo_url) if repo_key is not None: WORepo.add_key(self, repo_key) - proc = subprocess.Popen("apt-get update && DEBIAN_FRONTEND=noninteractive " + proc = subprocess.Popen("apt-get update && " + "DEBIAN_FRONTEND=noninteractive " "apt-get install -o " "Dpkg::Options::=\"--force-confdef\"" " -o " @@ -224,7 +231,8 @@ class WOAptGet(): return True else: Log.error( - self, "Error in fetching dpkg package.\nReverting changes ..", False) + self, "Error in fetching dpkg package.\n" + "Reverting changes ..", False) if repo_url is not None: WORepo.remove(self, repo_url=repo_url) return False diff --git a/wo/core/mysql.py b/wo/core/mysql.py index af2d638..c289405 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -100,20 +100,22 @@ class WOMysql(): '/var/wo-mysqlbackup') os.makedirs('/var/wo-mysqlbackup') - db = subprocess.check_output(["mysql -Bse \'show databases\'"], + db = subprocess.check_output(["/usr/bin/mysql " + "-Bse \'show databases\'"], universal_newlines=True, shell=True).split('\n') for dbs in db: if dbs == "": continue Log.info(self, "Backing up {0} database".format(dbs)) - p1 = subprocess.Popen("mysqldump {0}" + p1 = subprocess.Popen("/usr/bin/mysqldump {0}" " --max_allowed_packet=1024M" " --single-transaction".format(dbs), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - p2 = subprocess.Popen("pigz -c > /var/wo-mysqlbackup/{0}{1}.s" - "ql.gz".format(dbs, WOVariables.wo_date), + p2 = subprocess.Popen("/usr/bin/pigz -c > " + "/var/wo-mysqlbackup/{0}{1}.sql.gz" + .format(dbs, WOVariables.wo_date), stdin=p1.stdout, shell=True) diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index 161a484..98f33d5 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -19,7 +19,8 @@ class SSL: current_date = WOShellExec.cmd_exec_stdout(self, "date -d \"now\" +%s") expiration_date = WOShellExec.cmd_exec_stdout(self, "date -d \"" - "`openssl x509 -in " + "`/usr/bin/openssl " + "x509 -in " "/etc/letsencrypt/live/" "{0}/cert.pem" " -text -noout|grep " @@ -45,7 +46,8 @@ class SSL: "`tail /var/log/wo/wordops.log` and please try again...") expiration_date = WOShellExec.cmd_exec_stdout(self, "date -d " - "\"`openssl x509 -in " + "\"`/usr/bin/openssl " + "x509 -in " "/etc/letsencrypt/live/" "{0}/cert.pem" " -text -noout|grep " diff --git a/wo/core/variables.py b/wo/core/variables.py index 293d79a..c43002f 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -69,8 +69,9 @@ class WOVariables(): except Exception as e: wo_user = input("Enter your name: ") wo_email = input("Enter your email: ") - os.system("git config --global user.name {0}".format(wo_user)) - os.system("git config --global user.email {0}".format(wo_email)) + os.system("/usr/bin/git config --global user.name {0}".format(wo_user)) + os.system( + "/usr/bin/git config --global user.email {0}".format(wo_email)) # Get System RAM and SWAP details wo_ram = psutil.virtual_memory().total / (1024 * 1024) From 702647587c05c96491fcff8361fd8066f3e268b4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 00:16:46 +0200 Subject: [PATCH 27/45] fix install on raspbian --- wo/cli/plugins/stack.py | 84 ++++++++++++++++++++++++++++------------- wo/core/variables.py | 46 +++++++++++++--------- 2 files changed, 85 insertions(+), 45 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index f0dbfc3..5cfffdd 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -111,34 +111,64 @@ class WOStackController(CementBaseController): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) # configure MySQL non-interactive install - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.error("Failed to initialize MySQL package") + if (not WOVariables.wo_platform_distro == 'raspbian'): + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.error("Failed to initialize MySQL package") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.error("Failed to initialize MySQL package") + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.error("Failed to initialize MySQL package") + else: + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.error("Failed to initialize MySQL package") # generate my.cnf root credentials mysql_config = """ [client] diff --git a/wo/core/variables.py b/wo/core/variables.py index c43002f..eb7ea80 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -94,24 +94,35 @@ class WOVariables(): # WordOps stack installation variables # Nginx repo and packages - if wo_platform_codename == 'trusty': + if wo_platform_distro == 'ubuntu': + if wo_platform_codename == 'trusty': + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/xUbuntu_14.04/ /") + elif wo_platform_codename == 'xenial': + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/xUbuntu_16.04/ /") + elif wo_platform_codename == 'bionic': + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/xUbuntu_18.04/ /") + else: + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/xUbuntu_19.04/ /") + elif wo_platform_distro == 'debian': + if wo_platform_codename == 'jessie': + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/Debian_8.0/ /") + elif wo_platform_codename == 'stretch': + wo_nginx_repo = ("deb http://download.opensuse.org" + "/repositories/home:" + "/virtubox:/WordOps/Debian_9.0/ /") + else: wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/xUbuntu_14.04/ /") - elif wo_platform_codename == 'xenial': - wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/xUbuntu_16.04/ /") - elif wo_platform_codename == 'bionic': - wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/xUbuntu_18.04/ /") - elif wo_platform_codename == 'disco': - wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/xUbuntu_19.04/ /") - elif wo_platform_codename == 'jessie': - wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/Debian_8.0/ /") - elif wo_platform_codename == 'stretch': - wo_nginx_repo = ("deb http://download.opensuse.org/repositories/home:" - "/virtubox:/WordOps/Debian_9.0/ /") + "/virtubox:/WordOps/Raspbian_9.0/ /") wo_nginx = ["nginx-custom", "nginx-wo"] wo_nginx_key = '188C9FB063F0247A' @@ -161,7 +172,6 @@ class WOVariables(): "http://sfo1.mirrors.digitalocean.com/mariadb/repo/" "10.3/debian {codename} main" .format(codename=wo_platform_codename)) - wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb"] wo_fail2ban = "fail2ban" From e85b4c4b23a302c19d714a10bb4597bee16b154f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 00:38:09 +0200 Subject: [PATCH 28/45] fix raspbian detection --- wo/core/variables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wo/core/variables.py b/wo/core/variables.py index eb7ea80..86057cd 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -32,7 +32,8 @@ class WOVariables(): wo_date = datetime.datetime.now().strftime('%d%b%Y%H%M%S') # WordOps core variables - wo_platform_distro = platform.linux_distribution()[0].lower() + wo_platform_distro = os.popen("lsb_release -si " + "| tr -d \'\\n\'").read().lower() wo_platform_version = platform.linux_distribution()[1] wo_platform_codename = os.popen("lsb_release -sc | tr -d \'\\n\'").read() From e19bb32d20665938c70228a21faa134d3f72db47 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 00:44:28 +0200 Subject: [PATCH 29/45] fix php packages on raspbian --- wo/core/variables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wo/core/variables.py b/wo/core/variables.py index 86057cd..86156a3 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -144,7 +144,7 @@ class WOVariables(): wo_php_extra = ["php-memcached", "php-imagick", "memcached", "graphviz", "php-xdebug", "php-msgpack", "php-redis"] wo_php_key = '' - elif wo_platform_distro == 'debian': + else: wo_php_repo = ( "deb https://packages.sury.org/php/ {codename} main" .format(codename=wo_platform_codename)) @@ -160,6 +160,7 @@ class WOVariables(): "php7.3-zip", "php7.3-xml", "php7.3-soap"] wo_php_extra = ["php-memcached", "php-imagick", "memcached", "graphviz", "php-xdebug", "php-msgpack", "php-redis"] + wo_php_key = 'AC0E47584A7A714D' # MySQL repo and packages From 157546a554520a2259b2e2bcc1de197a2a4ec963 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 01:35:51 +0200 Subject: [PATCH 30/45] Update variable and changelog --- CHANGELOG.md | 4 ++-- wo/core/variables.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9365a7..932df22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,8 +47,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - letsencrypt validation with webroot folder - hardened nginx ssl_ecdh_curve - Update phpredisadmin -- Increase MySQL root password size to 16 characters -- Increase MySQL users password size to 16 characters +- Increase MySQL root password size to 24 characters +- Increase MySQL users password size to 24 characters - Nginx locations template is the same for php7.2 & 7.3 - backend SSL configuration now stored in /var/www/22222/conf/nginx/ssl.conf - Install Netdata with static pre-built binaries instead of having to compile it from source diff --git a/wo/core/variables.py b/wo/core/variables.py index 86156a3..d3af7bd 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -169,7 +169,7 @@ class WOVariables(): "http://sfo1.mirrors.digitalocean.com/mariadb/repo/" "10.3/ubuntu {codename} main" .format(codename=wo_platform_codename)) - elif wo_platform_distro == 'debian': + else: wo_mysql_repo = ("deb [arch=amd64,ppc64el] " "http://sfo1.mirrors.digitalocean.com/mariadb/repo/" "10.3/debian {codename} main" From 06dcf5d156fb4a1ffc478c7010059e5324154a7a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 12:06:39 +0200 Subject: [PATCH 31/45] replace /tmp for security according to codacy --- README.md | 1 + install | 15 +++----- wo/cli/plugins/stack.py | 80 ++++++++++++++++++++--------------------- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 17df59f..5a17cb2 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ - Ubuntu 19.04 (Disco) - Debian 8 (Jessie) - Debian 9 (Stretch) +- Raspbian 9 (Stretch) ### Ports requirements diff --git a/install b/install index a534f34..0b552c3 100755 --- a/install +++ b/install @@ -113,6 +113,7 @@ if [ -z "$wo_branch" ]; then fi readonly wo_log_dir=/var/log/wo/ readonly wo_backup_dir=/var/lib/wo-backup/ +readonly wo_tmp_dir=/var/lib/wo/tmp readonly wo_install_log=/var/log/wo/install.log readonly wo_linux_distro=$(lsb_release -is) readonly wo_distro_version=$(lsb_release -sc) @@ -152,21 +153,13 @@ fi ### if [ ! -d "$wo_log_dir" ]; then - wo_lib_echo "Creating WordOps log directory, just a second..." - mkdir -p "$wo_log_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? + wo_lib_echo "Creating WordOps backup, tmp & log directory, just a second..." + mkdir -p "$wo_backup_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? # create wordops log files touch /var/log/wo/{wordops.log,install.log} - chmod -R 700 /var/log/wo || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? -fi - -if [ ! -d "$wo_backup_dir" ]; then - - wo_lib_echo "Creating WordOps backup directory, just a second..." - mkdir -p "$wo_backup_dir" || wo_lib_error "Whoops - seems we are unable to create the backup directory $wo_backup_dir, exit status " $? - chmod -R 600 "$wo_backup_dir" - + chmod -R 600 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? fi ### diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 5cfffdd..8d5bac5 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1079,11 +1079,11 @@ class WOStackController(CementBaseController): " to /usr/local/bin/wp file ") WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - if any('/tmp/pma.tar.gz' == x[1] + if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] for x in packages): - WOExtract.extract(self, '/tmp/pma.tar.gz', '/tmp/') - Log.debug(self, 'Extracting file /tmp/pma.tar.gz to ' - 'location /tmp/') + WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' + 'location /var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/db' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating new directory " @@ -1093,7 +1093,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot)) if not os.path.exists('{0}22222/htdocs/db/pma/' .format(WOVariables.wo_webroot)): - shutil.move('/tmp/phpmyadmin-STABLE/', + shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', '{0}22222/htdocs/db/pma/' .format(WOVariables.wo_webroot)) shutil.copyfile('{0}22222/htdocs/db/pma' @@ -1137,12 +1137,12 @@ class WOStackController(CementBaseController): recursive=True) # composer install and phpmyadmin update - if any('/tmp/composer-install' == x[1] + if any('/var/lib/wo/tmp/composer-install' == x[1] for x in packages): Log.info(self, "Installing composer, please wait...") - WOShellExec.cmd_exec(self, "php -q /tmp/composer-install " - "--install-dir=/tmp/") - shutil.copyfile('/tmp/composer.phar', + WOShellExec.cmd_exec(self, "php -q /var/lib/wo/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") + shutil.copyfile('/var/lib/wo/tmp/composer.phar', '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) Log.info(self, "Updating phpMyAdmin, please wait...") @@ -1150,12 +1150,12 @@ class WOStackController(CementBaseController): "update -n --no-dev -d " "/var/www/22222/htdocs/db/pma/") # netdata install - if any('/tmp/kickstart.sh' == x[1] + if any('/var/lib/wo/tmp/kickstart.sh' == x[1] for x in packages): if ((not os.path.exists('/opt/netdata')) and (not os.path.exists('/etc/netdata'))): Log.info(self, "Installing Netdata, please wait...") - WOShellExec.cmd_exec(self, "bash /tmp/kickstart.sh " + WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/kickstart.sh " "--dont-wait") # disable mail notifications WOFileUtils.searchreplace(self, "/opt/netdata/usr/" @@ -1183,14 +1183,14 @@ class WOStackController(CementBaseController): WOService.restart_service(self, 'netdata') # WordOps Dashboard - if any('/tmp/wo-dashboard.tar.gz' == x[1] + if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] for x in packages): if not os.path.isfile('{0}22222/htdocs/index.php' .format(WOVariables.wo_webroot)): Log.debug(self, "Extracting wo-dashboard.tar.gz " "to location {0}22222/htdocs/" .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/tmp/wo-dashboard.tar.gz', + WOExtract.extract(self, '/var/lib/wo/tmp/wo-dashboard.tar.gz', '{0}22222/htdocs' .format(WOVariables.wo_webroot)) if WOVariables.wo_wan != 'eth0': @@ -1209,16 +1209,16 @@ class WOStackController(CementBaseController): recursive=True) # Extplorer FileManager - if any('/tmp/extplorer.tar.gz' == x[1] + if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] for x in packages): if not os.path.exists('{0}22222/htdocs/files' .format(WOVariables.wo_webroot)): Log.debug(self, "Extracting explorer.tar.gz " "to location {0}22222/htdocs/files" .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/tmp/extplorer.tar.gz', - '/tmp/') - shutil.move('/tmp/extplorer-2.1.11/', + WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', + '/var/lib/wo/tmp/') + shutil.move('/var/lib/wo/tmp/extplorer-2.1.11/', '{0}22222/htdocs/files' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " @@ -1231,12 +1231,12 @@ class WOStackController(CementBaseController): recursive=True) # phpmemcachedadmin - if any('/tmp/memcached.tar.gz' == x[1] + if any('/var/lib/wo/tmp/memcached.tar.gz' == x[1] for x in packages): Log.debug(self, "Extracting memcached.tar.gz to location" " {0}22222/htdocs/cache/memcache " .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/tmp/memcached.tar.gz', + WOExtract.extract(self, '/var/lib/wo/tmp/memcached.tar.gz', '{0}22222/htdocs/cache/memcache/' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " @@ -1248,11 +1248,11 @@ class WOStackController(CementBaseController): WOVariables.wo_php_user, recursive=True) # webgrind - if any('/tmp/webgrind.tar.gz' == x[1] + if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] for x in packages): Log.debug(self, "Extracting file webgrind.tar.gz to " - "location /tmp/ ") - WOExtract.extract(self, '/tmp/webgrind.tar.gz', '/tmp/') + "location /var/lib/wo/tmp/ ") + WOExtract.extract(self, '/var/lib/wo/tmp/webgrind.tar.gz', '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/php' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directroy " @@ -1262,7 +1262,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot)) if not os.path.exists('{0}22222/htdocs/php/webgrind' .format(WOVariables.wo_webroot)): - shutil.move('/tmp/webgrind-master/', + shutil.move('/var/lib/wo/tmp/webgrind-master/', '{0}22222/htdocs/php/webgrind' .format(WOVariables.wo_webroot)) @@ -1290,11 +1290,11 @@ class WOStackController(CementBaseController): WOVariables.wo_php_user, recursive=True) # anemometer - if any('/tmp/anemometer.tar.gz' == x[1] + if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] for x in packages): Log.debug(self, "Extracting file anemometer.tar.gz to " - "location /tmp/ ") - WOExtract.extract(self, '/tmp/anemometer.tar.gz', '/tmp/') + "location /var/lib/wo/tmp/ ") + WOExtract.extract(self, '/var/lib/wo/tmp/anemometer.tar.gz', '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/db/' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directory") @@ -1302,7 +1302,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot)) if not os.path.exists('{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)): - shutil.move('/tmp/Anemometer-master', + shutil.move('/var/lib/wo/tmp/Anemometer-master', '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)) chars = ''.join(random.sample(string.ascii_letters, 8)) @@ -1347,7 +1347,7 @@ class WOStackController(CementBaseController): for x in packages): WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) # phpredisadmin - if any('/tmp/pra.tar.gz' == x[1] + if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] for x in packages): if not os.path.exists('{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)): @@ -1515,29 +1515,29 @@ class WOStackController(CementBaseController): if (not self.app.pargs.composer): packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", - "/tmp/pma.tar.gz", "phpMyAdmin"], + "/var/lib/wo/tmp/pma.tar.gz", "phpMyAdmin"], ["https://getcomposer.org/" "installer", - "/tmp/composer-install", + "/var/lib/wo/tmp/composer-install", "Composer"]] else: packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", - "/tmp/pma.tar.gz", "phpMyAdmin"]] + "/var/lib/wo/tmp/pma.tar.gz", "phpMyAdmin"]] # Composer if self.app.pargs.composer: Log.debug(self, "Setting packages variable for Composer ") packages = packages + [["https://getcomposer.org/installer", - "/tmp/composer-install", "Composer"]] + "/var/lib/wo/tmp/composer-install", "Composer"]] # PHPREDISADMIN if self.app.pargs.phpredisadmin: Log.debug(self, "Setting packages variable for phpRedisAdmin") packages = packages + [["https://github.com/ErikDubbelboer/" "phpRedisAdmin/archive/master.tar.gz", - "/tmp/pra.tar.gz", "phpRedisAdmin"], + "/var/lib/wo/tmp/pra.tar.gz", "phpRedisAdmin"], ["https://github.com/nrk/predis/" "archive/v1.1.1.tar.gz", - "/tmp/predis.tar.gz", "Predis"]] + "/var/lib/wo/tmp/predis.tar.gz", "Predis"]] # ADMINER if self.app.pargs.adminer: Log.debug(self, "Setting packages variable for Adminer ") @@ -1562,7 +1562,7 @@ class WOStackController(CementBaseController): if not os.path.exists('/opt/netdata'): packages = packages + [['https://my-netdata.io/' 'kickstart-static64.sh', - '/tmp/kickstart.sh', + '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] # WordOps Dashboard @@ -1572,11 +1572,11 @@ class WOStackController(CementBaseController): [["https://github.com/WordOps/" "wordops-dashboard/releases/" "download/v1.0/wo-dashboard.tar.gz", - "/tmp/wo-dashboard.tar.gz", + "/var/lib/wo/tmp/wo-dashboard.tar.gz", "WordOps Dashboard"], ["https://github.com/soerennb/" "extplorer/archive/v2.1.11.tar.gz", - "/tmp/extplorer.tar.gz", + "/var/lib/wo/tmp/extplorer.tar.gz", "eXtplorer"]] # UTILS @@ -1585,7 +1585,7 @@ class WOStackController(CementBaseController): packages = packages + [["https://github.com/elijaa/" "phpmemcachedadmin/archive/" "1.3.0.tar.gz", - '/tmp/memcached.tar.gz', + '/var/lib/wo/tmp/memcached.tar.gz', 'phpMemcachedAdmin'], ["https://raw.githubusercontent.com" "/rtCamp/eeadmin/master/cache/nginx/" @@ -1615,7 +1615,7 @@ class WOStackController(CementBaseController): "OCP.php"], ["https://github.com/jokkedk/webgrind/" "archive/master.tar.gz", - '/tmp/webgrind.tar.gz', 'Webgrind'], + '/var/lib/wo/tmp/webgrind.tar.gz', 'Webgrind'], ["http://bazaar.launchpad.net/~" "percona-toolkit-dev/percona-toolkit/" "2.1/download/head:/ptquerydigest-" @@ -1625,7 +1625,7 @@ class WOStackController(CementBaseController): "pt-query-advisor"], ["https://github.com/box/Anemometer/" "archive/master.tar.gz", - '/tmp/anemometer.tar.gz', 'Anemometer'] + '/var/lib/wo/tmp/anemometer.tar.gz', 'Anemometer'] ] except Exception as e: pass From 6765c16e953c6e3153f794539db286167c5ba170 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 13:21:25 +0200 Subject: [PATCH 32/45] fix wo log dir --- install | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install b/install index 0b552c3..839564b 100755 --- a/install +++ b/install @@ -151,15 +151,15 @@ fi ### # 1 - To prevent errors or unexpected behaviour, create the log and ACL it ### -if [ ! -d "$wo_log_dir" ]; then +if [ ! -d "$wo_log_dir" ] || [ ! -d "$wo_backup_dir" ] || [ ! -d "$wo_tmp_dir" ]; then wo_lib_echo "Creating WordOps backup, tmp & log directory, just a second..." - mkdir -p "$wo_backup_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? + mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? # create wordops log files touch /var/log/wo/{wordops.log,install.log} - chmod -R 600 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? + chmod -R 700 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? fi ### @@ -670,7 +670,7 @@ if [ -x /usr/local/bin/wo ]; then wo_upgrade_nginx | tee -ai $wo_install_log fi wo_update_latest | tee -ai $wo_install_log - wo_lib_echo "Installing acme.sh" | tee -ai $wo_install_log + wo_lib_echo "Updating acme.sh" | tee -ai $wo_install_log wo_install_acme_sh | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log wo_git_init | tee -ai $wo_install_log From 81e6eadb438374922ab0398110686049f83643ac Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 13:46:39 +0200 Subject: [PATCH 33/45] add vhostonly option --- wo/cli/plugins/site.py | 16 +++++++++++----- wo/cli/plugins/site_functions.py | 8 +++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index a21c198..e6e90b0 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -404,12 +404,14 @@ class WOSiteCreateController(CementBaseController): "{0} already exists".format(wo_domain)) if stype == 'proxy': - data['site_name'] = wo_domain - data['www_domain'] = wo_www_domain + data = dict(site_name=wo_domain, www_domain=wo_www_domain, + static=True, basic=False, php73=False, wp=False, + wpfc=False, wpsc=False, multisite=False, + wpsubdir=False, webroot=wo_site_webroot) data['proxy'] = True data['host'] = host data['port'] = port - wo_site_webroot = WOVariables.wo_webroot + wo_domain + data['basic'] = True if self.app.pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, @@ -418,6 +420,9 @@ class WOSiteCreateController(CementBaseController): wpsubdir=False, webroot=wo_site_webroot) data['basic'] = True + if self.app.pargs.vhostonly: + data['vhostonly'] = True + if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, @@ -583,11 +588,12 @@ class WOSiteCreateController(CementBaseController): "and please try again") # Setup WordPress if Wordpress site - if data['wp']: + if (data['wp'] and (not data['vhostonly'])): try: wo_wp_creds = setupwordpress(self, data) # Add database information for site into database - updateSiteInfo(self, wo_domain, db_name=data['wo_db_name'], + updateSiteInfo(self, wo_domain, + db_name=data['wo_db_name'], db_user=data['wo_db_user'], db_password=data['wo_db_pass'], db_host=data['wo_db_host']) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index e23ee87..716dd0e 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -49,8 +49,7 @@ def pre_run_checks(self): def check_domain_exists(self, domain): if getSiteInfo(self, domain): return True - else: - return False + return False def setupdomain(self, data): @@ -1202,9 +1201,8 @@ def deleteWebRoot(self, webroot): Log.debug(self, "Removing {0}".format(webroot)) WOFileUtils.rm(self, webroot) return True - else: - Log.debug(self, "{0} does not exist".format(webroot)) - return False + Log.debug(self, "{0} does not exist".format(webroot)) + return False def removeNginxConf(self, domain): From 2bffb50b8db51583306060f118c90ccd7955fbd6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 1 May 2019 16:26:27 +0200 Subject: [PATCH 34/45] add php-fpm emergency restart --- wo/cli/plugins/site_functions.py | 8 +++++--- wo/cli/templates/php-fpm.mustache | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 716dd0e..39cb072 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -59,7 +59,7 @@ def setupdomain(self, data): # print (key, value) wo_domain_name = data['site_name'] - wo_site_webroot = data['webroot'] if 'webroot' in data.keys() else '' + wo_site_webroot = data['webroot'] # Check if nginx configuration already exists # if os.path.isfile('/etc/nginx/sites-available/{0}' @@ -312,10 +312,12 @@ def setupwordpress(self, data): "--dbname=\'{0}\' --dbprefix=\'{1}\' " "--dbuser=\'{2}\' --dbhost=\'{3}\' " .format(data['wo_db_name'], wo_wp_prefix, - data['wo_db_user'], data['wo_db_host'] + data['wo_db_user'], + data['wo_db_host'] ) + "--dbpass=\'{0}\' " - "--extra-php< Date: Wed, 1 May 2019 16:39:22 +0200 Subject: [PATCH 35/45] use read -r instead of read -p --- install | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install b/install index 839564b..99e976e 100755 --- a/install +++ b/install @@ -651,8 +651,8 @@ if [ -x /usr/local/bin/wo ]; then if ! { wo -v 2>&1 | grep $wo_version_new }; then - read -p "Update WordOps to $wo_version_new (y/n): " wo_ans - if [ "$wo_ans" = "y" ] || [ "$wo_ans" = "Y" ]; then + echo -e "Update WordOps to $wo_version_new (y/n): " && read -r WO_ANSWER + if [ "$WO_ANSWER" = "y" ] || [ "$WO_ANSWER" = "Y" ]; then wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log wo_install_dep | tee -ai $wo_install_log wo_lib_echo "Backing-up WO install" | tee -ai $wo_install_log @@ -684,8 +684,8 @@ if [ -x /usr/local/bin/wo ]; then else # 2 - Migration from EEv3 if [ -x /usr/local/bin/ee ]; then - read -p "Migrate from EasyEngine to WordOps (y/n): " wo_ans - if [ "$wo_ans" = "y" ] || [ "$wo_ans" = "Y" ]; then + echo -e "Migrate from EasyEngine to WordOps (y/n): " && read -r WO_ANSWER + if [ "$WO_ANSWER" = "y" ] || [ "$WO_ANSWER" = "Y" ]; then wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log wo_install_dep | tee -ai $wo_install_log wo_lib_echo "Backing-up EE install" | tee -ai $wo_install_log From c2da42df3bcbfc885f257128e60769254f54900b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 10:07:37 +0200 Subject: [PATCH 36/45] testing open_basedir --- CHANGELOG.md | 6 ++++-- install | 10 +--------- wo/cli/plugins/stack.py | 19 ++++++++++++++++--- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 932df22..51c1dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,9 +37,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Extplorer filemanager - Enable OSCP Stapling with Let's Encrypt - Compress database backup with pigz before updating sites -- Add support for Ubuntu 19.04 disco -- Add support for Raspbian +- Support for Ubuntu 19.04 (disco) - few php extensions missing +- Support for Raspbian 9 (stretch) - tested on Raspberry Pi 3b+ - backup letsencrypt certificate before upgrade +- directives emergency_restart_threshold & emergency_restart_interval to restart php-fpm in case of failure #### Changed @@ -73,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - --hsts flag with basic html site - hsts flag on site not secure with letsencrypt - fix import of previous acme.sh certificate +- fix proxy webroot folder creation ### v3.9.4 - 2019-03-15 diff --git a/install b/install index 99e976e..7d875b8 100755 --- a/install +++ b/install @@ -357,15 +357,7 @@ wo_install_acme_sh() { fi cd /opt/acme.sh || exit 1 # create conf directories - [ ! -d /etc/letsencrypt/config ] && { - mkdir -p /etc/letsencrypt/config - } - [ ! -d /etc/letsencrypt/live ] && { - mkdir -p /etc/letsencrypt/live - } - [ ! -d /etc/letsencrypt/renewal ] && { - mkdir -p /etc/letsencrypt/renewal - } + mkdir -p /etc/letsencrypt/{config,live,renewal} # install acme.sh ./acme.sh --install \ --home /etc/letsencrypt \ diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 8d5bac5..7f7901b 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -787,6 +787,11 @@ class WOStackController(CementBaseController): "/etc/php/7.2/fpm/pool.d/www.conf") config.write(configfile) + with open("/etc/php/7.2/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www:/tmp/:/var/run/nginx-cache\"\n") + # Generate /etc/php/7.2/fpm/pool.d/www-two.conf WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", "/etc/php/7.2/fpm/pool.d/www-two.conf") @@ -942,6 +947,11 @@ class WOStackController(CementBaseController): "/etc/php/7.3/fpm/pool.d/www.conf") config.write(configfile) + with open("/etc/php/7.3/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www:/tmp/:/var/run/nginx-cache\"\n") + # Generate /etc/php/7.3/fpm/pool.d/www-two.conf WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", "/etc/php/7.3/fpm/pool.d/www-two.conf") @@ -1081,7 +1091,8 @@ class WOStackController(CementBaseController): if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] for x in packages): - WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + WOExtract.extract( + self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' 'location /var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/db' @@ -1252,7 +1263,8 @@ class WOStackController(CementBaseController): for x in packages): Log.debug(self, "Extracting file webgrind.tar.gz to " "location /var/lib/wo/tmp/ ") - WOExtract.extract(self, '/var/lib/wo/tmp/webgrind.tar.gz', '/var/lib/wo/tmp/') + WOExtract.extract( + self, '/var/lib/wo/tmp/webgrind.tar.gz', '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/php' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directroy " @@ -1294,7 +1306,8 @@ class WOStackController(CementBaseController): for x in packages): Log.debug(self, "Extracting file anemometer.tar.gz to " "location /var/lib/wo/tmp/ ") - WOExtract.extract(self, '/var/lib/wo/tmp/anemometer.tar.gz', '/var/lib/wo/tmp/') + WOExtract.extract( + self, '/var/lib/wo/tmp/anemometer.tar.gz', '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/db/' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directory") From 1edd099e8d836437f9dd4b41d7856f8ed854b789 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 11:18:05 +0200 Subject: [PATCH 37/45] fix vhostonly --- wo/cli/plugins/site.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index e6e90b0..a5c6db6 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -420,9 +420,6 @@ class WOSiteCreateController(CementBaseController): wpsubdir=False, webroot=wo_site_webroot) data['basic'] = True - if self.app.pargs.vhostonly: - data['vhostonly'] = True - if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, From 9bac60304a30e23d740b6de7e2266bec4798385d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 11:24:37 +0200 Subject: [PATCH 38/45] another fix for vhost only --- wo/cli/plugins/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index a5c6db6..3deb9fe 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -585,7 +585,7 @@ class WOSiteCreateController(CementBaseController): "and please try again") # Setup WordPress if Wordpress site - if (data['wp'] and (not data['vhostonly'])): + if (data['wp'] and (not self.app.pargs.vhostonly)): try: wo_wp_creds = setupwordpress(self, data) # Add database information for site into database From 21efc1262237780e40d5a1905c8da8ded538aa50 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 11:46:27 +0200 Subject: [PATCH 39/45] fix wo_wp_creds --- wo/cli/plugins/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 3deb9fe..c22e156 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -657,7 +657,7 @@ class WOSiteCreateController(CementBaseController): for msg in wo_auth: Log.info(self, Log.ENDC + msg, log=False) - if data['wp']: + if data['wp'] and (not self.app.pargs.vhostonly): Log.info(self, Log.ENDC + "WordPress admin user :" " {0}".format(wo_wp_creds['wp_user']), log=False) Log.info(self, Log.ENDC + "WordPress admin user password : {0}" From 05bd3c85319c66a75d8d9fdcb289ec6c0c437f40 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 12:28:44 +0200 Subject: [PATCH 40/45] add database creation with --vhostonly --- wo/cli/plugins/site.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index c22e156..6c27262 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -610,6 +610,60 @@ class WOSiteCreateController(CementBaseController): "`tail /var/log/wo/wordops.log` " "and please try again") + if (data['wp'] and (self.app.pargs.vhostonly)): + try: + data = setupdatabase(self, data) + # Add database information for site into database + updateSiteInfo(self, wo_domain, db_name=data['wo_db_name'], + db_user=data['wo_db_user'], + db_password=data['wo_db_pass'], + db_host=data['wo_db_host']) + except SiteError as e: + # call cleanup actions on failure + Log.debug(self, str(e)) + Log.info(self, Log.FAIL + + "There was a serious error encountered...") + Log.info(self, Log.FAIL + "Cleaning up afterwards...") + doCleanupAction(self, domain=wo_domain, + webroot=data['webroot'], + dbname=data['wo_db_name'], + dbuser=data['wo_db_user'], + dbhost=data['wo_db_host']) + deleteSiteInfo(self, wo_domain) + Log.error(self, "Check the log for details: " + "`tail /var/log/wo/wordops.log` " + "and please try again") + try: + wodbconfig = open("{0}/wo-config.php" + .format(wo_site_webroot), + encoding='utf-8', mode='w') + wodbconfig.write("" + .format(data['wo_db_name'], + data['wo_db_user'], + data['wo_db_pass'], + data['wo_db_host'])) + wodbconfig.close() + + except IOError as e: + Log.debug(self, str(e)) + Log.debug(self, "Error occured while generating " + "wo-config.php") + Log.info(self, Log.FAIL + + "There was a serious error encountered...") + Log.info(self, Log.FAIL + "Cleaning up afterwards...") + doCleanupAction(self, domain=wo_domain, + webroot=data['webroot'], + dbname=data['wo_db_name'], + dbuser=data['wo_db_user'], + dbhost=data['wo_db_host']) + deleteSiteInfo(self, wo_domain) + Log.error(self, "Check the log for details: " + "`tail /var/log/wo/wordops.log` " + "and please try again") + # Service Nginx Reload call cleanup if failed to reload nginx if not WOService.reload_service(self, 'nginx'): Log.info(self, Log.FAIL + From 92cf52c69cc3b6b5ef9c32228428dbf37cb511c7 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 13:45:45 +0200 Subject: [PATCH 41/45] fix phpmemcached admin & eXtplorer --- .travis.yml | 3 +-- wo/cli/plugins/stack.py | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index ecad218..7fd6ba8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,8 +51,7 @@ script: - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql || sudo tail -n50 /var/log/wo/wordops.log - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis || sudo tail -n50 /var/log/wo/wordops.log - - sudo ls /var/www/ - - sudo ls /var/www/22222/htdocs/ - sudo wp --allow-root --info - sudo wo info || sudo tail -n50 /var/log/wo/wordops.log - sudo tree -L 2 /etc/nginx + - sudo tree -L 2 /var/www/22222/htdocs/ diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 7f7901b..16f5656 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1229,7 +1229,7 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot)) WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', '/var/lib/wo/tmp/') - shutil.move('/var/lib/wo/tmp/extplorer-2.1.11/', + shutil.move('/var/lib/wo/tmp/extplorer-2.1.11', '{0}22222/htdocs/files' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " @@ -1248,7 +1248,7 @@ class WOStackController(CementBaseController): " {0}22222/htdocs/cache/memcache " .format(WOVariables.wo_webroot)) WOExtract.extract(self, '/var/lib/wo/tmp/memcached.tar.gz', - '{0}22222/htdocs/cache/memcache/' + '{0}22222/htdocs/cache/memcache' .format(WOVariables.wo_webroot)) Log.debug(self, "Setting Privileges to " "{0}22222/htdocs/cache/memcache file" @@ -1528,7 +1528,8 @@ class WOStackController(CementBaseController): if (not self.app.pargs.composer): packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", - "/var/lib/wo/tmp/pma.tar.gz", "phpMyAdmin"], + "/var/lib/wo/tmp/pma.tar.gz", + "phpMyAdmin"], ["https://getcomposer.org/" "installer", "/var/lib/wo/tmp/composer-install", @@ -1536,21 +1537,25 @@ class WOStackController(CementBaseController): else: packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", - "/var/lib/wo/tmp/pma.tar.gz", "phpMyAdmin"]] + "/var/lib/wo/tmp/pma.tar.gz", + "phpMyAdmin"]] # Composer if self.app.pargs.composer: Log.debug(self, "Setting packages variable for Composer ") packages = packages + [["https://getcomposer.org/installer", - "/var/lib/wo/tmp/composer-install", "Composer"]] + "/var/lib/wo/tmp/composer-install", + "Composer"]] # PHPREDISADMIN if self.app.pargs.phpredisadmin: Log.debug(self, "Setting packages variable for phpRedisAdmin") packages = packages + [["https://github.com/ErikDubbelboer/" "phpRedisAdmin/archive/master.tar.gz", - "/var/lib/wo/tmp/pra.tar.gz", "phpRedisAdmin"], + "/var/lib/wo/tmp/pra.tar.gz", + "phpRedisAdmin"], ["https://github.com/nrk/predis/" "archive/v1.1.1.tar.gz", - "/var/lib/wo/tmp/predis.tar.gz", "Predis"]] + "/var/lib/wo/tmp/predis.tar.gz", + "Predis"]] # ADMINER if self.app.pargs.adminer: Log.debug(self, "Setting packages variable for Adminer ") @@ -1628,7 +1633,8 @@ class WOStackController(CementBaseController): "OCP.php"], ["https://github.com/jokkedk/webgrind/" "archive/master.tar.gz", - '/var/lib/wo/tmp/webgrind.tar.gz', 'Webgrind'], + '/var/lib/wo/tmp/webgrind.tar.gz', + 'Webgrind'], ["http://bazaar.launchpad.net/~" "percona-toolkit-dev/percona-toolkit/" "2.1/download/head:/ptquerydigest-" @@ -1638,7 +1644,8 @@ class WOStackController(CementBaseController): "pt-query-advisor"], ["https://github.com/box/Anemometer/" "archive/master.tar.gz", - '/var/lib/wo/tmp/anemometer.tar.gz', 'Anemometer'] + '/var/lib/wo/tmp/anemometer.tar.gz', + 'Anemometer'] ] except Exception as e: pass From 080a874c3266af5783a7125d3eac07e3684763e3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 14:38:18 +0200 Subject: [PATCH 42/45] add kernel tweaks and ee cron removal --- CHANGELOG.md | 2 ++ install | 44 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51c1dde..d9c0bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support for Raspbian 9 (stretch) - tested on Raspberry Pi 3b+ - backup letsencrypt certificate before upgrade - directives emergency_restart_threshold & emergency_restart_interval to restart php-fpm in case of failure +- EasyEngine cronjob removal during install +- Kernel tweaks via systctl.conf #### Changed diff --git a/install b/install index 7d875b8..0d5e9b7 100755 --- a/install +++ b/install @@ -7,7 +7,7 @@ # Copyright (c) 2019 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- -# Version 3.9.5 - 2019-04-26 +# Version 3.9.5 - 2019-05-02 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" readonly wo_version_new="3.9.5" @@ -123,6 +123,7 @@ TIME=$(date +"$TIME_FORMAT") NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" +WO_ARCH="$(uname -m)" if [ -x /usr/local/bin/ee ]; then ee_migration=1 @@ -628,11 +629,34 @@ wo_clean_ee() { rm -f /usr/local/bin/ee /etc/bash_completion.d/ee_auto.rc /usr/lib/ee/templates /usr/local/lib/python3.6/dist-packages/ee-*.egg /etc/ee /var/lib/ee >> /var/log/wo/install.log 2>&1 } -# wo_tweak_kernel() { -# if [ ! -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf ]; then +wo_remove_ee_cron() { -# fi -# } + crontab -l | sed '/ee site update --le=renew --all 2> \/dev\/null/d' | crontab - + +} + +wo_tweak_kernel() { + + if [ ! -f /etc/sysctl.d/60-wo-tweaks.conf ]; then + if [ "$WO_ARCH" = "x86_64" ]; then + rm -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf + wget -qO /etc/sysctl.d/60-wo-tweaks.conf https://raw.githubusercontent.com/WordOps/WordOps/updating-configuration/wo/cli/templates/sysctl.mustache + if [ "$wo_distro_version" = "bionic" ] || [ "$wo_distro_version" = "disco" ]; then + modprobe tcp_bbr && echo 'tcp_bbr' >> /etc/modules-load.d/bbr.conf + echo -e '\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_notsent_lowat = 16384' >> /etc/sysctl.d/60-wo-tweaks.conf + else + modprobe tcp_htcp && echo 'tcp_htcp' >> /etc/modules-load.d/htcp.conf + echo 'net.ipv4.tcp_congestion_control = htcp' >> /etc/sysctl.d/60-wo-tweaks.conf + fi + + fi + LIMIT_CHECK=$(grep "500000" /etc/security/limits.conf) + if [ -z "$LIMIT_CHECK" ]; then + echo -e "* hard nofile 500000\n* soft nofile 500000\nroot hard nofile 500000\nroot soft nofile 500000\n" >> /etc/security/limits.conf + fi + fi + +} ### # 4 - WO MAIN SETUP @@ -643,7 +667,7 @@ if [ -x /usr/local/bin/wo ]; then if ! { wo -v 2>&1 | grep $wo_version_new }; then - echo -e "Update WordOps to $wo_version_new (y/n): " && read -r WO_ANSWER + echo -e "Update WordOps to $wo_version_new (y/n): " && read -r WO_ANSWER if [ "$WO_ANSWER" = "y" ] || [ "$WO_ANSWER" = "Y" ]; then wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log wo_install_dep | tee -ai $wo_install_log @@ -664,6 +688,8 @@ if [ -x /usr/local/bin/wo ]; then wo_update_latest | tee -ai $wo_install_log wo_lib_echo "Updating acme.sh" | tee -ai $wo_install_log wo_install_acme_sh | tee -ai $wo_install_log + wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log + wo_tweak_kernel | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log wo_git_init | tee -ai $wo_install_log wo_update_wp_cli | tee -ai $wo_install_log @@ -682,6 +708,8 @@ else wo_install_dep | tee -ai $wo_install_log wo_lib_echo "Backing-up EE install" | tee -ai $wo_install_log wo_backup_ee | tee -ai $wo_install_log + wo_lib_echo "Removing EasyEngine cronjob" | tee -ai $wo_install_log + wo_remove_ee_cron | tee -ai $wo_install_log wo_lib_echo "Syncing WO database" | tee -ai $wo_install_log wo_sync_db | tee -ai $wo_install_log secure_wo_db | tee -ai $wo_install_log @@ -698,6 +726,8 @@ else wo_update_latest | tee -ai $wo_install_log wo_lib_echo "Installing acme.sh" | tee -ai $wo_install_log wo_install_acme_sh | tee -ai $wo_install_log + wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log + wo_tweak_kernel | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log wo_git_init | tee -ai $wo_install_log wo_update_wp_cli | tee -ai $wo_install_log @@ -712,6 +742,8 @@ else wo_install_dep | tee -ai $wo_install_log wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log wo_install | tee -ai $wo_install_log + wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log + wo_tweak_kernel | tee -ai $wo_install_log wo_lib_echo "Installing acme.sh" | tee -ai $wo_install_log wo_install_acme_sh | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log From 462dc715656bea4c932aa07a7649c69655d3bb7b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 14:52:05 +0200 Subject: [PATCH 43/45] update travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7fd6ba8..2ee76d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,4 +54,4 @@ script: - sudo wp --allow-root --info - sudo wo info || sudo tail -n50 /var/log/wo/wordops.log - sudo tree -L 2 /etc/nginx - - sudo tree -L 2 /var/www/22222/htdocs/ + - sudo tree -L 3 /var/www/22222/htdocs/ From a1f5041968ea36e3f5d1efbd90f7455b2ea49bb8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 15:12:26 +0200 Subject: [PATCH 44/45] fix access to /usr/share/php --- wo/cli/plugins/stack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 16f5656..14cb867 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -790,7 +790,8 @@ class WOStackController(CementBaseController): with open("/etc/php/7.2/fpm/pool.d/www.conf", encoding='utf-8', mode='a') as myfile: myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www:/tmp/:/var/run/nginx-cache\"\n") + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/\"\n") # Generate /etc/php/7.2/fpm/pool.d/www-two.conf WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", @@ -950,7 +951,8 @@ class WOStackController(CementBaseController): with open("/etc/php/7.3/fpm/pool.d/www.conf", encoding='utf-8', mode='a') as myfile: myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www:/tmp/:/var/run/nginx-cache\"\n") + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/\"\n") # Generate /etc/php/7.3/fpm/pool.d/www-two.conf WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", From 147369bc7533da4e597c131a865358b208a3e0b0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 2 May 2019 18:41:07 +0200 Subject: [PATCH 45/45] prepare for release --- CHANGELOG.md | 16 ++++++++++------ README.md | 3 +-- wo/cli/templates/fastcgi.mustache | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c0bc5..ab727f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), --- -### v3.9.5 - [Unreleased] +### v3.9.6 - [Unreleased] + +### v3.9.5 - 2019-05-02 #### Added @@ -28,21 +30,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support for Netdata on backend : https://server.hostname:22222/netdata/ - New Stacks : composer and netdata - additional argument for letsencrypt : --hsts -- Theme for adminer +- Clean Theme for adminer - Credits for tools shipped with WordOps - Cache exception for Easy Digital Download -- Additional cache exception for Woocommerce +- Additional cache exceptions for Woocommerce - MySQL monitoring with Netdata -- WordOps-dashboard on 22222 -- Extplorer filemanager +- WordOps-dashboard on 22222, can be installed with `wo stack install` +- Extplorer filemanager in WordOps backend - Enable OSCP Stapling with Let's Encrypt -- Compress database backup with pigz before updating sites +- Compress database backup with pigz (faster than gzip) before updating sites - Support for Ubuntu 19.04 (disco) - few php extensions missing - Support for Raspbian 9 (stretch) - tested on Raspberry Pi 3b+ - backup letsencrypt certificate before upgrade - directives emergency_restart_threshold & emergency_restart_interval to restart php-fpm in case of failure - EasyEngine cronjob removal during install - Kernel tweaks via systctl.conf +- open_basedir on php-fpm process to forbid access with php outside of /var/www & /run/nginx-cache #### Changed @@ -56,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - backend SSL configuration now stored in /var/www/22222/conf/nginx/ssl.conf - Install Netdata with static pre-built binaries instead of having to compile it from source - Nginx updated to new stable release (1.16.0) +- New packages (phpmyadmin, adminer, composer) are not download in /tmp anymore #### Fixed diff --git a/README.md b/README.md index 5a17cb2..12dd5c6 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ build MIT Commits -GitHub release +GitHub release
Badge Twitter Badge Slack @@ -33,7 +33,6 @@ WordOps.net • Documentation • Community forum -

--- diff --git a/wo/cli/templates/fastcgi.mustache b/wo/cli/templates/fastcgi.mustache index 72cd66b..271e751 100644 --- a/wo/cli/templates/fastcgi.mustache +++ b/wo/cli/templates/fastcgi.mustache @@ -1,4 +1,4 @@ -# FastCGI cache settings - WO v3.9.5 +# FastCGI cache settings fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=60m max_size=256M; fastcgi_cache_key "$scheme$request_method$host$request_uri"; fastcgi_cache_use_stale error timeout invalid_header updating http_500 http_503;