From 44d0514da68f0f6be01fe925eddacba72e800b1e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 11:47:04 +0200 Subject: [PATCH 01/44] Improve logging --- wo/cli/main.py | 10 ++++++---- wo/cli/plugins/secure.py | 4 ++-- wo/cli/plugins/site.py | 1 + wo/cli/plugins/stack.py | 4 ++-- wo/cli/plugins/stack_migrate.py | 14 +++++++++++++- wo/core/logging.py | 6 +++--- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/wo/cli/main.py b/wo/cli/main.py index 4579177..d0c009e 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -1,12 +1,14 @@ """WordOps main application entry point.""" -import sys import os +import sys + from cement.core import foundation -from cement.utils.misc import init_defaults -from cement.core.exc import FrameworkError, CaughtSignal +from cement.core.exc import CaughtSignal, FrameworkError from cement.ext.ext_argparse import ArgParseArgumentHandler -from wo.core import exc +from cement.utils.misc import init_defaults + from wo.cli.ext.wo_outputhandler import WOOutputHandler +from wo.core import exc # this has to happen after you import sys, but before you import anything # from Cement "source: https://github.com/datafolklabs/cement/issues/290" diff --git a/wo/cli/plugins/secure.py b/wo/cli/plugins/secure.py index e8ca67e..38f8190 100644 --- a/wo/cli/plugins/secure.py +++ b/wo/cli/plugins/secure.py @@ -86,12 +86,12 @@ class WOSecureController(CementBaseController): while not pargs.user_input.isdigit(): Log.info(self, "Please enter a valid port number ") pargs.user_input = input("WordOps " - "admin port [22222]:") + "admin port [22222]:") if not pargs.user_input: port = input("WordOps admin port [22222]:") if port == "": pargs.user_input = 22222 - while not port.isdigit() and port != "": + while (not port.isdigit()) and (port != "") and (not port < 65556): Log.info(self, "Please Enter valid port number :") port = input("WordOps admin port [22222]:") pargs.user_input = port diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 2050fad..4e19cfc 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -7,6 +7,7 @@ from subprocess import Popen from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import (addNewSite, deleteSiteInfo, getAllsites, getSiteInfo, updateSiteInfo) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 72f6c79..ec55bf9 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -9,11 +9,11 @@ import re import shutil import string -import requests - import psutil +import requests from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_migrate import WOStackMigrateController diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 5c47906..7b64b37 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -3,6 +3,7 @@ import os from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.logging import Log @@ -28,6 +29,17 @@ class WOStackMigrateController(CementBaseController): # Backup all database WOMysql.backupAll(self) + if not WOVariables.wo_distro == 'raspbian': + if (not WOVariables.wo_platform_codename == 'jessie'): + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb", "mariadb-backup"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysql.connector"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb"] + # Add MariaDB repo Log.info(self, "Adding repository for MariaDB, please wait...") @@ -78,7 +90,7 @@ class WOStackMigrateController(CementBaseController): log=False) # Install MariaDB - apt_packages = ["mariadb-server"] + apt_packages = wo_mysql Log.info(self, "Updating apt-cache, hang on...") WOAptGet.update(self) diff --git a/wo/core/logging.py b/wo/core/logging.py index 3078f59..3bdb9b5 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -51,7 +51,7 @@ class Log: Logs info messages with validation step """ print( - Log.OKBLUE + msg + + Log.OKBLUE + "{0:22}".format(msg) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -61,7 +61,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + msg + + Log.OKBLUE + "{0:22}".format(msg) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -72,7 +72,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + msg + + Log.OKBLUE + "{0:22}".format(msg) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: From 63188b1905b64464547db4e08a531831d6dc463b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 11:59:57 +0200 Subject: [PATCH 02/44] Changes in logging --- wo/core/logging.py | 6 +++--- wo/core/services.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 3bdb9b5..54a3e8c 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -52,7 +52,7 @@ class Log: """ print( Log.OKBLUE + "{0:22}".format(msg) + - " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -62,7 +62,7 @@ class Log: """ print( Log.OKBLUE + "{0:22}".format(msg) + - " [" + Log.ENDC + Log.OKGREEN + "OK" + + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKGREEN + msg + Log.ENDC) @@ -73,7 +73,7 @@ class Log: """ print( Log.OKBLUE + "{0:22}".format(msg) + - " [" + Log.ENDC + Log.FAIL + "OK" + + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.FAIL + msg + Log.ENDC) diff --git a/wo/core/services.py b/wo/core/services.py index 5d4a1bd..39e8225 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -59,7 +59,8 @@ class WOService(): retcode = subprocess.getstatusoutput('service {0} stop' .format(service_name)) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.info(self, "[" + Log.ENDC + Log.GREEN "OK" + + Log.ENDC + Log.OKBLUE + "]") return True else: Log.debug(self, "{0}".format(retcode[1])) From 2e151cc74b6750c01c5b92703e02a554bc6ca61a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:00:46 +0200 Subject: [PATCH 03/44] Fix services --- wo/core/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/services.py b/wo/core/services.py index 39e8225..585b19c 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -59,7 +59,7 @@ class WOService(): retcode = subprocess.getstatusoutput('service {0} stop' .format(service_name)) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + Log.GREEN "OK" + + Log.info(self, "[" + Log.ENDC + Log.GREEN + "OK" + Log.ENDC + Log.OKBLUE + "]") return True else: From eb8aa1bd4eda644bc2b71041ba4ca9f1d148484d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:01:33 +0200 Subject: [PATCH 04/44] Fix log color --- wo/core/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/services.py b/wo/core/services.py index 585b19c..70ac857 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -59,7 +59,7 @@ class WOService(): retcode = subprocess.getstatusoutput('service {0} stop' .format(service_name)) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + Log.GREEN + "OK" + + Log.info(self, "[" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]") return True else: From 391f9028c619fd0ddceaf111142b54834b2dac4f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:06:28 +0200 Subject: [PATCH 05/44] another change in logging --- wo/core/logging.py | 6 +++--- wo/core/services.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 54a3e8c..763ede4 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -51,7 +51,7 @@ class Log: Logs info messages with validation step """ print( - Log.OKBLUE + "{0:22}".format(msg) + + Log.OKBLUE + "{0:20}".format(msg) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -61,7 +61,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + "{0:22}".format(msg) + + Log.OKBLUE + ("{0:20}".format(msg)) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -72,7 +72,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + "{0:22}".format(msg) + + Log.OKBLUE + "{0:20}".format(msg) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: diff --git a/wo/core/services.py b/wo/core/services.py index 70ac857..dc874b2 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -38,7 +38,8 @@ class WOService(): Log.info(self, "Start : {0:10}" .format(service_name), end='') retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.info(self, "[" + Log.ENDC + Log.OKGREEN + + "OK" + Log.ENDC + Log.OKBLUE + "]") return True else: Log.debug(self, "{0}".format(retcode[1])) @@ -55,7 +56,7 @@ class WOService(): Similar to `service xyz stop` """ try: - Log.info(self, "Stop : {0:10}" .format(service_name), end='') + Log.info(self, "Stop : {0:10}" .format(service_name), end='') retcode = subprocess.getstatusoutput('service {0} stop' .format(service_name)) if retcode[0] == 0: From 6cdfe8318c3d8aa1f6a6fe575bd0067786a59e49 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:09:53 +0200 Subject: [PATCH 06/44] Antoehr change in logging --- wo/core/logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 763ede4..60ae26d 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -51,7 +51,7 @@ class Log: Logs info messages with validation step """ print( - Log.OKBLUE + "{0:20}".format(msg) + + Log.OKBLUE + "{0}".format(msg[0:22]) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -61,7 +61,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + ("{0:20}".format(msg)) + + Log.OKBLUE + ("{0}".format(msg[0:22])) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -72,7 +72,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + "{0:20}".format(msg) + + Log.OKBLUE + "{0}".format(msg[0:22]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: From d019fc66f047b6ba180509d4451aef96d7aa206d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:10:36 +0200 Subject: [PATCH 07/44] Increase log size --- wo/core/logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 60ae26d..cb2c77a 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -51,7 +51,7 @@ class Log: Logs info messages with validation step """ print( - Log.OKBLUE + "{0}".format(msg[0:22]) + + Log.OKBLUE + "{0}".format(msg[0:30]) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -61,7 +61,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + ("{0}".format(msg[0:22])) + + Log.OKBLUE + ("{0}".format(msg[0:30])) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -72,7 +72,7 @@ class Log: Logs info messages after validation step """ print( - Log.OKBLUE + "{0}".format(msg[0:22]) + + Log.OKBLUE + "{0}".format(msg[0:30]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: From c4fe974dc81e0f1af9be9ce62ccaa72a6b4333f9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:18:29 +0200 Subject: [PATCH 08/44] Change logging --- wo/core/logging.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index cb2c77a..3d1b6fe 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -50,8 +50,11 @@ class Log: """ Logs info messages with validation step """ + space_to_add = int(30 - len(msg)) + space = " print( Log.OKBLUE + "{0}".format(msg[0:30]) + + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -60,8 +63,11 @@ class Log: """ Logs info messages after validation step """ + space_to_add = int(30 - len(msg)) + space = " " print( - Log.OKBLUE + ("{0}".format(msg[0:30])) + + Log.OKBLUE + "{0}".format(msg[0:30]) + + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -71,8 +77,11 @@ class Log: """ Logs info messages after validation step """ + space_to_add = int(30 - len(msg)) + space = " " print( Log.OKBLUE + "{0}".format(msg[0:30]) + + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: From 897df3bda7519b4d7c8ffa541445e310c79335c9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:19:23 +0200 Subject: [PATCH 09/44] Fix logging --- wo/core/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 3d1b6fe..6417cf6 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -51,7 +51,7 @@ class Log: Logs info messages with validation step """ space_to_add = int(30 - len(msg)) - space = " + space = " " print( Log.OKBLUE + "{0}".format(msg[0:30]) + "{0}".format(space[0:space_to_add]) + From 93ef756cb14c57c3adda2d818005f0a85db98ac3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:22:31 +0200 Subject: [PATCH 10/44] Adjust length --- wo/core/logging.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 6417cf6..268ca9d 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -50,8 +50,8 @@ class Log: """ Logs info messages with validation step """ - space_to_add = int(30 - len(msg)) - space = " " + space_to_add = int(40 - len(msg[0:40])) + space = " " print( Log.OKBLUE + "{0}".format(msg[0:30]) + "{0}".format(space[0:space_to_add]) + @@ -63,10 +63,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(30 - len(msg)) - space = " " + space_to_add = int(40 - len(msg[0:40])) + space = " " print( - Log.OKBLUE + "{0}".format(msg[0:30]) + + Log.OKBLUE + "{0}".format(msg[0:40]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) @@ -77,10 +77,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(30 - len(msg)) - space = " " + space_to_add = int(40 - len(msg[0:40])) + space = " " print( - Log.OKBLUE + "{0}".format(msg[0:30]) + + Log.OKBLUE + "{0}".format(msg[0:40]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) From 4dfdf92d2d0678a5fc994edadaee301c24ade95d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:31:34 +0200 Subject: [PATCH 11/44] Updating logging --- wo/cli/plugins/site_functions.py | 12 +++++------ wo/core/logging.py | 10 ++++----- wo/core/sslutils.py | 36 ++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 80bbb1a..fdafca8 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1354,7 +1354,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, if subdomain: Log.info(self, "Certificate type: Subdomain") Log.info(self, "Validation mode : {0}".format(validation_mode)) - Log.wait(self, "Issuing SSL certificate with acme.sh") + Log.wait(self, "Issuing SSL cert with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} {1} " @@ -1365,7 +1365,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, elif wildcard: Log.info(self, "Certificate type: Wildcard") Log.info(self, "Validation mode : {0}".format(validation_mode)) - Log.wait(self, "Issuing SSL certificate with acme.sh") + Log.wait(self, "Issuing SSL cert with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d '*.{0}' --dns {1} " @@ -1376,7 +1376,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, else: Log.info(self, "Certificate type: Domain + www") Log.info(self, "Validation mode : {0}".format(validation_mode)) - Log.wait(self, "Issuing SSL certificate with acme.sh") + Log.wait(self, "Issuing SSL cert with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d www.{0} {1} " @@ -1384,7 +1384,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, .format(wo_domain_name, acme_mode, keylenght)) if ssl: - Log.valide(self, "Issuing SSL certificate with acme.sh") + Log.valide(self, "Issuing SSL cert with acme.sh") Log.wait(self, "Deploying SSL cert with acme.sh") Log.debug(self, "Cert deployment for domain: {0}" .format(wo_domain_name)) @@ -1582,8 +1582,8 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): "\tlisten [::]:80;\n" + "\tserver_name www.{0} {0};\n" .format(wo_domain_name) + - "\treturn 301 https://{0}" - .format(wo_domain_name)+"$request_uri;\n}") + "\treturn 301 https://$host" + "$request_uri;\n}") sslconf.close() except IOError as e: diff --git a/wo/core/logging.py b/wo/core/logging.py index 268ca9d..82c4794 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -50,7 +50,7 @@ class Log: """ Logs info messages with validation step """ - space_to_add = int(40 - len(msg[0:40])) + space_to_add = int(30 - len(msg[0:30])) space = " " print( Log.OKBLUE + "{0}".format(msg[0:30]) + @@ -63,10 +63,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(40 - len(msg[0:40])) + space_to_add = int(30 - len(msg[0:30])) space = " " print( - Log.OKBLUE + "{0}".format(msg[0:40]) + + Log.OKBLUE + "{0}".format(msg[0:30]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) @@ -77,10 +77,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(40 - len(msg[0:40])) + space_to_add = int(30 - len(msg[0:30])) space = " " print( - Log.OKBLUE + "{0}".format(msg[0:40]) + + Log.OKBLUE + "{0}".format(msg[0:30]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index d380557..a54dfe2 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -68,22 +68,26 @@ class SSL: "--allow-root --quiet")) test_url = re.split(":", wo_siteurl) if not (test_url[0] == 'https'): - WOShellExec.cmd_exec( - self, "{0} option update siteurl " - "\'https://{1}\' --allow-root".format( - WOVariables.wo_wpcli_path, domain)) - WOShellExec.cmd_exec( - self, "{0} option update home " - "\'https://{1}\' --allow-root".format( - WOVariables.wo_wpcli_path, domain)) - WOShellExec.cmd_exec( - self, "{0} search-replace \'http://{0}\'" - "\'https://{0}\' --skip-columns=guid " - "--skip-tables=wp_users" - .format(domain)) - Log.info( - self, "Site address updated " - "successfully to https://{0}".format(domain)) + Log.wait(self, "Updating site url with https") + try: + WOShellExec.cmd_exec( + self, "{0} option update siteurl " + "\'https://{1}\' --allow-root".format( + WOVariables.wo_wpcli_path, domain)) + WOShellExec.cmd_exec( + self, "{0} option update home " + "\'https://{1}\' --allow-root".format( + WOVariables.wo_wpcli_path, domain)) + WOShellExec.cmd_exec( + self, "{0} search-replace \'http://{0}\'" + "\'https://{0}\' --skip-columns=guid " + "--skip-tables=wp_users" + .format(domain)) + except Exception as e: + Log.debug(self, str(e)) + Log.failed(self, "Updating site url with https") + else: + Log.valide(self, "Updating site url with https") # check if a wildcard exist to secure a new subdomain From 5565938819dc9182a0c7a4b68fe5d6e038c9f168 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:38:02 +0200 Subject: [PATCH 12/44] Updating logging for sites --- wo/cli/plugins/site_functions.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index fdafca8..3d58a94 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -643,7 +643,7 @@ def setupwordpressnetwork(self, data): def installwp_plugin(self, plugin_name, data): wo_site_webroot = data['webroot'] - Log.info(self, "Installing plugin {0}, please wait..." + Log.wait(self, "Installing plugin {0}" .format(plugin_name)) WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) try: @@ -667,7 +667,11 @@ def installwp_plugin(self, plugin_name, data): except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) raise SiteError("plugin activation failed") - + Log.failed(self, "Installing plugin {0}" + .format(plugin_name)) + else: + Log.valide(self, "Installing plugin {0}" + .format(plugin_name)) return 1 @@ -676,7 +680,7 @@ def uninstallwp_plugin(self, plugin_name, data): Log.debug(self, "Uninstalling plugin {0}, please wait..." .format(plugin_name)) WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) - Log.info(self, "Uninstalling plugin {0}, please wait..." + Log.wait(self, "Uninstalling plugin {0}" .format(plugin_name)) try: WOShellExec.cmd_exec(self, "{0} plugin " @@ -691,6 +695,11 @@ def uninstallwp_plugin(self, plugin_name, data): except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) raise SiteError("plugin uninstall failed") + Log.failed(self, "Uninstalling plugin {0}" + .format(plugin_name)) + else: + Log.valide(self, "Uninstalling plugin {0}" + .format(plugin_name)) def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): @@ -1553,6 +1562,8 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): "/etc/nginx/conf.d/force-ssl-{0}.conf" .format(wo_domain_name)) else: + Log.wait(self, "Adding HTTPS redirection" + " http://{0}".format(wo_domain_name)) if wildcard: try: sslconf = open("/etc/nginx/conf.d/force-ssl-{0}.conf" @@ -1591,9 +1602,11 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): Log.debug(self, "Error occured while generating " "/etc/nginx/conf.d/force-ssl-{0}.conf" .format(wo_domain_name)) - - Log.info(self, "Added HTTPS Force Redirection for Site " - " http://{0}".format(wo_domain_name)) + Log.failed(self, "Adding HTTPS redirection" + " http://{0}".format(wo_domain_name)) + else: + Log.valide(self, "Adding HTTPS redirection" + " http://{0}".format(wo_domain_name)) # Nginx Configation into GIT WOGit.add(self, ["/etc/nginx"], msg="Adding /etc/nginx/conf.d/" From 335ceec966b8acd0ae1221374bf6577784a95537 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:41:02 +0200 Subject: [PATCH 13/44] Few fixes in logging --- wo/cli/plugins/site_functions.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 3d58a94..1b1b755 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1394,7 +1394,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, acme_mode, keylenght)) if ssl: Log.valide(self, "Issuing SSL cert with acme.sh") - Log.wait(self, "Deploying SSL cert with acme.sh") + Log.wait(self, "Deploying SSL cert") Log.debug(self, "Cert deployment for domain: {0}" .format(wo_domain_name)) try: @@ -1414,7 +1414,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, "service nginx restart\" " .format(WOVariables.wo_ssl_live, wo_domain_name)) - Log.valide(self, "Deploying SSL cert with acme.sh") + Log.valide(self, "Deploying SSL cert") if os.path.isdir('/var/www/{0}/conf/nginx' .format(wo_domain_name)): @@ -1562,8 +1562,7 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): "/etc/nginx/conf.d/force-ssl-{0}.conf" .format(wo_domain_name)) else: - Log.wait(self, "Adding HTTPS redirection" - " http://{0}".format(wo_domain_name)) + Log.wait(self, "Adding HTTPS redirection") if wildcard: try: sslconf = open("/etc/nginx/conf.d/force-ssl-{0}.conf" @@ -1602,11 +1601,9 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): Log.debug(self, "Error occured while generating " "/etc/nginx/conf.d/force-ssl-{0}.conf" .format(wo_domain_name)) - Log.failed(self, "Adding HTTPS redirection" - " http://{0}".format(wo_domain_name)) + Log.failed(self, "Adding HTTPS redirection") else: - Log.valide(self, "Adding HTTPS redirection" - " http://{0}".format(wo_domain_name)) + Log.valide(self, "Adding HTTPS redirection") # Nginx Configation into GIT WOGit.add(self, ["/etc/nginx"], msg="Adding /etc/nginx/conf.d/" From 9b9386ceec010d81b55d3f699ce7e02b03f81241 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:41:59 +0200 Subject: [PATCH 14/44] Adjusting log length --- wo/core/logging.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 82c4794..a0ee0e1 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -50,10 +50,10 @@ class Log: """ Logs info messages with validation step """ - space_to_add = int(30 - len(msg[0:30])) + space_to_add = int(31 - len(msg[0:31])) space = " " print( - Log.OKBLUE + "{0}".format(msg[0:30]) + + Log.OKBLUE + "{0}".format(msg[0:31]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: @@ -63,10 +63,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(30 - len(msg[0:30])) + space_to_add = int(31 - len(msg[0:31])) space = " " print( - Log.OKBLUE + "{0}".format(msg[0:30]) + + Log.OKBLUE + "{0}".format(msg[0:31]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.OKGREEN + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) @@ -77,10 +77,10 @@ class Log: """ Logs info messages after validation step """ - space_to_add = int(30 - len(msg[0:30])) + space_to_add = int(31 - len(msg[0:31])) space = " " print( - Log.OKBLUE + "{0}".format(msg[0:30]) + + Log.OKBLUE + "{0}".format(msg[0:31]) + "{0}".format(space[0:space_to_add]) + " [" + Log.ENDC + Log.FAIL + "OK" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) From 877e09e08643095b5c6ff0418834f482cff12d6a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:44:33 +0200 Subject: [PATCH 15/44] Updating logging --- wo/cli/plugins/site_functions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 1b1b755..d5b318a 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -696,15 +696,15 @@ def uninstallwp_plugin(self, plugin_name, data): Log.debug(self, "{0}".format(e)) raise SiteError("plugin uninstall failed") Log.failed(self, "Uninstalling plugin {0}" - .format(plugin_name)) + .format(plugin_name)) else: Log.valide(self, "Uninstalling plugin {0}" - .format(plugin_name)) + .format(plugin_name)) def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): wo_site_webroot = data['webroot'] - Log.info(self, "Setting plugin {0}, please wait..." + Log.wait(self, "Setting plugin {0}" .format(plugin_name)) WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) @@ -729,6 +729,11 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) raise SiteError("plugin setup failed") + Log.failed(self, "Setting plugin {0}" + .format(plugin_name)) + else: + Log.valide(self, "Setting plugin {0}" + .format(plugin_name)) def setwebrootpermissions(self, webroot): From 77c0bdfeb8b3aa86ff58886609d8fbae68acefb5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:45:27 +0200 Subject: [PATCH 16/44] Fix logging --- wo/cli/plugins/site_functions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d5b318a..8d679be 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -718,6 +718,11 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) raise SiteError("plugin setup failed") + Log.failed(self, "Setting plugin {0}" + .format(plugin_name)) + else: + Log.valide(self, "Setting plugin {0}" + .format(plugin_name)) else: try: WOShellExec.cmd_exec(self, "{0} " From 0453c0701d3598a9791019a39aca42a96a328a7a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:47:12 +0200 Subject: [PATCH 17/44] Improve logging --- wo/cli/plugins/site_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 8d679be..3a9be5d 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -35,7 +35,7 @@ class SiteError(Exception): def pre_run_checks(self): # Check nginx configuration - Log.info(self, "Running pre-update checks, please wait...") + Log.wait(self, "Running pre-update checks") try: Log.debug(self, "checking NGINX configuration ...") FNULL = open('/dev/null', 'w') @@ -44,6 +44,9 @@ def pre_run_checks(self): except CalledProcessError as e: Log.debug(self, "{0}".format(str(e))) raise SiteError("nginx configuration check failed.") + Log.failed(self, "Running pre-update checks") + else: + Log.valide(self, "Running pre-update checks") def check_domain_exists(self, domain): From edc1091ae1a4b3a3230baa30f4ab4f30fe8fd9a3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:49:04 +0200 Subject: [PATCH 18/44] Fox log failed --- wo/cli/plugins/site_functions.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 3a9be5d..80ff436 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -42,9 +42,9 @@ def pre_run_checks(self): subprocess.check_call(["/usr/sbin/nginx", "-t"], stdout=FNULL, stderr=subprocess.STDOUT) except CalledProcessError as e: + Log.failed(self, "Running pre-update checks") Log.debug(self, "{0}".format(str(e))) raise SiteError("nginx configuration check failed.") - Log.failed(self, "Running pre-update checks") else: Log.valide(self, "Running pre-update checks") @@ -668,10 +668,10 @@ def installwp_plugin(self, plugin_name, data): else '' )) except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("plugin activation failed") Log.failed(self, "Installing plugin {0}" .format(plugin_name)) + Log.debug(self, "{0}".format(e)) + raise SiteError("plugin activation failed") else: Log.valide(self, "Installing plugin {0}" .format(plugin_name)) @@ -696,10 +696,10 @@ def uninstallwp_plugin(self, plugin_name, data): "--allow-root uninstall " "{0}".format(plugin_name)) except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("plugin uninstall failed") Log.failed(self, "Uninstalling plugin {0}" .format(plugin_name)) + Log.debug(self, "{0}".format(e)) + raise SiteError("plugin uninstall failed") else: Log.valide(self, "Uninstalling plugin {0}" .format(plugin_name)) @@ -719,10 +719,10 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): "{0} \'{1}\' --format=json" .format(plugin_option, plugin_data)) except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("plugin setup failed") Log.failed(self, "Setting plugin {0}" .format(plugin_name)) + Log.debug(self, "{0}".format(e)) + raise SiteError("plugin setup failed") else: Log.valide(self, "Setting plugin {0}" .format(plugin_name)) @@ -735,10 +735,10 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): .format(plugin_option, plugin_data )) except CommandExecutionError as e: + Log.failed(self, "Setting plugin {0}" + .format(plugin_name)) Log.debug(self, "{0}".format(e)) raise SiteError("plugin setup failed") - Log.failed(self, "Setting plugin {0}" - .format(plugin_name)) else: Log.valide(self, "Setting plugin {0}" .format(plugin_name)) @@ -1610,11 +1610,11 @@ def httpsRedirect(self, wo_domain_name, redirect=True, wildcard=False): sslconf.close() except IOError as e: + Log.failed(self, "Adding HTTPS redirection") Log.debug(self, str(e)) Log.debug(self, "Error occured while generating " "/etc/nginx/conf.d/force-ssl-{0}.conf" .format(wo_domain_name)) - Log.failed(self, "Adding HTTPS redirection") else: Log.valide(self, "Adding HTTPS redirection") # Nginx Configation into GIT From 391e13c7096f68d8649196c0b25e9e0b5ad7a9ee Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 5 Sep 2019 12:49:42 +0200 Subject: [PATCH 19/44] Add KO instead of OK --- wo/core/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index a0ee0e1..bb6cbda 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -82,7 +82,7 @@ class Log: print( Log.OKBLUE + "{0}".format(msg[0:31]) + "{0}".format(space[0:space_to_add]) + - " [" + Log.ENDC + Log.FAIL + "OK" + + " [" + Log.ENDC + Log.FAIL + "KO" + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.FAIL + msg + Log.ENDC) From 0e356edba97b570a53bcb7db68143973c2dd11d3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 02:23:40 +0200 Subject: [PATCH 20/44] Several improvements * logging improvements * gpg keys import improvements --- install | 110 ++++++++++++++-------------- tests/travis.sh | 2 +- wo/cli/plugins/site_functions.py | 2 +- wo/cli/plugins/stack.py | 1 + wo/cli/plugins/stack_pref.py | 17 +++-- wo/cli/templates/nextcloud.mustache | 82 +++++++++++++++++++++ wo/core/apt_repo.py | 3 +- wo/core/variables.py | 6 +- 8 files changed, 152 insertions(+), 71 deletions(-) create mode 100644 wo/cli/templates/nextcloud.mustache diff --git a/install b/install index 97d8544..b571510 100755 --- a/install +++ b/install @@ -390,7 +390,7 @@ wo_update_wp_cli() { wo_install_acme_sh() { # check if acme.sh is already installed - if [ ! -x /etc/letsencrypt/acme.sh ]; then + if [ ! -x /etc/letsencrypt/acme.sh ] && [ ! -d /etc/letsencrypt/{renewal,archive} ]; then { # clone the git repository if [ -d /opt/acme.sh/.git ]; then @@ -731,75 +731,71 @@ wo_uninstall() { } wo_ufw_setup() { - - # get custom ssh port - if [ -f /etc/ssh/sshd_config ]; then - CURRENT_SSH_PORT=$(grep "Port" /etc/ssh/sshd_config | awk -F " " '{print $2}') - fi - # define firewall rules - if ! grep -q "LOGLEVEL=low" /etc/ufw/ufw.conf; then - ufw logging low - fi - if ! grep -q 'DEFAULT_OUTPUT_POLICY="ACCEPT"' /etc/default/ufw; then - ufw default allow outgoing - fi - if ! grep -q 'DEFAULT_INPUT_POLICY="DROP"' /etc/default/ufw; then - ufw default deny incoming - fi - if ! grep -q "\-\-dport 22 -j" /etc/ufw/user.rules; then - # default ssh port - ufw limit 22 - fi - - # custom ssh port - if [ "$CURRENT_SSH_PORT" != "22" ]; then - if ! grep -q "\-\-dport $CURRENT_SSH_PORT -j" /etc/ufw/user.rules; then - ufw limit "$CURRENT_SSH_PORT" + if ! grep -q "ENABLED=yes" /etc/ufw/ufw.conf; then + # get custom ssh port + if [ -f /etc/ssh/sshd_config ]; then + CURRENT_SSH_PORT=$(grep "Port" /etc/ssh/sshd_config | awk -F " " '{print $2}') + fi + # define firewall rules + if ! grep -q "LOGLEVEL=low" /etc/ufw/ufw.conf; then + ufw logging low + fi + if ! grep -q 'DEFAULT_OUTPUT_POLICY="ACCEPT"' /etc/default/ufw; then + ufw default allow outgoing + fi + if ! grep -q 'DEFAULT_INPUT_POLICY="DROP"' /etc/default/ufw; then + ufw default deny incoming + fi + if ! grep -q "\-\-dport 22 -j" /etc/ufw/user.rules; then + # default ssh port + ufw limit 22 fi - fi - # nginx - if ! grep -q "\-\-dport 80 -j" /etc/ufw/user.rules; then - # http - ufw allow http - fi - if ! grep -q "\-\-dport 443 -j" /etc/ufw/user.rules; then - # https - ufw allow https - fi + # custom ssh port + if [ "$CURRENT_SSH_PORT" != "22" ]; then + if ! grep -q "\-\-dport $CURRENT_SSH_PORT -j" /etc/ufw/user.rules; then + ufw limit "$CURRENT_SSH_PORT" + fi + fi - # ntp - if ! grep -q "\-\-dport 123 -j" /etc/ufw/user.rules; then - ufw allow 123 - fi + # nginx + if ! grep -q "\-\-dport 80 -j" /etc/ufw/user.rules; then + # http + ufw allow http + fi + if ! grep -q "\-\-dport 443 -j" /etc/ufw/user.rules; then + # https + ufw allow https + fi - if ! grep -q "\-\-dport 22222 -j" /etc/ufw/user.rules; then - # wordops backend - ufw limit 22222 - fi - # enable ufw - if [ -n "$CURRENT_SSH_PORT" ]; then - if ! grep -q "ENABLED=yes" /etc/ufw/ufw.conf; then + # ntp + if ! grep -q "\-\-dport 123 -j" /etc/ufw/user.rules; then + ufw allow 123 + fi + + if ! grep -q "\-\-dport 22222 -j" /etc/ufw/user.rules; then + # wordops backend + ufw limit 22222 + fi + # enable ufw + if [ -n "$CURRENT_SSH_PORT" ]; then ufw --force enable - else - ufw reload + fi + + # remove ufw from syslog + if [ -f /etc/rsyslog.d/20-ufw.conf ]; then + sed -i 's/\#\& stop/\& stop/' /etc/rsyslog.d/20-ufw.conf + service rsyslog restart fi fi - - # remove ufw from syslog - if [ -f /etc/rsyslog.d/20-ufw.conf ]; then - sed -i 's/\#\& stop/\& stop/' /etc/rsyslog.d/20-ufw.conf - service rsyslog restart - fi - } \ >> $wo_install_log wo_cheat_install() { curl -sL https://cht.sh/:cht.sh > /usr/local/bin/cht.sh chmod +x /usr/local/bin/cht.sh - [ ! -h /usr/local/bin/cheat ] && { - ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat + [ ! -h /usr/local/bin/cheat ] && { + ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat } curl -sL https://cheat.sh/:bash_completion > /etc/bash_completion.d/cht.sh } diff --git a/tests/travis.sh b/tests/travis.sh index 747379a..2542b61 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -10,7 +10,7 @@ CGREEN="${CSI}1;32m" CEND="${CSI}0m" exit_script() { - curl --progress-bar --upload-file /var/log/wo/wordops.log https://transfer.vtbox.net/"$(basename wordops.log)" && echo "" | sudo tee -a $HOME/.transfer.log && echo "" + curl --progress-bar --upload-file /var/log/wo/wordops.log https://transfer.vtbox.net/"$(basename wordops.log)" && echo "" exit 1 } diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 80ff436..db8b466 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -736,7 +736,7 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): )) except CommandExecutionError as e: Log.failed(self, "Setting plugin {0}" - .format(plugin_name)) + .format(plugin_name)) Log.debug(self, "{0}".format(e)) raise SiteError("plugin setup failed") else: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index ec55bf9..a0d8ebd 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -548,6 +548,7 @@ class WOStackController(CementBaseController): pargs.proftpd = True pargs.utils = True pargs.redis = True + pargs.security = True packages = packages + ['/var/www/22222/htdocs'] if pargs.web: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index b131c13..3ac7dca 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -26,6 +26,7 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): + apt_repo_key = [] """Pre settings to do before installation packages""" if (set(["mariadb-server"]).issubset(set(apt_packages)) or @@ -41,12 +42,8 @@ def pre_pref(self, apt_packages): 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVariables.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - WORepo.add_key(self, '0xF1656F24C74CD1D8', - keyserver="keyserver.ubuntu.com") + apt_repo_key = (apt_repo_key + + ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8']) if set(["mariadb-server"]).issubset(set(apt_packages)): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) @@ -114,6 +111,8 @@ def pre_pref(self, apt_packages): WORepo.add(self, ppa=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding ppa for Nginx') else: + apt_repo_key = apt_repo_key + WOVariables.wo_nginx_key + WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') WORepo.add_key(self, WOVariables.wo_nginx_key) @@ -137,6 +136,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding repo_url of php for debian') WORepo.add(self, repo_url=WOVariables.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') + apt_repo_key = apt_repo_key + WOVariables.wo_php_key WORepo.add_key(self, WOVariables.wo_php_key) # add redis repository if set(['redis-server']).issubset(set(apt_packages)): @@ -145,6 +145,9 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding ppa for redis') WORepo.add(self, ppa=WOVariables.wo_redis_repo) + if (apt_repo_key): + WORepo.add_key(self, apt_repo_key) + def post_pref(self, apt_packages, packages, upgrade=False): """Post activity after installation of packages""" @@ -367,7 +370,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOTemplate.render( self, '/etc/nginx/sites-available/22222', - '22222.mustache', data, overwrite=False) + '22222.mustache', data, overwrite=True) passwd = ''.join([random.choice (string.ascii_letters + string.digits) for n in range(24)]) diff --git a/wo/cli/templates/nextcloud.mustache b/wo/cli/templates/nextcloud.mustache new file mode 100644 index 0000000..f907de7 --- /dev/null +++ b/wo/cli/templates/nextcloud.mustache @@ -0,0 +1,82 @@ +# WordOps nextcloud configuration + add_header X-Robots-Tag none; + add_header X-Permitted-Cross-Domain-Policies none; + add_header Referrer-Policy no-referrer; + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + # Enable gzip but do not remove ETag headers + gzip on; + gzip_vary on; + gzip_comp_level 4; + gzip_min_length 256; + gzip_proxied expired no-cache no-store private no_last_modified no_etag auth; + gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy; + + location / { + rewrite ^ /index.php$request_uri; + } + + location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ { + deny all; + } + location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) { + deny all; + } + + location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) { + fastcgi_split_path_info ^(.+?\.php)(\/.*|)$; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_param HTTPS on; + # Avoid sending the security headers twice + fastcgi_param modHeadersAvailable true; + # Enable pretty urls + fastcgi_param front_controller_active true; + fastcgi_pass {{upstream}}; + fastcgi_intercept_errors on; + fastcgi_request_buffering off; + } + + location ~ ^\/(?:updater|oc[ms]-provider)(?:$|\/) { + try_files $uri/ =404; + index index.php; + } + + # Adding the cache control header for js, css and map files + # Make sure it is BELOW the PHP block + location ~ \.(?:css|js|woff2?|svg|gif|map)$ { + try_files $uri /index.php$request_uri; + add_header Cache-Control "public, max-age=15778463"; + # Add headers to serve security related headers (It is intended to + # have those duplicated to the ones above) + # Before enabling Strict-Transport-Security headers please read into + # this topic first. + #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;"; + # + # WARNING: Only add the preload option once you read about + # the consequences in https://hstspreload.org/. This option + # will add the domain to a hardcoded list that is shipped + # in all major browsers and getting removed from this list + # could take several months. + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Robots-Tag none; + add_header X-Download-Options noopen; + add_header X-Permitted-Cross-Domain-Policies none; + add_header Referrer-Policy no-referrer; + + # Optional: Don't log access to assets + access_log off; + } + + location ~ \.(?:png|html|ttf|ico|jpg|jpeg|bcmap)$ { + try_files $uri /index.php$request_uri; + # Optional: Don't log access to other assets + access_log off; + } diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index 1715639..fed5bac 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -83,7 +83,8 @@ class WORepo(): default keyserver is hkp://keyserver.ubuntu.com user can provide other keyserver with keyserver="hkp://xyz" """ + all_keys = ' '.join(keyids) WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" .format(serv=(keyserver or "hkp://keyserver.ubuntu.com")) + - " --recv-keys {key}".format(key=keyids)) + " --recv-keys {keys}".format(key=all_keys)) diff --git a/wo/core/variables.py b/wo/core/variables.py index 10349b5..22a8a17 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -109,7 +109,7 @@ class WOVariables(): .format(wo_deb_repo)) wo_nginx = ["nginx-custom", "nginx-wo"] - wo_nginx_key = '188C9FB063F0247A' + wo_nginx_key = ['188C9FB063F0247A'] # PHP repo and packages if wo_distro == 'ubuntu': @@ -118,7 +118,7 @@ class WOVariables(): wo_php_repo = ( "deb https://packages.sury.org/php/ {codename} main" .format(codename=wo_platform_codename)) - wo_php_key = 'AC0E47584A7A714D' + wo_php_key = ['AC0E47584A7A714D'] wo_php = ["php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap", "php7.2-readline", "php7.2-common", "php7.2-recode", @@ -150,8 +150,6 @@ class WOVariables(): else: wo_mysql_client = ["mariadb-client", "python3-mysql.connector"] - - wo_fail2ban = ["fail2ban"] wo_clamav = ["clamav", "clamav-freshclam"] From 9a231840c49f95fcc99a373538856f8411312e29 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 02:34:33 +0200 Subject: [PATCH 21/44] Fix wo upgrade --nginx --- wo/cli/plugins/stack_upgrade.py | 84 +++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 1f6e2b2..4f052c3 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -94,7 +94,7 @@ class WOStackUpgradeController(CementBaseController): if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): - nginx_packages = nginx_packages + WOVariables.wo_nginx + apt_packages = apt_packages + WOVariables.wo_nginx else: Log.info(self, "Nginx Stable is not already installed") @@ -178,41 +178,56 @@ class WOStackUpgradeController(CementBaseController): "Composer"]] else: Log.error(self, "Composer isn't installed") - if len(apt_packages) or len(packages): - if len(apt_packages): - Log.info(self, "Your site may be down for few seconds if " - "you are upgrading Nginx, PHP-FPM, MariaDB or Redis") + + if ((not (apt_packages)) and (not(packages))): + self.app.args.print_help() + else: + if (apt_packages): + if not (set(["php7.2-fpm"]).issubset(set(apt_packages)) and + set(["php7.3-fpm"]).issubset(set(apt_packages)) and + set(["nginx-custom", + "nginx-wo"]).issubset(set(apt_packages)) and + set(['mariadb-server']).issubset(set(apt_packages))): + pass + else: + Log.info( + self, "Your site may be down for few seconds if " + "you are upgrading Nginx, PHP-FPM, MariaDB or Redis") # Check prompt if ((not pargs.no_prompt) and (not pargs.force)): start_upgrade = input("Do you want to continue:[y/N]") if start_upgrade != "Y" and start_upgrade != "y": Log.error(self, "Not starting package update") - Log.info(self, "Updating APT packages, please wait...") - - pre_pref(self, nginx_packages) + Log.wait(self, "Updating APT packages") # apt-get update WOAptGet.update(self) - if set(WOVariables.wo_php).issubset(set(apt_packages)): + Log.valide(self, "Updating APT packages") + Log.wait(self, "Upgrading APT Packages") + + # additional pre_pref + if ["nginx-custom"] in apt_packages: + pre_pref(self, WOVariables.wo_nginx) + if ["php7.2-fpm"] in apt_packages: WOAptGet.remove(self, ['php7.2-fpm'], auto=False, purge=True) - if set(WOVariables.wo_php73).issubset(set(apt_packages)): + if ["php7.3-fpm"] in apt_packages: WOAptGet.remove(self, ['php7.3-fpm'], auto=False, purge=True) - # Update packages - if not os.path.isfile( - '/etc/apt/preferences.d/nginx-block'): - WOAptGet.install(self, nginx_packages) - Log.wait(self, "Upgrading APT Packages ") + # check if nginx upgrade is blocked + if os.path.isfile( + '/etc/apt/preferences.d/nginx-block'): + apt_packages.remove(WOVariables.wo_nginx) + post_pref(self, WOVariables.wo_nginx, [], True) + # upgrade packages WOAptGet.install(self, apt_packages) - Log.valide(self, "Upgrading APT Packages ") - Log.wait(self, "Configuring APT Packages ") - post_pref(self, nginx_packages, [], True) - Log.valide(self, "Configuring APT Packages ") + Log.valide(self, "Upgrading APT Packages") + Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) + Log.valide(self, "Configuring APT Packages") # Post Actions after package updates - if len(packages): + if (packages): if pargs.wpcli: WOFileUtils.rm(self, '/usr/local/bin/wp') @@ -229,7 +244,7 @@ class WOStackUpgradeController(CementBaseController): WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) if pargs.netdata: - Log.wait(self, "Upgrading Netdata ") + Log.wait(self, "Upgrading Netdata") if os.path.isdir('/opt/netdata'): WOShellExec.cmd_exec( self, "bash /opt/netdata/usr/" @@ -240,7 +255,7 @@ class WOStackUpgradeController(CementBaseController): self, "bash /usr/" "libexec/netdata/netdata-" "updater.sh") - Log.valide(self, "Upgrading Netdata ") + Log.valide(self, "Upgrading Netdata") if pargs.dashboard: Log.debug(self, "Extracting wo-dashboard.tar.gz " @@ -252,21 +267,22 @@ class WOStackUpgradeController(CementBaseController): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, "{0}22222/htdocs" .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + 'www-data', + 'www-data', recursive=True) if pargs.composer: - Log.wait(self, "Upgrading Composer ") - WOShellExec.cmd_exec(self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") + Log.wait(self, "Upgrading Composer ") + WOShellExec.cmd_exec( + self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") shutil.copyfile('/var/lib/wo/tmp/composer.phar', '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - Log.valide(self, "Upgrading Composer ") + Log.valide(self, "Upgrading Composer ") if pargs.phpmyadmin: - Log.wait(self, "Upgrading phpMyAdmin ") + Log.wait(self, "Upgrading phpMyAdmin ") WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') shutil.copyfile(('{0}22222/htdocs/db/pma' @@ -285,10 +301,8 @@ class WOStackUpgradeController(CementBaseController): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, "{0}22222/htdocs" .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) - Log.valide(self, "Upgrading phpMyAdmin ") + 'www-data', + 'www-data', recursive=True) + Log.valide(self, "Upgrading phpMyAdmin ") Log.info(self, "Successfully updated packages") - else: - self.app.args.print_help() From 1d1e85fd2098d2e9164e0df09b87d2747622ffb5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 02:41:11 +0200 Subject: [PATCH 22/44] Fix apt-key add --- wo/cli/plugins/stack_pref.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 3ac7dca..31ce5ff 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -26,12 +26,11 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): - apt_repo_key = [] """Pre settings to do before installation packages""" + apt_repo_keys = [] - if (set(["mariadb-server"]).issubset(set(apt_packages)) or - set(["mariadb-client"]).issubset(set(apt_packages)) or - set(["mariadb-backup"]).issubset((set(apt_packages)))): + if ((["mariadb-server"] in apt_packages) or + (["mariadb-client"] in apt_packages)): # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") @@ -42,9 +41,9 @@ def pre_pref(self, apt_packages): 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - apt_repo_key = (apt_repo_key + - ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8']) - if set(["mariadb-server"]).issubset(set(apt_packages)): + apt_repo_keys = (apt_repo_keys + + ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8']) + if ["mariadb-server"] in apt_packages: # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) @@ -111,11 +110,9 @@ def pre_pref(self, apt_packages): WORepo.add(self, ppa=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding ppa for Nginx') else: - apt_repo_key = apt_repo_key + WOVariables.wo_nginx_key - + apt_repo_keys = apt_repo_keys + WOVariables.wo_nginx_key WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') - WORepo.add_key(self, WOVariables.wo_nginx_key) # add php repository if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or @@ -136,8 +133,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding repo_url of php for debian') WORepo.add(self, repo_url=WOVariables.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') - apt_repo_key = apt_repo_key + WOVariables.wo_php_key - WORepo.add_key(self, WOVariables.wo_php_key) + apt_repo_keys = apt_repo_keys + WOVariables.wo_php_key # add redis repository if set(['redis-server']).issubset(set(apt_packages)): Log.info(self, "Adding repository for Redis, please wait...") @@ -145,7 +141,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding ppa for redis') WORepo.add(self, ppa=WOVariables.wo_redis_repo) - if (apt_repo_key): + if (apt_repo_keys): WORepo.add_key(self, apt_repo_key) From e4aff18587d45c6dbf8a01513e4e9589e5314ab6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 02:42:12 +0200 Subject: [PATCH 23/44] Fix install --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index b571510..c59c07c 100755 --- a/install +++ b/install @@ -390,7 +390,7 @@ wo_update_wp_cli() { wo_install_acme_sh() { # check if acme.sh is already installed - if [ ! -x /etc/letsencrypt/acme.sh ] && [ ! -d /etc/letsencrypt/{renewal,archive} ]; then + if [ ! -x /etc/letsencrypt/acme.sh ] && [ ! -d /etc/letsencrypt/archive ]; then { # clone the git repository if [ -d /opt/acme.sh/.git ]; then From 4887452819834d3525a21407ad332f9cb0629530 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 02:47:53 +0200 Subject: [PATCH 24/44] Fix typo --- wo/cli/plugins/stack_pref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 31ce5ff..048fe80 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -142,7 +142,7 @@ def pre_pref(self, apt_packages): WORepo.add(self, ppa=WOVariables.wo_redis_repo) if (apt_repo_keys): - WORepo.add_key(self, apt_repo_key) + WORepo.add_key(self, apt_repo_keys) def post_pref(self, apt_packages, packages, upgrade=False): From 75df2283c329fa9e1fc02fbb4744d5be52b31c08 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 12:07:34 +0200 Subject: [PATCH 25/44] Fix MySQL Install --- wo/cli/plugins/stack.py | 1 + wo/core/apt_repo.py | 18 +++++++++++++++--- wo/core/variables.py | 6 ++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index a0d8ebd..c1e9b02 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -11,6 +11,7 @@ import string import psutil import requests + from cement.core import handler, hook from cement.core.controller import CementBaseController, expose diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index fed5bac..e024180 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -48,8 +48,9 @@ class WORepo(): Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to add repo") if ppa is not None: - WOShellExec.cmd_exec(self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" - .format(ppa_name=ppa)) + WOShellExec.cmd_exec( + self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" + .format(ppa_name=ppa)) def remove(self, ppa=None, repo_url=None): """ @@ -77,7 +78,18 @@ class WORepo(): Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to remove repo") - def add_key(self, keyids, keyserver=None): + def add_key(self, keyid, keyserver=None): + """ + This function adds imports repository keys from keyserver. + default keyserver is hkp://keyserver.ubuntu.com + user can provide other keyserver with keyserver="hkp://xyz" + """ + WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" + .format(serv=(keyserver or + "hkp://keyserver.ubuntu.com")) + + " --recv-keys {keys}".format(key=keyid)) + + def add_keys(self, keyids, keyserver=None): """ This function adds imports repository keys from keyserver. default keyserver is hkp://keyserver.ubuntu.com diff --git a/wo/core/variables.py b/wo/core/variables.py index 22a8a17..e318e31 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -109,7 +109,7 @@ class WOVariables(): .format(wo_deb_repo)) wo_nginx = ["nginx-custom", "nginx-wo"] - wo_nginx_key = ['188C9FB063F0247A'] + wo_nginx_key = '188C9FB063F0247A' # PHP repo and packages if wo_distro == 'ubuntu': @@ -118,7 +118,7 @@ class WOVariables(): wo_php_repo = ( "deb https://packages.sury.org/php/ {codename} main" .format(codename=wo_platform_codename)) - wo_php_key = ['AC0E47584A7A714D'] + wo_php_key = 'AC0E47584A7A714D' wo_php = ["php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap", "php7.2-readline", "php7.2-common", "php7.2-recode", @@ -150,6 +150,8 @@ class WOVariables(): else: wo_mysql_client = ["mariadb-client", "python3-mysql.connector"] + wo_mysql_keys = ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8'] + wo_fail2ban = ["fail2ban"] wo_clamav = ["clamav", "clamav-freshclam"] From 46891dd12a441be771011e54abdb88bb7f2a4fe7 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 12:08:46 +0200 Subject: [PATCH 26/44] Fix repo add keys --- wo/core/apt_repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index e024180..be2e3e3 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -87,7 +87,7 @@ class WORepo(): WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" .format(serv=(keyserver or "hkp://keyserver.ubuntu.com")) + - " --recv-keys {keys}".format(key=keyid)) + " --recv-keys {key}".format(key=keyid)) def add_keys(self, keyids, keyserver=None): """ @@ -99,4 +99,4 @@ class WORepo(): WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" .format(serv=(keyserver or "hkp://keyserver.ubuntu.com")) + - " --recv-keys {keys}".format(key=all_keys)) + " --recv-keys {0}".format(all_keys)) From cc2cb6a98679dcef2362bce2bd3ff99e5f6880f8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 12:40:29 +0200 Subject: [PATCH 27/44] Fix MySQL install --- wo/cli/plugins/stack_pref.py | 11 ++++------- wo/cli/plugins/stack_upgrade.py | 1 - wo/core/apt_repo.py | 26 ++++++++++++++++++-------- wo/core/aptget.py | 2 +- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 048fe80..54a8e8e 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -27,7 +27,6 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - apt_repo_keys = [] if ((["mariadb-server"] in apt_packages) or (["mariadb-client"] in apt_packages)): @@ -41,8 +40,8 @@ def pre_pref(self, apt_packages): 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - apt_repo_keys = (apt_repo_keys + - ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8']) + WORepo.add_keys(self, WOVariables.wo_mysql_keys, + keyserver='hkp://keys.gnupg.net:80') if ["mariadb-server"] in apt_packages: # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) @@ -113,6 +112,7 @@ def pre_pref(self, apt_packages): apt_repo_keys = apt_repo_keys + WOVariables.wo_nginx_key WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') + WORepo.add_key(self, WOVariables.wo_nginx_key) # add php repository if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or @@ -133,7 +133,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding repo_url of php for debian') WORepo.add(self, repo_url=WOVariables.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') - apt_repo_keys = apt_repo_keys + WOVariables.wo_php_key + WORepo.add_key(self, WOVariables.wo_php_key) # add redis repository if set(['redis-server']).issubset(set(apt_packages)): Log.info(self, "Adding repository for Redis, please wait...") @@ -141,9 +141,6 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding ppa for redis') WORepo.add(self, ppa=WOVariables.wo_redis_repo) - if (apt_repo_keys): - WORepo.add_key(self, apt_repo_keys) - def post_pref(self, apt_packages, packages, upgrade=False): """Post activity after installation of packages""" diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 4f052c3..694a198 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -62,7 +62,6 @@ class WOStackUpgradeController(CementBaseController): # All package update apt_packages = [] packages = [] - nginx_packages = [] self.msg = [] pargs = self.app.pargs diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index be2e3e3..1684b6e 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -84,10 +84,15 @@ class WORepo(): default keyserver is hkp://keyserver.ubuntu.com user can provide other keyserver with keyserver="hkp://xyz" """ - WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" - .format(serv=(keyserver or - "hkp://keyserver.ubuntu.com")) + - " --recv-keys {key}".format(key=keyid)) + try: + WOShellExec.cmd_exec( + self, "apt-key adv --keyserver {serv}" + .format(serv=(keyserver or + "hkp://keyserver.ubuntu.com")) + + " --recv-keys {key}".format(key=keyid)) + except Exception as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to import repo key") def add_keys(self, keyids, keyserver=None): """ @@ -96,7 +101,12 @@ class WORepo(): user can provide other keyserver with keyserver="hkp://xyz" """ all_keys = ' '.join(keyids) - WOShellExec.cmd_exec(self, "apt-key adv --keyserver {serv}" - .format(serv=(keyserver or - "hkp://keyserver.ubuntu.com")) + - " --recv-keys {0}".format(all_keys)) + try: + WOShellExec.cmd_exec( + self, "apt-key adv --keyserver {serv}" + .format(serv=(keyserver or + "hkp://keyserver.ubuntu.com")) + + " --recv-keys {keys}".format(keys=all_keys)) + except Exception as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to import repo keys") diff --git a/wo/core/aptget.py b/wo/core/aptget.py index 3bff57e..e41a4f6 100644 --- a/wo/core/aptget.py +++ b/wo/core/aptget.py @@ -54,7 +54,7 @@ class WOAptGet(): "`tail /var/log/wo/wordops.log` " "and please try again...") - except Exception as e: + except Exception: Log.error(self, "apt-get update exited with error") def check_upgrade(self): From ea449e2626459d87d6a476252205db12b5e80e35 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 13:00:24 +0200 Subject: [PATCH 28/44] Fix MariaDB setup --- wo/cli/plugins/stack_pref.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 54a8e8e..9d368bb 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -40,8 +40,10 @@ def pre_pref(self, apt_packages): 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - WORepo.add_keys(self, WOVariables.wo_mysql_keys, - keyserver='hkp://keys.gnupg.net:80') + WORepo.add_key(self, '0xcbcb082a1bb943db', + keyserver='hkp://keys.gnupg.net') + WORepo.add_key(self, '0xF1656F24C74CD1D8', + keyserver='hkp://keys.gnupg.net') if ["mariadb-server"] in apt_packages: # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) From d27d17cd4b1c5412bee1c9e5d9818b6d19a5abf4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 13:09:26 +0200 Subject: [PATCH 29/44] Another fix --- wo/cli/plugins/stack_pref.py | 8 +++++--- wo/core/variables.py | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 9d368bb..4a41253 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -28,8 +28,10 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - if ((["mariadb-server"] in apt_packages) or - (["mariadb-client"] in apt_packages)): + if (not(["mariadb-server"] in apt_packages) and + (not ["mariadb-client"] in apt_packages)): + pass + else: # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") @@ -41,7 +43,7 @@ def pre_pref(self, apt_packages): mysql_pref_file.write(mysql_pref) WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver='hkp://keys.gnupg.net') + keyserver='keys.gnupg.net') WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver='hkp://keys.gnupg.net') if ["mariadb-server"] in apt_packages: diff --git a/wo/core/variables.py b/wo/core/variables.py index e318e31..69b223c 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -150,8 +150,6 @@ class WOVariables(): else: wo_mysql_client = ["mariadb-client", "python3-mysql.connector"] - wo_mysql_keys = ['0xcbcb082a1bb943db', '0xF1656F24C74CD1D8'] - wo_fail2ban = ["fail2ban"] wo_clamav = ["clamav", "clamav-freshclam"] From e105aff3a3ab57a1209dc31aa868e17c7d4b4c9e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 13:17:37 +0200 Subject: [PATCH 30/44] Fix MariaDB variable --- wo/cli/plugins/stack.py | 12 +----------- wo/core/variables.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index c1e9b02..d430ff8 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -229,17 +229,7 @@ class WOStackController(CementBaseController): pargs.mysqltuner = True Log.debug(self, "Setting apt_packages variable for MySQL") if not WOShellExec.cmd_exec(self, "mysqladmin ping"): - if not WOVariables.wo_distro == 'raspbian': - if (not WOVariables.wo_platform_codename == 'jessie'): - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb", "mariadb-backup"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] - apt_packages = apt_packages + wo_mysql + apt_packages = apt_packages + WOVariables.wo_mysql else: Log.debug(self, "MySQL already installed and alive") Log.info(self, "MySQL already installed and alive") diff --git a/wo/core/variables.py b/wo/core/variables.py index 69b223c..5e9b19b 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -145,6 +145,17 @@ class WOVariables(): "10.3/debian {codename} main" .format(codename=wo_platform_codename)) + if not wo_distro == 'raspbian': + if (not wo_platform_codename == 'jessie'): + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb", "mariadb-backup"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysql.connector"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb"] + if wo_platform_codename == 'jessie': wo_mysql_client = ["mariadb-client", "python3-mysqldb"] else: From e2ae44714c0cc0e460d93305d8b9712776c36bf4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 13:20:34 +0200 Subject: [PATCH 31/44] Set back for stack pref --- wo/cli/plugins/stack_pref.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 4a41253..2f9fed2 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -28,10 +28,7 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - if (not(["mariadb-server"] in apt_packages) and - (not ["mariadb-client"] in apt_packages)): - pass - else: + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") From ecc938c11f087de76f50437f25f031b717db4540 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:27:45 +0200 Subject: [PATCH 32/44] Improve stack_pref --- wo/cli/plugins/stack.py | 4 +- wo/cli/plugins/stack_pref.py | 136 +++++++++++++---------------------- wo/core/sslutils.py | 86 ++++++++++++++++++++++ 3 files changed, 138 insertions(+), 88 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index d430ff8..537ffd5 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -493,13 +493,13 @@ class WOStackController(CementBaseController): WOAptGet.install(self, apt_packages) Log.valide(self, "Installing APT packages ") Log.wait(self, "Configuring APT packages ") - post_pref(self, apt_packages, empty_packages) + post_pref(self, apt_packages, []) Log.valide(self, "Configuring APT packages ") if (packages): Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) Log.debug(self, "Calling post_pref") - post_pref(self, empty_packages, packages) + post_pref(self, [], packages) if disp_msg: if (self.msg): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2f9fed2..d921382 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -23,12 +23,13 @@ from wo.core.services import WOService from wo.core.shellexec import CommandExecutionError, WOShellExec from wo.core.template import WOTemplate from wo.core.variables import WOVariables +from wo.core.sslutils import SSL def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if ("mariadb-server" in apt_packages or "mariadb-client" in apt_packages): # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") @@ -43,7 +44,7 @@ def pre_pref(self, apt_packages): keyserver='keys.gnupg.net') WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver='hkp://keys.gnupg.net') - if ["mariadb-server"] in apt_packages: + if "mariadb-server" in apt_packages: # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) @@ -389,98 +390,61 @@ def post_pref(self, apt_packages, packages, upgrade=False): '/etc/nginx/' 'sites-enabled/' '22222']) - # Create log and cert folder and softlinks - if not os.path.exists('{0}22222/logs' - .format(ngxroot)): - Log.debug(self, "Creating directory " - "{0}22222/logs " - .format(ngxroot)) - os.makedirs('{0}22222/logs' - .format(ngxroot)) - - if not os.path.exists('{0}22222/cert' - .format(ngxroot)): - Log.debug(self, "Creating directory " - "{0}22222/cert" - .format(ngxroot)) - os.makedirs('{0}22222/cert' - .format(ngxroot)) - - if not os.path.isdir('{0}22222/conf/nginx' - .format(ngxroot)): - Log.debug(self, "Creating directory " - "{0}22222/conf/nginx" - .format(ngxroot)) - os.makedirs('{0}22222/conf/nginx' - .format(ngxroot)) - - WOFileUtils.create_symlink( - self, - ['/var/log/nginx/' - '22222.access.log', - '{0}22222/' - 'logs/access.log' - .format(ngxroot)] - ) - - WOFileUtils.create_symlink( - self, - ['/var/log/nginx/' - '22222.error.log', - '{0}22222/' - 'logs/error.log' - .format(ngxroot)] - ) - - try: - WOShellExec.cmd_exec( - self, "openssl genrsa -out " - "{0}22222/cert/22222.key 2048" - .format(ngxroot)) - WOShellExec.cmd_exec( - self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key {0}22222/cert/22222.key " - "-out {0}22222/cert/" - "22222.csr" + # Create log and cert folder and softlinks + if not os.path.exists('{0}22222/logs' + .format(ngxroot)): + Log.debug(self, "Creating directory " + "{0}22222/logs " + .format(ngxroot)) + os.makedirs('{0}22222/logs' .format(ngxroot)) - WOFileUtils.mvfile( - self, "{0}22222/cert/22222.key" - .format(ngxroot), - "{0}22222/cert/" - "22222.key.org" + if not os.path.exists('{0}22222/cert' + .format(ngxroot)): + Log.debug(self, "Creating directory " + "{0}22222/cert" + .format(ngxroot)) + os.makedirs('{0}22222/cert' .format(ngxroot)) - WOShellExec.cmd_exec( - self, "openssl rsa -in " - "{0}22222/cert/" - "22222.key.org -out " - "{0}22222/cert/22222.key" + if not os.path.isdir('{0}22222/conf/nginx' + .format(ngxroot)): + Log.debug(self, "Creating directory " + "{0}22222/conf/nginx" + .format(ngxroot)) + os.makedirs('{0}22222/conf/nginx' .format(ngxroot)) - WOShellExec.cmd_exec( - self, "openssl x509 -req -days " - "3652 -in {0}22222/cert/" - "22222.csr -signkey {0}" - "22222/cert/22222.key -out " - "{0}22222/cert/22222.crt" - .format(ngxroot)) + WOFileUtils.create_symlink( + self, + ['/var/log/nginx/' + '22222.access.log', + '{0}22222/' + 'logs/access.log' + .format(ngxroot)] + ) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate HTTPS " - "certificate for 22222", False) + WOFileUtils.create_symlink( + self, + ['/var/log/nginx/' + '22222.error.log', + '{0}22222/' + 'logs/error.log' + .format(ngxroot)] + ) + if (not os.path.isfile('{0}22222/cert/22222.key' + .format(ngxroot))): + SSL.selfsignedcert(self, 'localhost', + '', backend=True) - if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' - .format(ngxroot)): - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "w") as php_file: - php_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") + if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' + .format(ngxroot)): + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "w") as php_file: + php_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") server_ip = requests.get('http://v4.wordops.eu') diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index a54dfe2..e329e85 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -39,6 +39,9 @@ class SSL: def getexpirationdate(self, domain): # check if exist + if os.path.islink('/var/www/{0}/conf/nginx/ssl.conf'): + split_domain = domain.split('.') + domain = ('.').join(split_domain[1:]) if not os.path.isfile('/etc/letsencrypt/live/{0}/cert.pem' .format(domain)): Log.error(self, 'File Not Found: /etc/letsencrypt/' @@ -115,3 +118,86 @@ class SSL: certfile.close() return iswildcard + + def setupHsts(self, wo_domain_name): + Log.info( + self, "Adding /var/www/{0}/conf/nginx/hsts.conf" + .format(wo_domain_name)) + + hstsconf = open("/var/www/{0}/conf/nginx/hsts.conf" + .format(wo_domain_name), + encoding='utf-8', mode='w') + hstsconf.write("more_set_headers " + "\"Strict-Transport-Security: " + "max-age=31536000; " + "includeSubDomains; " + "preload\";") + hstsconf.close() + return 0 + + def selfsignedcert(self, wo_domain_name, + cert_path, backend=False): + """issue a self-signed certificate""" + + selfs_tmp = '/var/lib/wo/tmp/selfssl' + # create self-signed tmp directory + if not os.path.isdir(selfs_tmp): + WOFileUtils.mkdir(selfs_tmp) + if wo_domain_name == '': + wo_domain_name = 'localhost' + try: + WOShellExec.cmd_exec( + self, "openssl genrsa -out " + "{0}/ssl.key 2048" + .format(selfs_tmp)) + WOShellExec.cmd_exec( + self, "openssl req -new -batch " + "-subj /commonName={0}/ " + "-key {1}/ssl.key -out {1}/ssl.csr" + .format(wo_domain_name, selfs_tmp)) + + WOFileUtils.mvfile( + self, "{0}/ssl.key" + .format(selfs_tmp), + "{0}/ssl.key.org" + .format(selfs_tmp)) + + WOShellExec.cmd_exec( + self, "openssl rsa -in " + "{0}/ssl.key.org -out " + "{0}/ssl.key" + .format(selfs_tmp)) + + WOShellExec.cmd_exec( + self, "openssl x509 -req -days " + "3652 -in {0}/ssl.csr -signkey {0}" + "/ssl.key -out {0}/ssl.crt" + .format(selfs_tmp)) + + except Exception as e: + Log.debug(self, "{0}".format(e)) + Log.error( + self, "Failed to generate HTTPS " + "certificate for 22222", False) + if backend: + WOFileUtils.mvfile( + self, "{0}/ssl.key" + .format(selfs_tmp), + "/var/www/22222/cert/22222.key") + WOFileUtils.mvfile( + self, "{0}/ssl.cert" + .format(selfs_tmp), + "/var/www/22222/cert/22222.crt") + else: + if not os.path.isdir(cert_path): + WOFileUtils.mkdir(self, cert_path) + WOFileUtils.mvfile( + self, "{0}/ssl.key" + .format(selfs_tmp), + "{0}/key.pem".format(cert_path)) + WOFileUtils.mvfile( + self, "{0}/ssl.crt" + .format(selfs_tmp), + "{0}/cert.pem".format(cert_path)) + # remove self-signed tmp directory + WOFileUtils.rm(self, selfs_tmp) From 322ba456620739f3ca89f129ebf670448dcfad54 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:37:47 +0200 Subject: [PATCH 33/44] Fix redis install --- wo/cli/plugins/stack.py | 18 ++++++------------ wo/cli/plugins/stack_pref.py | 8 ++++---- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 537ffd5..95e24bf 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -192,7 +192,7 @@ class WOStackController(CementBaseController): if pargs.redis: pargs.php = True if not WOAptGet.is_installed(self, 'redis-server'): - apt_packages = apt_packages + ["redis-server"] + apt_packages = apt_packages + WOVariables.wo_redis else: Log.info(self, "Redis already installed") @@ -201,11 +201,8 @@ class WOStackController(CementBaseController): if pargs.php: Log.debug(self, "Setting apt_packages variable for PHP 7.2") if not (WOAptGet.is_installed(self, 'php7.2-fpm')): - if not (WOAptGet.is_installed(self, 'php7.3-fpm')): - apt_packages = (apt_packages + WOVariables.wo_php + - WOVariables.wo_php_extra) - else: - apt_packages = apt_packages + WOVariables.wo_php + apt_packages = (apt_packages + WOVariables.wo_php + + WOVariables.wo_php_extra) else: Log.debug(self, "PHP 7.2 already installed") Log.info(self, "PHP 7.2 already installed") @@ -214,12 +211,9 @@ class WOStackController(CementBaseController): if pargs.php73: Log.debug(self, "Setting apt_packages variable for PHP 7.3") if not WOAptGet.is_installed(self, 'php7.3-fpm'): - if not (WOAptGet.is_installed(self, 'php7.2-fpm')): - apt_packages = (apt_packages + WOVariables.wo_php + - WOVariables.wo_php73 + - WOVariables.wo_php_extra) - else: - apt_packages = apt_packages + WOVariables.wo_php73 + apt_packages = (apt_packages + WOVariables.wo_php + + WOVariables.wo_php73 + + WOVariables.wo_php_extra) else: Log.debug(self, "PHP 7.3 already installed") Log.info(self, "PHP 7.3 already installed") diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index d921382..5be5dad 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -137,7 +137,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding deb.sury GPG key') WORepo.add_key(self, WOVariables.wo_php_key) # add redis repository - if set(['redis-server']).issubset(set(apt_packages)): + if set(WOVariables.wo_redis).issubset(set(apt_packages)): Log.info(self, "Adding repository for Redis, please wait...") if WOVariables.wo_distro == 'ubuntu': Log.debug(self, 'Adding ppa for redis') @@ -837,7 +837,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOService.restart_service(self, 'php7.3-fpm') # create mysql config if it doesn't exist - if set(["mariadb-server"]).issubset(set(apt_packages)): + if "mariadb-server" in apt_packages: if not os.path.isfile("/etc/mysql/my.cnf"): config = ("[mysqld]\nwait_timeout = 30\n" "interactive_timeout=60\nperformance_schema = 0" @@ -914,7 +914,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOService.reload_service(self, 'fail2ban') # Proftpd configuration - if set(["proftpd-basic"]).issubset(set(apt_packages)): + if "proftpd-basic" in apt_packages: if os.path.isfile("/etc/proftpd/proftpd.conf"): Log.info(self, "Configuring ProFTPd") Log.debug(self, "Setting up Proftpd configuration") @@ -1002,7 +1002,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOService.reload_service(self, 'proftpd') # Redis configuration - if set(['redus-server']).issubset(set(apt_packages)): + if "redis-server" in apt_packages: if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" "upstream.conf", From 4c415eeabba320fdfeeeb2b4126239a0f6702ba9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:43:07 +0200 Subject: [PATCH 34/44] Fix nginx keys --- wo/cli/plugins/stack_pref.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 5be5dad..bdd023e 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -111,7 +111,6 @@ def pre_pref(self, apt_packages): WORepo.add(self, ppa=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding ppa for Nginx') else: - apt_repo_keys = apt_repo_keys + WOVariables.wo_nginx_key WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') WORepo.add_key(self, WOVariables.wo_nginx_key) From 25382fcc131b04a454c8e9bfffbbbacac3964fff Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:45:55 +0200 Subject: [PATCH 35/44] Do not force php install with redis --- wo/cli/plugins/stack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 95e24bf..2622b7f 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -190,7 +190,6 @@ class WOStackController(CementBaseController): # Redis if pargs.redis: - pargs.php = True if not WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis From 13d8a10b29da5366a76ccae0fc9843de41f7ac90 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:47:50 +0200 Subject: [PATCH 36/44] Fix redis install log --- wo/cli/plugins/stack_pref.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index bdd023e..53a4dde 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1032,7 +1032,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): if (os.path.isfile("/etc/redis/redis.conf") and (not WOFileUtils.grep(self, "/etc/redis/redis.conf", "WordOps"))): - Log.info(self, "Tuning Redis configuration") + Log.wait(self, "Tuning Redis configuration") with open("/etc/redis/redis.conf", "a") as redis_file: redis_file.write("\n# WordOps v3.9.8\n") @@ -1075,7 +1075,8 @@ def post_pref(self, apt_packages, packages, upgrade=False): "tcp-backlog 32768") WOFileUtils.chown(self, '/etc/redis/redis.conf', 'redis', 'redis', recursive=False) - WOService.restart_service(self, 'redis-server') + WOService.restart_service(self, 'redis-server'): + Log.valide(self, "Tuning Redis configuration") # ClamAV configuration if set(WOVariables.wo_clamav).issubset(set(apt_packages)): From 8c21e0c3c344817e4fd78bb71559fe5d4d693a31 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 14:50:44 +0200 Subject: [PATCH 37/44] Fix WOFileUtils --- wo/core/sslutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index e329e85..bbc1873 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -142,7 +142,7 @@ class SSL: selfs_tmp = '/var/lib/wo/tmp/selfssl' # create self-signed tmp directory if not os.path.isdir(selfs_tmp): - WOFileUtils.mkdir(selfs_tmp) + WOFileUtils.mkdir(self, selfs_tmp) if wo_domain_name == '': wo_domain_name = 'localhost' try: From bb4e0404edfeecc26097bb1c3b98ae6a6fefb297 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 15:36:32 +0200 Subject: [PATCH 38/44] Fix typo --- wo/cli/plugins/stack_pref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 53a4dde..fd9f4af 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1075,7 +1075,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): "tcp-backlog 32768") WOFileUtils.chown(self, '/etc/redis/redis.conf', 'redis', 'redis', recursive=False) - WOService.restart_service(self, 'redis-server'): + WOService.restart_service(self, 'redis-server') Log.valide(self, "Tuning Redis configuration") # ClamAV configuration From eeb44ad1a1e720b08546bac8188f8deac1aaa80b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 15:39:52 +0200 Subject: [PATCH 39/44] Update changelog --- CHANGELOG.md | 9 +++++++++ install | 2 ++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad83701..23767a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Changed + +- Improved general logs display + +#### Fixed + +- Redis-server configuration and start +- Nginx upgrade with `wo stack upgrade` + ### v3.9.8.10 - 2019-09-04 #### Changed diff --git a/install b/install index c59c07c..926c222 100755 --- a/install +++ b/install @@ -539,6 +539,8 @@ wo_upgrade_nginx() { sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/php.conf/php72.conf/" /etc/nginx/sites-available/* + sed -i "s/php7.conf/php72.conf/" /etc/nginx/sites-available/* sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf # update redis.conf headers From 72f4bf78bcb544fc0c9324884b2419ffa3ef01f2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 16:01:35 +0200 Subject: [PATCH 40/44] Fix ssl.crt --- wo/core/sslutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index bbc1873..3e6dbf9 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -185,7 +185,7 @@ class SSL: .format(selfs_tmp), "/var/www/22222/cert/22222.key") WOFileUtils.mvfile( - self, "{0}/ssl.cert" + self, "{0}/ssl.crt" .format(selfs_tmp), "/var/www/22222/cert/22222.crt") else: From ea6cd874b5d6ac5259cfa2547931bb2c0bcb8e16 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 16:03:29 +0200 Subject: [PATCH 41/44] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23767a3..020bb19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Improved general logs display +- UFW configuration is only applied during initial installation if UFW is disabled #### Fixed From 4556fafa70299b0ae1fa66a3d86219bab2d14660 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 16:08:26 +0200 Subject: [PATCH 42/44] Add full purge for mysql --- wo/cli/plugins/stack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 2622b7f..5f0b022 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -808,6 +808,7 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing apt_packages variable of MySQL") apt_packages = apt_packages + ['mariadb-server', 'mysql-common', 'mariadb-client'] + packages = packages + ['/etc/mysql', '/var/lib/mysql'] # mysqlclient if pargs.mysqlclient: From 80df8b7e2fe1ee29a20dc44b74f35f0d3ae74660 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 16:13:46 +0200 Subject: [PATCH 43/44] Simplify proftpd setup --- wo/cli/plugins/stack_pref.py | 26 ++------------------------ wo/core/sslutils.py | 19 +++++++------------ 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index fd9f4af..7488f70 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -433,8 +433,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): ) if (not os.path.isfile('{0}22222/cert/22222.key' .format(ngxroot))): - SSL.selfsignedcert(self, 'localhost', - '', backend=True) + SSL.selfsignedcert(self, proftpd=False, backend=True) if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' .format(ngxroot)): @@ -936,28 +935,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): # proftpd TLS configuration if not os.path.isdir("/etc/proftpd/ssl"): WOFileUtils.mkdir(self, "/etc/proftpd/ssl") - - try: - WOShellExec.cmd_exec(self, "openssl genrsa -out " - "/etc/proftpd/ssl/proftpd.key 2048") - WOShellExec.cmd_exec(self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key /etc/proftpd/ssl/proftpd.key " - "-out /etc/proftpd/ssl/proftpd.csr") - WOFileUtils.mvfile(self, "/etc/proftpd/ssl/proftpd.key", - "/etc/proftpd/ssl/proftpd.key.org") - WOShellExec.cmd_exec(self, "openssl rsa -in " - "/etc/proftpd/ssl/proftpd.key.org " - "-out /etc/proftpd/ssl/proftpd.key") - WOShellExec.cmd_exec(self, "openssl x509 -req -days " - "3652 -in /etc/proftpd/ssl/proftpd.csr " - "-signkey /etc/proftpd/ssl/proftpd.key " - " -out /etc/proftpd/ssl/proftpd.crt") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate SSL " - "certificate for Proftpd") + SSL.selfsignedcert(self, proftpd=True, backend=False) WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.key", 0o700) WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.crt", 0o700) data = dict() diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index 3e6dbf9..b04745f 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -135,16 +135,13 @@ class SSL: hstsconf.close() return 0 - def selfsignedcert(self, wo_domain_name, - cert_path, backend=False): + def selfsignedcert(self, proftpd=False, backend=False): """issue a self-signed certificate""" selfs_tmp = '/var/lib/wo/tmp/selfssl' # create self-signed tmp directory if not os.path.isdir(selfs_tmp): WOFileUtils.mkdir(self, selfs_tmp) - if wo_domain_name == '': - wo_domain_name = 'localhost' try: WOShellExec.cmd_exec( self, "openssl genrsa -out " @@ -152,9 +149,9 @@ class SSL: .format(selfs_tmp)) WOShellExec.cmd_exec( self, "openssl req -new -batch " - "-subj /commonName={0}/ " - "-key {1}/ssl.key -out {1}/ssl.csr" - .format(wo_domain_name, selfs_tmp)) + "-subj /commonName=localhost/ " + "-key {0}/ssl.key -out {0}/ssl.csr" + .format(selfs_tmp)) WOFileUtils.mvfile( self, "{0}/ssl.key" @@ -188,16 +185,14 @@ class SSL: self, "{0}/ssl.crt" .format(selfs_tmp), "/var/www/22222/cert/22222.crt") - else: - if not os.path.isdir(cert_path): - WOFileUtils.mkdir(self, cert_path) + if proftpd: WOFileUtils.mvfile( self, "{0}/ssl.key" .format(selfs_tmp), - "{0}/key.pem".format(cert_path)) + "/etc/proftpd/ssl/proftpd.key") WOFileUtils.mvfile( self, "{0}/ssl.crt" .format(selfs_tmp), - "{0}/cert.pem".format(cert_path)) + "/etc/proftpd/ssl/proftpd.crt") # remove self-signed tmp directory WOFileUtils.rm(self, selfs_tmp) From d16f7efb3bb7b7b4282f20b7b333811cdf1b93d8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 6 Sep 2019 16:25:56 +0200 Subject: [PATCH 44/44] update changelog --- CHANGELOG.md | 2 ++ wo/cli/plugins/site_functions.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 020bb19..3777422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +### v3.9.8.11 - 2019-09-06 + #### Changed - Improved general logs display diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index db8b466..b6a1355 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1738,3 +1738,16 @@ def archivedCertificateHandle(self, domain): .format(domain)) return ssl + + +def setuprocketchat(self): + if ((not WOVariables.wo_platform_codename == 'bionic') and + (not WOVariables.wo_platform_codename == 'xenial')): + Log.info(self, "Rocket.chat is only available on Ubuntu 16.04 " + "& 18.04 LTS") + return False + else: + if not WOAptGet.is_installed(self, 'snapd'): + WOAptGet.install(self, ["snapd"]) + if WOShellExec.cmd_exec(self, "snap install rocketchat-server"): + return True