From ce2c63bd64fd3757d02384d87d27402a7f88966e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 03:07:24 +0200 Subject: [PATCH 01/22] Improve letsencrypt --- CHANGELOG.md | 6 +- wo/cli/plugins/site.py | 49 +++++++------- wo/cli/plugins/site_functions.py | 109 ++++++++----------------------- wo/core/logging.py | 20 ++++++ wo/core/sslutils.py | 66 ++++++++++++++++++- 5 files changed, 139 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48a9b5f..da20f13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Changed + +- Improve Let's Encrypt certificate issuance logging informations + #### Fixed - Fix cheat.sh install [PR #139](https://github.com/WordOps/WordOps/pull/139) - sslutils error when trying to display SSL certificate expiration -- Fix cheat.sh symbolink link check before creation +- Fix cheat.sh symbolic link check before creation ### v3.9.8.9 - 2019-09-03 diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index b4c027c..d26235e 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -163,9 +163,9 @@ class WOSiteController(CementBaseController): if os.path.islink("{0}/conf/nginx/ssl.conf" .format(wo_site_webroot)): sslexpiry = str( - SSL.getExpirationDate(self, wo_root_domain)) + SSL.getexpirationdays(self, wo_root_domain)) else: - sslexpiry = str(SSL.getExpirationDate(self, wo_domain)) + sslexpiry = str(SSL.getexpirationdays(self, wo_domain)) else: sslprovider = '' sslexpiry = '' @@ -745,9 +745,9 @@ class WOSiteCreateController(CementBaseController): # check if a wildcard cert for the root domain exist Log.debug(self, "checkWildcardExist on *.{0}" .format(wo_root_domain)) - isWildcard = checkWildcardExist(self, wo_root_domain) - Log.debug(self, "isWildcard = {0}".format(isWildcard)) - if isWildcard: + iswildcard = SSL.checkwildcardexist(self, wo_root_domain) + Log.debug(self, "iswildcard = {0}".format(iswildcard)) + if iswildcard: Log.info(self, "Using existing Wildcard SSL " "certificate from {0} to secure {1}" .format(wo_root_domain, wo_domain)) @@ -769,7 +769,7 @@ class WOSiteCreateController(CementBaseController): if pargs.hsts: setupHsts(self, wo_domain) - site_url_https(self, wo_domain) + SSL.siteurlhttps(self, wo_domain) if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") @@ -1174,7 +1174,7 @@ class WOSiteUpdateController(CementBaseController): # --letsencrypt=renew code goes here if pargs.letsencrypt == "renew" and not pargs.all: - expiry_days = SSL.getExpirationDays(self, wo_domain) + expiry_days = SSL.getexpirationdays(self, wo_domain) min_expiry_days = 45 if check_ssl: if (expiry_days <= min_expiry_days): @@ -1196,12 +1196,12 @@ class WOSiteUpdateController(CementBaseController): "check issues with `nginx -t` command") Log.info(self, "SUCCESS: Certificate was successfully renewed For" " https://{0}".format(wo_domain)) - if (SSL.getExpirationDays(self, wo_domain) > 0): + if (SSL.getexpirationdays(self, wo_domain) > 0): Log.info(self, "Your cert will expire within " + - str(SSL.getExpirationDays(self, wo_domain)) + + str(SSL.getexpirationdays(self, wo_domain)) + " days.") Log.info(self, "Expiration date: " + - str(SSL.getExpirationDate(self, wo_domain))) + str(SSL.getexpirationdate(self, wo_domain))) else: Log.warn( @@ -1236,12 +1236,12 @@ class WOSiteUpdateController(CementBaseController): self, "You have more than 45 days with the current " "certificate - refusing to run.\n") - if (SSL.getExpirationDays(self, wo_domain) > 0): + if (SSL.getexpirationdays(self, wo_domain) > 0): Log.info(self, "Your cert will expire within " + - str(SSL.getExpirationDays(self, wo_domain)) + + str(SSL.getexpirationdays(self, wo_domain)) + " days.") Log.info(self, "Expiration date: \n\n" + - str(SSL.getExpirationDate(self, wo_domain))) + str(SSL.getexpirationdate(self, wo_domain))) return 0 # else: # Log.warn(self, "Your cert already EXPIRED ! @@ -1354,11 +1354,11 @@ class WOSiteUpdateController(CementBaseController): # check if a wildcard cert for the root domain exist Log.debug(self, "checkWildcardExist on *.{0}" .format(wo_root_domain)) - isWildcard = checkWildcardExist(self, wo_root_domain) - Log.debug(self, "isWildcard = {0}".format(isWildcard)) + iswildcard = SSL.checkwildcardexist(self, wo_root_domain) + Log.debug(self, "iswildcard = {0}".format(iswildcard)) if not os.path.isfile("{0}/conf/nginx/ssl.conf.disabled"): if wo_subdomain: - if isWildcard: + if iswildcard: Log.info(self, "Using existing Wildcard SSL " "certificate from {0} to secure {1}" .format(wo_root_domain, wo_domain)) @@ -1375,9 +1375,6 @@ class WOSiteUpdateController(CementBaseController): else: setupLetsEncrypt(self, wo_domain, wo_subdomain, wo_wildcard, wo_dns, wo_acme_dns) - - httpsRedirect(self, wo_domain, True, wo_wildcard) - site_url_https(self, wo_domain) else: WOFileUtils.mvfile(self, "{0}/conf/nginx/ssl.conf.disabled" .format(wo_site_webroot), @@ -1389,8 +1386,8 @@ class WOSiteUpdateController(CementBaseController): '/etc/nginx/conf.d/force-ssl-{0}.conf' .format(wo_domain)) - httpsRedirect(self, wo_domain, True, wo_wildcard) - site_url_https(self, wo_domain) + httpsRedirect(self, wo_domain, True, wo_wildcard) + SSL.siteUrlHttps(self, wo_domain) if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " @@ -1398,20 +1395,20 @@ class WOSiteUpdateController(CementBaseController): Log.info(self, "Congratulations! Successfully " "Configured SSL for Site " " https://{0}".format(wo_domain)) - if wo_subdomain and isWildcard: - if (SSL.getExpirationDays(self, wo_root_domain) > 0): + if wo_subdomain and iswildcard: + if (SSL.getexpirationdays(self, wo_root_domain) > 0): Log.info( self, "Your cert will expire within " + - str(SSL.getExpirationDays(self, wo_root_domain)) + + str(SSL.getexpirationdays(self, wo_root_domain)) + " days.") else: Log.warn( self, "Your cert already EXPIRED ! " ".PLEASE renew soon . ") else: - if (SSL.getExpirationDays(self, wo_domain) > 0): + if (SSL.getexpirationdays(self, wo_domain) > 0): Log.info(self, "Your cert will expire within " + - str(SSL.getExpirationDays(self, wo_domain)) + + str(SSL.getexpirationdays(self, wo_domain)) + " days.") else: Log.warn( diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 99d11d3..3a0eee2 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -6,7 +6,6 @@ import json import re import string import subprocess -import csv from subprocess import CalledProcessError from wo.cli.plugins.sitedb import getSiteInfo @@ -1290,37 +1289,6 @@ def removeAcmeConf(self, domain): WOService.restart_service(self, "nginx") -def site_url_https(self, domain): - if os.path.isfile('/var/www/{0}/wp-config.php'.format(domain)): - wo_site_webroot = ('/var/www/{0}'.format(domain)) - Log.info(self, "Checking if site url already " - "use https, please wait...") - WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) - wo_siteurl = \ - WOShellExec.cmd_exec_stdout(self, - "{0} option get siteurl " - .format(WOVariables.wo_wpcli_path) + - "--allow-root --quiet") - test_url = re.split(":", wo_siteurl) - if not (test_url[0] == 'https'): - try: - WOShellExec.cmd_exec(self, "{0} option update siteurl " - "\'https://{1}\' --allow-root".format( - WOVariables.wo_wpcli_path, domain)) - WOShellExec.cmd_exec(self, "{0} option update home " - "\'https://{1}\' --allow-root".format( - WOVariables.wo_wpcli_path, domain)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("migration to https failed") - Log.info( - self, "Site address updated " - "successfully to https://{0}".format(domain)) - else: - Log.info( - self, "Site address was already using https") - - def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', dbhost=''): """ @@ -1350,7 +1318,7 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, - wo_dns=False, wo_acme_dns='dns_cf', backend=False): + wo_dns=False, wo_acme_dns='dns_cf'): if os.path.isfile("/etc/letsencrypt/" "renewal/{0}_ecc/" @@ -1371,11 +1339,12 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, self, "Validation : DNS mode with {0}".format(wo_acme_dns)) else: acme_mode = "-w /var/www/html" - validation_mode = "Subdomain Webroot challenge" + validation_mode = "Webroot challenge" Log.debug(self, "Validation : Webroot mode") if subdomain: - Log.info(self, "Issuing subdomain SSL cert with acme.sh") + Log.info(self, "Certificate type: Subdomain") Log.info(self, "Validation mode : {0}".format(validation_mode)) + Log.wait(self, "Issuing SSL certificate with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} {1} " @@ -1384,8 +1353,9 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, acme_mode, keylenght)) elif wildcard: - Log.info(self, "Issuing Wildcard SSL cert with acme.sh") + Log.info(self, "Certificate type: Wildcard") Log.info(self, "Validation mode : {0}".format(validation_mode)) + Log.wait(self, "Issuing SSL certificate with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d '*.{0}' --dns {1} " @@ -1394,8 +1364,9 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, wo_acme_dns, keylenght)) else: - Log.info(self, "Issuing domain SSL cert with acme.sh") + Log.info(self, "Certificate type: Domain + www") Log.info(self, "Validation mode : {0}".format(validation_mode)) + Log.wait(self, "Issuing SSL certificate with acme.sh") ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d www.{0} {1} " @@ -1403,7 +1374,8 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, .format(wo_domain_name, acme_mode, keylenght)) if ssl: - Log.info(self, "Deploying SSL cert with acme.sh") + Log.valide(self, "Issuing SSL certificate with acme.sh") + Log.wait(self, "Deploying SSL cert with acme.sh") Log.debug(self, "Cert deployment for domain: {0}" .format(wo_domain_name)) try: @@ -1423,21 +1395,22 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, "service nginx restart\" " .format(WOVariables.wo_ssl_live, wo_domain_name)) - Log.info( - self, "Adding /var/www/{0}/conf/nginx/ssl.conf" - .format(wo_domain_name)) + Log.valide(self, "Deploying SSL cert with acme.sh") + if os.path.isdir('/var/www/{0}/conf/nginx' + .format(wo_domain_name)): - sslconf = open("/var/www/{0}/conf/nginx/ssl.conf" - .format(wo_domain_name), - encoding='utf-8', mode='w') - sslconf.write("listen 443 ssl http2;\n" - "listen [::]:443 ssl http2;\n" - "ssl_certificate {0}/{1}/fullchain.pem;\n" - "ssl_certificate_key {0}/{1}/key.pem;\n" - "ssl_trusted_certificate {0}/{1}/ca.pem;\n" - "ssl_stapling_verify on;\n" - .format(WOVariables.wo_ssl_live, wo_domain_name)) - sslconf.close() + sslconf = open("/var/www/{0}/conf/nginx/ssl.conf" + .format(wo_domain_name), + encoding='utf-8', mode='w') + sslconf.write( + "listen 443 ssl http2;\n" + "listen [::]:443 ssl http2;\n" + "ssl_certificate {0}/{1}/fullchain.pem;\n" + "ssl_certificate_key {0}/{1}/key.pem;\n" + "ssl_trusted_certificate {0}/{1}/ca.pem;\n" + "ssl_stapling_verify on;\n" + .format(WOVariables.wo_ssl_live, wo_domain_name)) + sslconf.close() # updateSiteInfo(self, wo_domain_name, ssl=True) if not WOFileUtils.grep(self, '/var/www/22222/conf/nginx/ssl.conf', '/etc/letsencrypt'): @@ -1466,34 +1439,6 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, "you are running Let\'s Encrypt Client " "\n to allow it to verify the site automatically.") -# check if a wildcard exist to secure a new subdomain - - -def checkWildcardExist(self, wo_domain_name): - - wo_acme_exec = ("/etc/letsencrypt/acme.sh --config-home " - "'/etc/letsencrypt/config'") - # export certificates list from acme.sh - WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + - "--list --listraw > /var/lib/wo/cert.csv") - - # define new csv dialect - csv.register_dialect('acmeconf', delimiter='|') - # open file - certfile = open('/var/lib/wo/cert.csv', mode='r', encoding='utf-8') - reader = csv.reader(certfile, 'acmeconf') - wo_wildcard_domain = ("*.{0}".format(wo_domain_name)) - for row in reader: - if wo_wildcard_domain in row[2]: - isWildcard = True - break - else: - isWildcard = False - certfile.close() - - return isWildcard - - # copy wildcard certificate to a subdomain @@ -1537,9 +1482,9 @@ def renewLetsEncrypt(self, wo_domain_name): if not ssl: Log.error(self, "ERROR : Let's Encrypt certificate renewal FAILED!", False) - if (SSL.getExpirationDays(self, wo_domain_name) > 0): + if (SSL.getexpirationdays(self, wo_domain_name) > 0): Log.error(self, "Your current certificate will expire within " + - str(SSL.getExpirationDays(self, wo_domain_name)) + + str(SSL.getexpirationdays(self, wo_domain_name)) + " days.", False) else: Log.error(self, "Your current certificate already expired!", False) diff --git a/wo/core/logging.py b/wo/core/logging.py index 84a3508..19787ff 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -45,3 +45,23 @@ class Log: Logs debug messages into log file """ self.app.log.debug(Log.HEADER + msg + Log.ENDC) + + def wait(self, msg, end='\r', log=True): + """ + Logs info messages with validation step + """ + print( + Log.OKBLUE + msg + + "[" + Log.ENDC + ".." + Log.OKBLUE + "]", end=end) + if log: + self.app.log.info(Log.OKBLUE + msg + Log.ENDC) + + def valide(self, msg, end='\n', log=True): + """ + Logs info messages after validation step + """ + print( + Log.OKBLUE + msg + + "[" + Log.ENDC + "OK" + Log.OKBLUE + "]", end=end) + if log: + self.app.log.info(Log.OKBLUE + msg + Log.ENDC) diff --git a/wo/core/sslutils.py b/wo/core/sslutils.py index b164913..d380557 100644 --- a/wo/core/sslutils.py +++ b/wo/core/sslutils.py @@ -1,12 +1,16 @@ +import csv import os +import re +from wo.core.fileutils import WOFileUtils from wo.core.logging import Log from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables class SSL: - def getExpirationDays(self, domain, returnonerror=False): + def getexpirationdays(self, domain, returnonerror=False): # check if exist if not os.path.isfile('/etc/letsencrypt/live/{0}/cert.pem' .format(domain)): @@ -33,7 +37,7 @@ class SSL: # return "Certificate Already Expired ! Please Renew soon." return -1 - def getExpirationDate(self, domain): + def getexpirationdate(self, domain): # check if exist if not os.path.isfile('/etc/letsencrypt/live/{0}/cert.pem' .format(domain)): @@ -49,3 +53,61 @@ class SSL: "\"Not After\" | cut -c 25-)\" " .format(domain)) return expiration_date + + def siteurlhttps(self, domain): + wo_site_webroot = ('/var/www/{0}'.format(domain)) + WOFileUtils.chdir( + self, '{0}/htdocs/'.format(wo_site_webroot)) + if WOShellExec.cmd_exec( + self, "{0} --allow-root core is-installed" + .format(WOVariables.wo_wp_cli)): + wo_siteurl = ( + WOShellExec.cmd_exec_stdout( + self, "{0} option get siteurl " + .format(WOVariables.wo_wpcli_path) + + "--allow-root --quiet")) + test_url = re.split(":", wo_siteurl) + if not (test_url[0] == 'https'): + WOShellExec.cmd_exec( + self, "{0} option update siteurl " + "\'https://{1}\' --allow-root".format( + WOVariables.wo_wpcli_path, domain)) + WOShellExec.cmd_exec( + self, "{0} option update home " + "\'https://{1}\' --allow-root".format( + WOVariables.wo_wpcli_path, domain)) + WOShellExec.cmd_exec( + self, "{0} search-replace \'http://{0}\'" + "\'https://{0}\' --skip-columns=guid " + "--skip-tables=wp_users" + .format(domain)) + Log.info( + self, "Site address updated " + "successfully to https://{0}".format(domain)) + + # check if a wildcard exist to secure a new subdomain + + def checkwildcardexist(self, wo_domain_name): + + wo_acme_exec = ("/etc/letsencrypt/acme.sh --config-home " + "'/etc/letsencrypt/config'") + # export certificates list from acme.sh + WOShellExec.cmd_exec( + self, "{0} ".format(wo_acme_exec) + + "--list --listraw > /var/lib/wo/cert.csv") + + # define new csv dialect + csv.register_dialect('acmeconf', delimiter='|') + # open file + certfile = open('/var/lib/wo/cert.csv', mode='r', encoding='utf-8') + reader = csv.reader(certfile, 'acmeconf') + wo_wildcard_domain = ("*.{0}".format(wo_domain_name)) + for row in reader: + if wo_wildcard_domain in row[2]: + iswildcard = True + break + else: + iswildcard = False + certfile.close() + + return iswildcard From 99a2206a76c50d9c7d70f23be3d66ab3302ea95a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 04:21:07 +0200 Subject: [PATCH 02/22] Improve logging and fix typo --- wo/cli/plugins/site.py | 2 +- wo/cli/plugins/stack.py | 14 ++++++++++---- wo/cli/plugins/stack_upgrade.py | 15 ++++++++++----- wo/core/logging.py | 5 +++-- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index d26235e..dbf3f93 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1387,7 +1387,7 @@ class WOSiteUpdateController(CementBaseController): .format(wo_domain)) httpsRedirect(self, wo_domain, True, wo_wildcard) - SSL.siteUrlHttps(self, wo_domain) + SSL.siteurlhttps(self, wo_domain) if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 02585fb..b393986 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -484,11 +484,15 @@ class WOStackController(CementBaseController): # wo_mem = int(memsplit[0]) # if (wo_mem < 4000000): # WOSwap.add(self) - Log.info(self, "Updating apt-cache, please wait...") + Log.wait(self, "Updating apt-cache ") WOAptGet.update(self) - Log.info(self, "Installing packages, please wait...") + Log.valide(self, "Updating apt-cache ") + Log.info(self, "Installing APT packages ") WOAptGet.install(self, apt_packages) + Log.valide(self, "Installing APT packages ") + Log.wait(self, "Configuring APT packages ") post_pref(self, apt_packages, empty_packages) + Log.valide(self, "Configuring APT packages ") if (packages): Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) @@ -714,9 +718,10 @@ class WOStackController(CementBaseController): if (apt_packages): Log.debug(self, "Removing apt_packages") - Log.info(self, "Removing apt packages, please wait...") + Log.wait(self, "Removing APT packages ") WOAptGet.remove(self, apt_packages) WOAptGet.auto_remove(self) + Log.valide(self, "Removing APT packages ") Log.info(self, "Successfully removed packages") @@ -934,9 +939,10 @@ class WOStackController(CementBaseController): "uninstaller.sh -y -f") if (apt_packages): - Log.info(self, "Purging apt packages, please wait...") + Log.wait(self, "Purging APT Packages ") WOAptGet.remove(self, apt_packages, purge=True) WOAptGet.auto_remove(self) + Log.valide(self, "Purging APT Packages ") if (packages): Log.info(self, "Purging packages, please wait...") diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 2295a47..0857160 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -87,8 +87,6 @@ class WOStackUpgradeController(CementBaseController): if pargs.web: if WOAptGet.is_installed(self, 'nginx-custom'): pargs.nginx = True - else: - Log.info(self, "Nginx is not already installed") pargs.php = True pargs.mysql = True pargs.wpcli = True @@ -204,8 +202,12 @@ class WOStackUpgradeController(CementBaseController): '/etc/apt/preferences.d/nginx-block'): WOAptGet.install(self, nginx_packages) + Log.wait(self, "Upgrading APT Packages ") WOAptGet.install(self, apt_packages) + Log.valide(self, "Upgrading APT Packages ") + Log.wait(self, "Configuring APT Packages ") post_pref(self, nginx_packages, [], True) + Log.valide(self, "Configuring APT Packages ") post_pref(self, apt_packages, [], True) # Post Actions after package updates @@ -226,7 +228,7 @@ class WOStackUpgradeController(CementBaseController): WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) if pargs.netdata: - Log.info(self, "Upgrading Netdata, please wait...") + Log.wait(self, "Upgrading Netdata ") if os.path.isdir('/opt/netdata'): WOShellExec.cmd_exec( self, "bash /opt/netdata/usr/" @@ -237,6 +239,7 @@ class WOStackUpgradeController(CementBaseController): self, "bash /usr/" "libexec/netdata/netdata-" "updater.sh") + Log.valide(self, "Upgrading Netdata ") if pargs.dashboard: Log.debug(self, "Extracting wo-dashboard.tar.gz " @@ -252,16 +255,17 @@ class WOStackUpgradeController(CementBaseController): WOVariables.wo_php_user, recursive=True) if pargs.composer: - Log.info(self, "Upgrading Composer, please wait...") + Log.wait(self, "Upgrading Composer ") WOShellExec.cmd_exec(self, "php -q /var/lib/wo" "/tmp/composer-install " "--install-dir=/var/lib/wo/tmp/") shutil.copyfile('/var/lib/wo/tmp/composer.phar', '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) + Log.valide(self, "Upgrading Composer ") if pargs.phpmyadmin: - Log.info(self, "Upgrading phpMyAdmin, please wait...") + Log.wait(self, "Upgrading phpMyAdmin ") WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') shutil.copyfile(('{0}22222/htdocs/db/pma' @@ -282,6 +286,7 @@ class WOStackUpgradeController(CementBaseController): .format(WOVariables.wo_webroot), WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) + Log.valide(self, "Upgrading phpMyAdmin ") Log.info(self, "Successfully updated packages") else: diff --git a/wo/core/logging.py b/wo/core/logging.py index 19787ff..7dbaca2 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -52,7 +52,7 @@ class Log: """ print( Log.OKBLUE + msg + - "[" + Log.ENDC + ".." + Log.OKBLUE + "]", end=end) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]", end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -62,6 +62,7 @@ class Log: """ print( Log.OKBLUE + msg + - "[" + Log.ENDC + "OK" + Log.OKBLUE + "]", end=end) + " [" + Log.ENDC + Log.OKGREEN + "OK" + + Log.ENDC + Log.OKBLUE + "]", end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) From 5323c2864df3877aeafd4ace04c26046fa402c53 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 04:26:09 +0200 Subject: [PATCH 03/22] Improve identation --- wo/cli/plugins/site.py | 184 +++++++++++++++++++++-------------------- 1 file changed, 96 insertions(+), 88 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index dbf3f93..5d420cc 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1004,21 +1004,23 @@ class WOSiteUpdateController(CementBaseController): data['currcachetype'] = oldcachetype if stype == 'php': - data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, wprocket=False, wpce=False, - multisite=False, wpsubdir=False, webroot=wo_site_webroot, - currsitetype=oldsitetype, currcachetype=oldcachetype) + data = dict( + site_name=wo_domain, www_domain=wo_www_domain, + static=False, basic=True, wp=False, wpfc=False, + wpsc=False, wpredis=False, wprocket=False, wpce=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot, + currsitetype=oldsitetype, currcachetype=oldcachetype) elif stype in ['mysql', 'wp', 'wpsubdir', 'wpsubdomain']: - data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, wprocket=False, wpce=False, - multisite=False, wpsubdir=False, webroot=wo_site_webroot, - wo_db_name='', wo_db_user='', wo_db_pass='', - wo_db_host='', - currsitetype=oldsitetype, currcachetype=oldcachetype) + data = dict( + site_name=wo_domain, www_domain=wo_www_domain, + static=False, basic=True, wp=False, wpfc=False, + wpsc=False, wpredis=False, wprocket=False, wpce=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot, + wo_db_name='', wo_db_user='', wo_db_pass='', + wo_db_host='', + currsitetype=oldsitetype, currcachetype=oldcachetype) if stype in ['wp', 'wpsubdir', 'wpsubdomain']: data['wp'] = True @@ -1031,10 +1033,11 @@ class WOSiteUpdateController(CementBaseController): if pargs.php73: if not data: - data = dict(site_name=wo_domain, www_domain=wo_www_domain, - currsitetype=oldsitetype, - currcachetype=oldcachetype, - webroot=wo_site_webroot) + data = dict( + site_name=wo_domain, www_domain=wo_www_domain, + currsitetype=oldsitetype, + currcachetype=oldcachetype, + webroot=wo_site_webroot) stype = oldsitetype cache = oldcachetype if oldsitetype == 'html' or oldsitetype == 'proxy': @@ -1572,26 +1575,27 @@ class WOSiteUpdateController(CementBaseController): (data['wpfc'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpfc'])): try: - plugin_data_object = {"log_level": "INFO", - "log_filesize": 5, - "enable_purge": 1, - "enable_map": "0", - "enable_log": 0, - "enable_stamp": 1, - "purge_homepage_on_new": 1, - "purge_homepage_on_edit": 1, - "purge_homepage_on_del": 1, - "purge_archive_on_new": 1, - "purge_archive_on_edit": 0, - "purge_archive_on_del": 0, - "purge_archive_on_new_comment": 0, - "purge_archive_on_deleted_comment": 0, - "purge_page_on_mod": 1, - "purge_page_on_new_comment": 1, - "purge_page_on_deleted_comment": 1, - "cache_method": "enable_fastcgi", - "purge_method": "get_request", - "redis_hostname": "127.0.0.1", + plugin_data_object = { + "log_level": "INFO", + "log_filesize": 5, + "enable_purge": 1, + "enable_map": "0", + "enable_log": 0, + "enable_stamp": 1, + "purge_homepage_on_new": 1, + "purge_homepage_on_edit": 1, + "purge_homepage_on_del": 1, + "purge_archive_on_new": 1, + "purge_archive_on_edit": 0, + "purge_archive_on_del": 0, + "purge_archive_on_new_comment": 0, + "purge_archive_on_deleted_comment": 0, + "purge_page_on_mod": 1, + "purge_page_on_new_comment": 1, + "purge_page_on_deleted_comment": 1, + "cache_method": "enable_fastcgi", + "purge_method": "get_request", + "redis_hostname": "127.0.0.1", "redis_port": "6379", "redis_prefix": "nginx-cache:"} plugin_data = json.dumps(plugin_data_object) @@ -1613,26 +1617,27 @@ class WOSiteUpdateController(CementBaseController): data['multisite'] and data['wpredis'])): try: - plugin_data_object = {"log_level": "INFO", - "log_filesize": 5, - "enable_purge": 1, - "enable_map": "0", - "enable_log": 0, - "enable_stamp": 1, - "purge_homepage_on_new": 1, - "purge_homepage_on_edit": 1, - "purge_homepage_on_del": 1, - "purge_archive_on_new": 1, - "purge_archive_on_edit": 0, - "purge_archive_on_del": 0, - "purge_archive_on_new_comment": 0, - "purge_archive_on_deleted_comment": 0, - "purge_page_on_mod": 1, - "purge_page_on_new_comment": 1, - "purge_page_on_deleted_comment": 1, - "cache_method": "enable_redis", - "purge_method": "get_request", - "redis_hostname": "127.0.0.1", + plugin_data_object = { + "log_level": "INFO", + "log_filesize": 5, + "enable_purge": 1, + "enable_map": "0", + "enable_log": 0, + "enable_stamp": 1, + "purge_homepage_on_new": 1, + "purge_homepage_on_edit": 1, + "purge_homepage_on_del": 1, + "purge_archive_on_new": 1, + "purge_archive_on_edit": 0, + "purge_archive_on_del": 0, + "purge_archive_on_new_comment": 0, + "purge_archive_on_deleted_comment": 0, + "purge_page_on_mod": 1, + "purge_page_on_new_comment": 1, + "purge_page_on_deleted_comment": 1, + "cache_method": "enable_redis", + "purge_method": "get_request", + "redis_hostname": "127.0.0.1", "redis_port": "6379", "redis_prefix": "nginx-cache:"} plugin_data = json.dumps(plugin_data_object) @@ -1650,26 +1655,27 @@ class WOSiteUpdateController(CementBaseController): else: try: # disable nginx-helper - plugin_data_object = {"log_level": "INFO", - "log_filesize": 5, - "enable_purge": 0, - "enable_map": 0, - "enable_log": 0, - "enable_stamp": 0, - "purge_homepage_on_new": 1, - "purge_homepage_on_edit": 1, - "purge_homepage_on_del": 1, - "purge_archive_on_new": 1, - "purge_archive_on_edit": 0, - "purge_archive_on_del": 0, - "purge_archive_on_new_comment": 0, - "purge_archive_on_deleted_comment": 0, - "purge_page_on_mod": 1, - "purge_page_on_new_comment": 1, - "purge_page_on_deleted_comment": 1, - "cache_method": "enable_redis", - "purge_method": "get_request", - "redis_hostname": "127.0.0.1", + plugin_data_object = { + "log_level": "INFO", + "log_filesize": 5, + "enable_purge": 0, + "enable_map": 0, + "enable_log": 0, + "enable_stamp": 0, + "purge_homepage_on_new": 1, + "purge_homepage_on_edit": 1, + "purge_homepage_on_del": 1, + "purge_archive_on_new": 1, + "purge_archive_on_edit": 0, + "purge_archive_on_del": 0, + "purge_archive_on_new_comment": 0, + "purge_archive_on_deleted_comment": 0, + "purge_page_on_mod": 1, + "purge_page_on_new_comment": 1, + "purge_page_on_deleted_comment": 1, + "cache_method": "enable_redis", + "purge_method": "get_request", + "redis_hostname": "127.0.0.1", "redis_port": "6379", "redis_prefix": "nginx-cache:"} plugin_data = json.dumps(plugin_data_object) @@ -1685,24 +1691,26 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - if ((oldcachetype in ['wpsc', 'basic', 'wpfc', 'wprocket', 'wpredis'] and + if ((oldcachetype in ['wpsc', 'basic', + 'wpfc', 'wprocket', 'wpredis'] and (data['wpce'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpce'])): try: installwp_plugin(self, 'cache-enabler', data) # setup cache-enabler - plugin_data_object = {"expires": 24, - "new_post": 1, - "new_comment": 0, - "webp": 0, - "clear_on_upgrade": 1, - "compress": 0, - "excl_ids": "", - "excl_regexp": "", - "excl_cookies": "", - "incl_attributes": "", - "minify_html": 1} + plugin_data_object = { + "expires": 24, + "new_post": 1, + "new_comment": 0, + "webp": 0, + "clear_on_upgrade": 1, + "compress": 0, + "excl_ids": "", + "excl_regexp": "", + "excl_cookies": "", + "incl_attributes": "", + "minify_html": 1} plugin_data = json.dumps(plugin_data_object) setupwp_plugin(self, 'cache-enabler', 'cache-enabler', plugin_data, data) From f814bbea5d0da24952c05855609b3d4643e13dbd Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 16:13:46 +0200 Subject: [PATCH 04/22] Fix subdomain --- wo/core/domainvalidate.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index 5d1d65d..31b197c 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -29,7 +29,7 @@ def GetDomainlevel(domain): """ This function returns the domain type : domain, subdomain, """ - domain_name = domain.lower().split('.') + domain_name = domain.lower().strip().split('.') if domain_name[0] == 'www': domain_name = domain_name[1:] domain_type = '' @@ -39,16 +39,17 @@ def GetDomainlevel(domain): "/var/lib/wo/public_suffix_list.dat", encoding='utf-8', ) # Read all the lines into a list. for domain_suffix in Suffix_file: - if (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): - domain_type = 'domain' - root_domain = ('.'.join(domain_name[0:])) - break - elif (str(domain_suffix).strip()) == ('.'.join(domain_name[2:])): + if (str(domain_suffix).strip()) == ('.'.join(domain_name[2:])): domain_type = 'subdomain' root_domain = ('.'.join(domain_name[1:])) break + elif (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): + domain_type = 'domain' + root_domain = ('.'.join(domain_name)) + break else: - domain_type = 'other' + domain_type = 'domain' + root_domain = ('.'.join(domain_name)) Suffix_file.close() return (domain_type, root_domain) From 5cb2c394b1bb5f476f0c93cb57c861d0f2749272 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 16:38:55 +0200 Subject: [PATCH 05/22] Fix domain is_subdomain --- wo/core/domainvalidate.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index 31b197c..667fc21 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -1,6 +1,6 @@ """WordOps domain validation module.""" -from urllib.parse import urlparse import os +from urllib.parse import urlparse def ValidateDomain(url): @@ -35,21 +35,17 @@ def GetDomainlevel(domain): domain_type = '' if os.path.isfile("/var/lib/wo/public_suffix_list.dat"): # Read mode opens a file for reading only. - Suffix_file = open( + suffix_file = open( "/var/lib/wo/public_suffix_list.dat", encoding='utf-8', ) # Read all the lines into a list. - for domain_suffix in Suffix_file: - if (str(domain_suffix).strip()) == ('.'.join(domain_name[2:])): - domain_type = 'subdomain' - root_domain = ('.'.join(domain_name[1:])) - break - elif (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): + for domain_suffix in suffix_file: + if (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): domain_type = 'domain' - root_domain = ('.'.join(domain_name)) + root_domain = ('.'.join(domain_name[0:])) break else: - domain_type = 'domain' - root_domain = ('.'.join(domain_name)) - Suffix_file.close() + domain_type = 'subdomain' + root_domain = ('.'.join(domain_name[1:])) + suffix_file.close() return (domain_type, root_domain) From d3df2b641263df6b0b36262144f63669398878a2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 16:55:58 +0200 Subject: [PATCH 06/22] Fix mariadb-backup install --- CHANGELOG.md | 3 +++ README.md | 2 +- install | 2 +- setup.py | 2 +- wo/cli/plugins/site_functions.py | 2 +- wo/cli/plugins/stack.py | 30 ++++++++++++++++++++++++++---- wo/cli/plugins/stack_pref.py | 13 +++++++------ wo/cli/plugins/stack_upgrade.py | 4 ++-- wo/core/variables.py | 16 ++++------------ 9 files changed, 46 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da20f13..68e26ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +### v3.9.8.10 - 2019-09-04 + #### Changed - Improve Let's Encrypt certificate issuance logging informations @@ -17,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix cheat.sh install [PR #139](https://github.com/WordOps/WordOps/pull/139) - sslutils error when trying to display SSL certificate expiration - Fix cheat.sh symbolic link check before creation +- subdomain detection with complex suffixes like com.br ### v3.9.8.9 - 2019-09-03 diff --git a/README.md b/README.md index 48d4992..d0dfe1a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@
WordOps install codacy Badge Twitter -Badge Slack +Badge Slack

diff --git a/install b/install index e185267..97d8544 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.9.8.9 - 2019-09-03 +# Version 3.9.8.10 - 2019-09-04 # ------------------------------------------------------------------------- # CONTENTS diff --git a/setup.py b/setup.py index 4b7b445..7db7c83 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ if not os.path.isfile('/root/.gitconfig'): shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') setup(name='wo', - version='3.9.8.9', + version='3.9.8.10', description=long_description, long_description=long_description, classifiers=[], diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 3a0eee2..1181ef0 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -876,7 +876,7 @@ def site_package_check(self, stype): if self.app.pargs.wpredis: Log.debug(self, "Setting apt_packages variable for redis") if not WOAptGet.is_installed(self, 'redis-server'): - apt_packages = apt_packages + WOVariables.wo_redis + apt_packages = apt_packages + ["redis-server"] if self.app.pargs.php73: Log.debug(self, "Setting apt_packages variable for PHP 7.3") diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index b393986..b09b4f3 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -115,6 +115,7 @@ class WOStackController(CementBaseController): empty_packages = [] wo_webroot = "/var/www/" pargs = self.app.pargs + try: # Default action for stack installation if ((not pargs.web) and (not pargs.admin) and @@ -190,7 +191,7 @@ class WOStackController(CementBaseController): if pargs.redis: pargs.php = True if not WOAptGet.is_installed(self, 'redis-server'): - apt_packages = apt_packages + WOVariables.wo_redis + apt_packages = apt_packages + ["redis-server"] else: Log.info(self, "Redis already installed") @@ -224,10 +225,31 @@ class WOStackController(CementBaseController): # MariaDB 10.3 if pargs.mysql: + if not WOVariables.wo_distro == 'raspbian': + if (not WOVariables.wo_platform_codename == 'jessie'): + if ((not WOFileUtils.grep( + self, "/etc/apt/sources.list.d/wo-repo.list", + "10.1")) and + (not WOFileUtils.grep( + self, "/etc/apt/sources.list.d/wo-repo.list", + "10.1"))): + + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb", "mariadb-backup"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysql.connector"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb"] + pargs.mysqltuner = True Log.debug(self, "Setting apt_packages variable for MySQL") if not WOShellExec.cmd_exec(self, "mysqladmin ping"): - apt_packages = apt_packages + WOVariables.wo_mysql + apt_packages = apt_packages + wo_mysql else: Log.debug(self, "MySQL already installed and alive") Log.info(self, "MySQL already installed and alive") @@ -585,7 +607,7 @@ class WOStackController(CementBaseController): # REDIS if pargs.redis: Log.debug(self, "Remove apt_packages variable of Redis") - apt_packages = apt_packages + WOVariables.wo_redis + apt_packages = apt_packages + ["redis-server"] # MariaDB if pargs.mysql: @@ -804,7 +826,7 @@ class WOStackController(CementBaseController): # REDIS if pargs.redis: Log.debug(self, "Remove apt_packages variable of Redis") - apt_packages = apt_packages + WOVariables.wo_redis + apt_packages = apt_packages + ["redis-server"] # MariaDB if pargs.mysql: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index b9ce87e..5576598 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -29,8 +29,9 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - if (set(WOVariables.wo_mysql).issubset(set(apt_packages)) or - set(WOVariables.wo_mysql_client).issubset(set(apt_packages))): + if (set(["mariadb-server"]).issubset(set(apt_packages)) or + set(["mariadb-client"]).issubset(set(apt_packages)) or + set(["mariadb-backup"]).issubset((set(apt_packages)))): # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") @@ -47,7 +48,7 @@ def pre_pref(self, apt_packages): keyserver="keyserver.ubuntu.com") WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver="keyserver.ubuntu.com") - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if set(["mariadb-server"]).issubset(set(apt_packages)): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) @@ -139,7 +140,7 @@ def pre_pref(self, apt_packages): Log.debug(self, 'Adding deb.sury GPG key') WORepo.add_key(self, WOVariables.wo_php_key) # add redis repository - if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if set(['redis-server']).issubset(set(apt_packages)): Log.info(self, "Adding repository for Redis, please wait...") if WOVariables.wo_distro == 'ubuntu': Log.debug(self, 'Adding ppa for redis') @@ -189,7 +190,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): '\t$request_filename;\n') try: data = dict(php="9000", debug="9001", - php7="9070", debug7="9170") + php7="9070", debug7="9170") WOTemplate.render( self, '{0}/upstream.conf'.format(ngxcnf), 'upstream.mustache', data, overwrite=True) @@ -1106,7 +1107,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOService.reload_service(self, 'proftpd') # Redis configuration - if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if set(['redus-server']).issubset(set(apt_packages)): if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" "upstream.conf", diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0857160..ebe8284 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -119,13 +119,13 @@ class WOStackUpgradeController(CementBaseController): if pargs.mysql: if WOAptGet.is_installed(self, 'mariadb-server'): - apt_packages = apt_packages + WOVariables.wo_mysql + apt_packages = apt_packages + ['mariadb-server'] else: Log.info(self, "MariaDB is not installed") if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): - apt_packages = apt_packages + WOVariables.wo_redis + apt_packages = apt_packages + ['redis-server'] else: Log.info(self, "Redis is not installed") diff --git a/wo/core/variables.py b/wo/core/variables.py index ba33486..96563b8 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -10,7 +10,7 @@ class WOVariables(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.8.9" + wo_version = "3.9.8.10" # WordOps packages versions wo_wp_cli = "2.2.0" wo_adminer = "4.7.2" @@ -144,21 +144,13 @@ class WOVariables(): "10.3/debian {codename} main" .format(codename=wo_platform_codename)) - if wo_distro == 'raspbian': - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] - elif wo_platform_codename == 'jessie': - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb", "mariadb-backup"] - if wo_platform_codename == 'jessie': wo_mysql_client = ["mariadb-client", "python3-mysqldb"] else: wo_mysql_client = ["mariadb-client", "python3-mysql.connector"] + + wo_fail2ban = ["fail2ban"] wo_clamav = ["clamav", "clamav-freshclam"] @@ -170,7 +162,7 @@ class WOVariables(): wo_redis_repo = ("deb https://packages.sury.org/php/ {codename} all" .format(codename=wo_platform_codename)) - wo_redis = ['redis-server', 'php-redis'] + wo_redis = ['redis-server'] # Repo path wo_repo_file = "wo-repo.list" From aaa073ffd566bf2f0ea5fd8d4ccde7c73d07d292 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 16:57:25 +0200 Subject: [PATCH 07/22] Fix mariadb post_pref --- wo/cli/plugins/stack_pref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 5576598..2227765 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -877,7 +877,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOService.restart_service(self, 'php7.3-fpm') # create mysql config if it doesn't exist - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if set(["mariadb-server"]).issubset(set(apt_packages)): if not os.path.isfile("/etc/mysql/my.cnf"): config = ("[mysqld]\nwait_timeout = 30\n" "interactive_timeout=60\nperformance_schema = 0" From 2123bcfbbc8763955de210e05e21e2c33d4abe29 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 16:58:20 +0200 Subject: [PATCH 08/22] [skip travis] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68e26ed..8c302c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - sslutils error when trying to display SSL certificate expiration - Fix cheat.sh symbolic link check before creation - subdomain detection with complex suffixes like com.br +- Fix mariadb install/upgrade when running mariadb-10.1 +- Fix mariadb install/upgrade on raspbian and debian 8 ### v3.9.8.9 - 2019-09-03 From e5d685e054c3852e1e452735eb027f7358ed703a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 17:04:25 +0200 Subject: [PATCH 09/22] Make stack install cleaner --- wo/cli/plugins/site_functions.py | 12 +++++++++++- wo/cli/plugins/stack.py | 27 ++++++++------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 1181ef0..7afdc8a 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -863,7 +863,17 @@ def site_package_check(self, stype): if stype in ['mysql', 'wp', 'wpsubdir', 'wpsubdomain']: Log.debug(self, "Setting apt_packages variable for MySQL") if not WOShellExec.cmd_exec(self, "/usr/bin/mysqladmin ping"): - apt_packages = apt_packages + WOVariables.wo_mysql + if not WOVariables.wo_distro == 'raspbian': + if (not WOVariables.wo_platform_codename == 'jessie'): + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb", "mariadb-backup"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysql.connector"] + else: + wo_mysql = ["mariadb-server", "percona-toolkit", + "python3-mysqldb"] + apt_packages = apt_packages + wo_mysql if stype in ['wp', 'wpsubdir', 'wpsubdomain']: Log.debug(self, "Setting packages variable for WP-CLI") diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index b09b4f3..46e5555 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -225,31 +225,20 @@ class WOStackController(CementBaseController): # MariaDB 10.3 if pargs.mysql: - if not WOVariables.wo_distro == 'raspbian': - if (not WOVariables.wo_platform_codename == 'jessie'): - if ((not WOFileUtils.grep( - self, "/etc/apt/sources.list.d/wo-repo.list", - "10.1")) and - (not WOFileUtils.grep( - self, "/etc/apt/sources.list.d/wo-repo.list", - "10.1"))): - + pargs.mysqltuner = True + Log.debug(self, "Setting apt_packages variable for MySQL") + if not WOShellExec.cmd_exec(self, "mysqladmin ping"): + if not WOVariables.wo_distro == 'raspbian': + if (not WOVariables.wo_platform_codename == 'jessie'): wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb", "mariadb-backup"] else: wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] + "python3-mysql.connector"] else: wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] - - pargs.mysqltuner = True - Log.debug(self, "Setting apt_packages variable for MySQL") - if not WOShellExec.cmd_exec(self, "mysqladmin ping"): - apt_packages = apt_packages + wo_mysql + "python3-mysqldb"] + apt_packages = apt_packages + wo_mysql else: Log.debug(self, "MySQL already installed and alive") Log.info(self, "MySQL already installed and alive") From 10e311fb407df4cded43e03488e46e45bf1e075a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 17:56:57 +0200 Subject: [PATCH 10/22] Fix mysql install --- wo/cli/plugins/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 46e5555..8daf1c4 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -238,7 +238,7 @@ class WOStackController(CementBaseController): else: wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb"] - apt_packages = apt_packages + wo_mysql + apt_packages = apt_packages + wo_mysql else: Log.debug(self, "MySQL already installed and alive") Log.info(self, "MySQL already installed and alive") From ff506142ce1e664a332bd3cbc8da4aab410faea4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 18:33:13 +0200 Subject: [PATCH 11/22] MariaDB optimization based on template --- wo/cli/plugins/stack_pref.py | 104 ++++--------------- wo/cli/templates/my.mustache | 195 +++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 85 deletions(-) create mode 100644 wo/cli/templates/my.mustache diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2227765..a0f3def 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -886,101 +886,35 @@ def post_pref(self, apt_packages, packages, upgrade=False): encoding='utf-8', mode='w') config_file.write(config) config_file.close() - elif (not WOFileUtils.grep(self, "/etc/mysql/my.cnf", "WordOps")): + else: Log.info(self, "Tuning MariaDB configuration") - with open("/etc/mysql/my.cnf", - "a") as mysql_file: - mysql_file.write("\n# WordOps v3.9.8\n") + WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", + "/etc/mysql/my.cnf.default-pkg") wo_ram = psutil.virtual_memory().total / (1024 * 1024) # set InnoDB variable depending on the RAM available wo_ram_innodb = int(wo_ram*0.3) wo_ram_log_buffer = int(wo_ram_innodb*0.25) wo_ram_log_size = int(wo_ram_log_buffer*0.5) + if (wo_ram_innodb > 2000) and (wo_ram_innodb < 64000): + wo_innodb_instance = int(wo_ram_innodb/1000) + tmp_table_size = int(128) + elif (wo_ram_innodb < 2000): + wo_innodb_instance = int(1) + tmp_table_size = int(32) + elif (wo_ram_innodb > 64000): + wo_innodb_instance = int(64) + tmp_table_size = int(256) + data = dict( + tmp_table_size=tmp_table_size, inno_log=wo_ram_log_size, + inno_buffer=wo_ram_innodb, + inno_log_buffer=wo_ram_log_buffer, + innodb_instances=wo_innodb_instance) + WOTemplate.render( + self, '/etc/mysql/my.cnf', 'my.mustache', data) # replacing default values Log.debug(self, "Tuning MySQL configuration") # set innodb_buffer_pool_instances depending # on the amount of RAM - if (wo_ram_innodb > 1000) and (wo_ram_innodb < 64000): - wo_innodb_instance = int( - wo_ram_innodb/1000) - elif (wo_ram_innodb < 1000): - wo_innodb_instance = int(1) - elif (wo_ram_innodb > 64000): - wo_innodb_instance = int(64) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_buffer_pool_size = 256M", - "innodb_buffer_pool_size " - "= {0}M\n" - "innodb_buffer_pool_instances " - "= {1}\n" - .format(wo_ram_innodb, - wo_innodb_instance)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_log_buffer_size = 8M", - "innodb_log_buffer_size = {0}M" - .format(wo_ram_log_buffer)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "#innodb_log_file_size = 50M", - "innodb_log_file_size = {0}M" - .format(wo_ram_log_size)) - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "wait_timeout " - "= 600", - "wait_timeout " - "= 120\n" - "skip-name-resolve = 1\n") - # disabling mariadb binlog - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "log_bin " - "= /var/log/mysql/" - "mariadb-bin", - "#log_bin " - " = /var/log/" - "mysql/mariadb-bin") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - 'log_bin_index ' - "= /var/log/mysql/" - "mariadb-bin.index", - "#log_bin_index " - "= /var/log/mysql/" - "mariadb-bin.index") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "expire_logs_days = 10", - "#expire_logs_days " - "= 10") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "max_binlog_size " - "= 100M", - "#max_binlog_size " - "= 100M") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_open_files =" - " 400", - "innodb_open_files =" - " 16000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_io_capacity =" - " 400", - "innodb_io_capacity =" - " 16000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "query_cache_size = 64M", - "query_cache_size = 0") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "#query_cache_type = DEMAND", - "query_cache_type = 0") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "#open-files-limit = 2000", - "open-files-limit = 10000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "table_open_cache = 400", - "table_open_cache = 16000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "max_allowed_packet = 16M", - "max_allowed_packet = 64M\n") - WOService.stop_service(self, 'mysql') WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', '/var/lib/mysql/ib_logfile0.bak') diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache new file mode 100644 index 0000000..5fec5ca --- /dev/null +++ b/wo/cli/templates/my.mustache @@ -0,0 +1,195 @@ +# MariaDB database server configuration file. +# +# You can copy this file to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# +user = mysql +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +lc_messages_dir = /usr/share/mysql +lc_messages = en_US +skip-external-locking +skip-name-resolve = 1 +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +bind-address = ::ffff:127.0.0.1 +# +# * Fine Tuning +# +max_connections = 100 +connect_timeout = 5 +wait_timeout = 60 +max_allowed_packet = 64M +thread_cache_size = 128 +sort_buffer_size = 4M +bulk_insert_buffer_size = 16M +tmp_table_size = {{tmp_table_size}}M +max_heap_table_size = {{tmp_table_size}}M +# +# * MyISAM +# +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched. On error, make copy and try a repair. +myisam_recover_options = BACKUP +key_buffer_size = 128M +open-files-limit = 500000 +table_open_cache = 16000 +myisam_sort_buffer_size = 512M +concurrent_insert = 2 +read_buffer_size = 2M +read_rnd_buffer_size = 1M +# +# * Query Cache Configuration +# +# Cache only tiny result sets, so we can fit more in the query cache. +query_cache_limit = 128K +query_cache_size = 0 +# for more write intensive setups, set to DEMAND or OFF +query_cache_type = 0 +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 +# +# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. +# +# we do want to know about network errors and such +log_warnings = 2 +# +# Enable the slow query log to see queries with especially long duration +slow_query_log = 1 +slow_query_log_file = /var/log/mysql/mariadb-slow.log +long_query_time = 10 +#log_slow_rate_limit = 1000 +log_slow_verbosity = query_plan + +#log-queries-not-using-indexes +#log_slow_admin_statements +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +#server-id = 1 +#report_host = master1 +#auto_increment_increment = 2 +#auto_increment_offset = 1 +#log_bin = /var/log/mysql/mariadb-bin +#log_bin_index = /var/log/mysql/mariadb-bin.index +# not fab for performance, but safer +#sync_binlog = 1 +#expire_logs_days = 10 +#max_binlog_size = 100M +transaction_isolation = READ-COMMITTED +binlog_format = ROW + +# slaves +#relay_log = /var/log/mysql/relay-bin +#relay_log_index = /var/log/mysql/relay-bin.index +#relay_log_info_file = /var/log/mysql/relay-bin.info +#log_slave_updates +#read_only +# +# If applications support it, this stricter sql_mode prevents some +# mistakes like inserting invalid dates etc. +#sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL +# +# * InnoDB +# +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +default_storage_engine = InnoDB +# you can't just change log file size, requires special procedure +innodb_log_file_size = {{inno_log}}M +innodb_buffer_pool_size = {{inno_buffer}}M +innodb_log_buffer_size = {{inno_log_buffer}}M +innodb_file_per_table = 1 +innodb_open_files = 500000 +innodb_io_capacity = 500000 +innodb_flush_method = O_DIRECT +innodb_buffer_pool_instances = {{innodb_instances}} +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + +# +# * Galera-related settings +# +[galera] +# Mandatory settings +#wsrep_on=ON +#wsrep_provider= +#wsrep_cluster_address= +#binlog_format=row +#default_storage_engine=InnoDB +#innodb_autoinc_lock_mode=2 +# +# Allow server to accept connections on all interfaces. +# +#bind-address=0.0.0.0 +# +# Optional setting +#wsrep_slave_threads=1 +#innodb_flush_log_at_trx_commit=0 + +[mysqldump] +quick +quote-names +max_allowed_packet = 64M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completion + +[isamchk] +key_buffer = 16M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!include /etc/mysql/mariadb.cnf +!includedir /etc/mysql/conf.d/ From 56895cdc5506dd961e3896576a161fd345ac72f0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 18:38:28 +0200 Subject: [PATCH 12/22] Fix mariadb removal --- wo/cli/plugins/stack.py | 8 ++++---- wo/cli/plugins/stack_migrate.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 8daf1c4..4da3b90 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -601,7 +601,7 @@ class WOStackController(CementBaseController): # MariaDB if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") - apt_packages = apt_packages + WOVariables.wo_mysql + apt_packages = apt_packages + ['mariadb-server'] # mysqlclient if pargs.mysqlclient: @@ -706,7 +706,7 @@ class WOStackController(CementBaseController): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') - if (set(WOVariables.wo_mysql).issubset(set(apt_packages))): + if (set(["mariadb-server"]).issubset(set(apt_packages))): WOMysql.backupAll(self) WOService.stop_service(self, 'mysql') @@ -820,7 +820,7 @@ class WOStackController(CementBaseController): # MariaDB if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") - apt_packages = apt_packages + WOVariables.wo_mysql + apt_packages = apt_packages + ['mariadb-server'] # mysqlclient if pargs.mysqlclient: @@ -932,7 +932,7 @@ class WOStackController(CementBaseController): if (set(["fail2ban"]).issubset(set(apt_packages))): WOService.stop_service(self, 'fail2ban') - if (set(WOVariables.wo_mysql).issubset(set(apt_packages))): + if (set(["mariadb-server"]).issubset(set(apt_packages))): WOMysql.backupAll(self) WOService.stop_service(self, 'mysql') diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 63a940b..5c47906 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -78,7 +78,7 @@ class WOStackMigrateController(CementBaseController): log=False) # Install MariaDB - apt_packages = WOVariables.wo_mysql + apt_packages = ["mariadb-server"] Log.info(self, "Updating apt-cache, hang on...") WOAptGet.update(self) From 353fd4192e0a563737a957d96925e8d187ba2877 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 18:40:11 +0200 Subject: [PATCH 13/22] Fix my.cnf copy several time --- wo/cli/plugins/stack_pref.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index a0f3def..41c4da6 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -888,8 +888,9 @@ def post_pref(self, apt_packages, packages, upgrade=False): config_file.close() else: Log.info(self, "Tuning MariaDB configuration") - WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", - "/etc/mysql/my.cnf.default-pkg") + if not os.path.isfile("/etc/mysql/my.cnf.default-pkg"): + WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", + "/etc/mysql/my.cnf.default-pkg") wo_ram = psutil.virtual_memory().total / (1024 * 1024) # set InnoDB variable depending on the RAM available wo_ram_innodb = int(wo_ram*0.3) From fe6519f7022b68ac8656d72f7e5e3c09f2316793 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:10:18 +0200 Subject: [PATCH 14/22] Fix packages purge all --- wo/cli/plugins/stack.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 4da3b90..3179bd3 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -548,7 +548,7 @@ class WOStackController(CementBaseController): pargs.proftpd = True pargs.utils = True pargs.redis = True - packages = packages + ['/var/www/22222/htdocs/*'] + packages = packages + ['/var/www/22222/htdocs'] if pargs.web: pargs.nginx = True @@ -767,7 +767,7 @@ class WOStackController(CementBaseController): pargs.proftpd = True pargs.utils = True pargs.redis = True - packages = packages + ['/var/www/22222/htdocs/*'] + packages = packages + ['/var/www/22222/htdocs'] if pargs.web: pargs.nginx = True @@ -958,7 +958,6 @@ class WOStackController(CementBaseController): if (packages): Log.info(self, "Purging packages, please wait...") WOFileUtils.remove(self, packages) - WOAptGet.auto_remove(self) Log.info(self, "Successfully purged packages") From 9da4bff5e9551d5efd91706357c0846d89d0f05a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:25:09 +0200 Subject: [PATCH 15/22] Improve logging display --- wo/core/logging.py | 17 +++++++++++++--- wo/core/services.py | 49 +++++++++++++++++++++++++-------------------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index 7dbaca2..3078f59 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -52,7 +52,7 @@ class Log: """ print( Log.OKBLUE + msg + - " [" + Log.ENDC + ".." + Log.OKBLUE + "]", end=end) + " [" + Log.ENDC + ".." + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: self.app.log.info(Log.OKBLUE + msg + Log.ENDC) @@ -63,6 +63,17 @@ class Log: print( Log.OKBLUE + msg + " [" + Log.ENDC + Log.OKGREEN + "OK" + - Log.ENDC + Log.OKBLUE + "]", end=end) + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) if log: - self.app.log.info(Log.OKBLUE + msg + Log.ENDC) + self.app.log.info(Log.OKGREEN + msg + Log.ENDC) + + def failed(self, msg, end='\n', log=True): + """ + Logs info messages after validation step + """ + print( + Log.OKBLUE + msg + + " [" + Log.ENDC + Log.FAIL + "OK" + + Log.ENDC + Log.OKBLUE + "]" + Log.ENDC, end=end) + if log: + self.app.log.info(Log.FAIL + msg + Log.ENDC) diff --git a/wo/core/services.py b/wo/core/services.py index 7b73c04..350a84a 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -15,21 +15,21 @@ class WOService(): """ try: if service_name in ['nginx']: + Log.wait(self, "Testing Nginx configuration ") # Check Nginx configuration before executing command sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) output, error_output = sub.communicate() if 'emerg' not in str(error_output): + Log.valide(self, "Testing Nginx configuration ") + Log.wait(self, "Starting Nginx ") service_cmd = ('service {0} start'.format(service_name)) retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "Starting Nginx " + - "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.valide(self, "Starting Nginx ") return True else: - Log.info( - self, "Starting Nginx " + "[" + Log.FAIL + - "Failed" + Log.OKBLUE+"]") + Log.failed(self, "Testing Nginx configuration ") return False else: service_cmd = ('service {0} start'.format(service_name)) @@ -76,32 +76,35 @@ class WOService(): """ try: if service_name in ['nginx']: + Log.wait(self, "Testing Nginx configuration ") # Check Nginx configuration before executing command sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) output, error_output = sub.communicate() if 'emerg' not in str(error_output): + Log.valide(self, "Testing Nginx configuration ") + Log.wait(self, "Restarting Nginx ") service_cmd = ('service {0} restart'.format(service_name)) retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "Restarting Nginx " + - "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.valide(self, "Restarting Nginx ") return True else: - Log.info(self, "Restarting Nginx " + "[" + Log.FAIL + - "Failed" + Log.OKBLUE+"]") + Log.failed(self, "Testing Nginx configuration ") return False else: service_cmd = ('service {0} restart'.format(service_name)) - Log.info(self, "Restart : {0:10}".format( - service_name), end='') + Log.wait(self, "Restarting {0:10}".format( + service_name)) retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.valide(self, "Restarting {0:10}".format( + service_name)) return True else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.failed(self, "Restarting {0:10}".format( + service_name)) return False except OSError as e: Log.debug(self, "{0} {1}".format(e.errno, e.strerror)) @@ -116,31 +119,33 @@ class WOService(): try: if service_name in ['nginx']: # Check Nginx configuration before executing command + Log.wait(self, "Testing Nginx configuration ") sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) output, error_output = sub.communicate() if 'emerg' not in str(error_output): + Log.valide(self, "Testing Nginx configuration ") + Log.wait(self, "Reloading Nginx ") service_cmd = ('service {0} reload'.format(service_name)) retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "Reloading Nginx " + - "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") + Log.valide(self, "Reloading Nginx ") return True else: - Log.info( - self, "Reloading Nginx " + "[" + Log.FAIL + - "Failed" + Log.OKBLUE+"]") + Log.failed(self, "Testing Nginx configuration ") return False else: service_cmd = ('service {0} reload'.format(service_name)) - Log.info(self, "Reload : {0:10}".format(service_name), end='') + Log.wait(self, "Reloading {0:10}".format( + service_name)) retcode = subprocess.getstatusoutput(service_cmd) if retcode[0] == 0: - Log.info(self, "[" + Log.ENDC + "OK" + Log.OKBLUE + "]") - return True + Log.valide(self, "Reloading {0:10}".format( + service_name)) else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.failed(self, "Reloading {0:10}".format( + service_name)) return False except OSError as e: Log.debug(self, "{0}".format(e)) From a2c1e812769f8fdd64661fb2340144f189030582 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:27:38 +0200 Subject: [PATCH 16/22] Fix install log --- wo/cli/plugins/stack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 3179bd3..9def7d3 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -498,7 +498,7 @@ class WOStackController(CementBaseController): Log.wait(self, "Updating apt-cache ") WOAptGet.update(self) Log.valide(self, "Updating apt-cache ") - Log.info(self, "Installing APT packages ") + Log.wait(self, "Installing APT packages ") WOAptGet.install(self, apt_packages) Log.valide(self, "Installing APT packages ") Log.wait(self, "Configuring APT packages ") @@ -724,9 +724,9 @@ class WOStackController(CementBaseController): errormsg='', log=False) if (packages): - Log.info(self, "Removing packages, please wait...") + Log.wait(self, "Removing packages ") WOFileUtils.remove(self, packages) - + Log.valide(self, "Removing packages ") if (apt_packages): Log.debug(self, "Removing apt_packages") Log.wait(self, "Removing APT packages ") @@ -956,9 +956,9 @@ class WOStackController(CementBaseController): Log.valide(self, "Purging APT Packages ") if (packages): - Log.info(self, "Purging packages, please wait...") + Log.wait(self, "Purging Packages ") WOFileUtils.remove(self, packages) - + Log.valide(self, "Purging Packages ") Log.info(self, "Successfully purged packages") From 494d0e34ddf08247d5ff0989ba10660e47efea65 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:44:57 +0200 Subject: [PATCH 17/22] Fix innodb_pool_instances calculation --- wo/cli/plugins/stack_pref.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 41c4da6..0e61cf7 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -896,13 +896,13 @@ def post_pref(self, apt_packages, packages, upgrade=False): wo_ram_innodb = int(wo_ram*0.3) wo_ram_log_buffer = int(wo_ram_innodb*0.25) wo_ram_log_size = int(wo_ram_log_buffer*0.5) - if (wo_ram_innodb > 2000) and (wo_ram_innodb < 64000): - wo_innodb_instance = int(wo_ram_innodb/1000) - tmp_table_size = int(128) - elif (wo_ram_innodb < 2000): + if (wo_ram < 2000): wo_innodb_instance = int(1) tmp_table_size = int(32) - elif (wo_ram_innodb > 64000): + elif (wo_ram > 2000) and (wo_ram < 64000): + wo_innodb_instance = int(wo_ram/1000) + tmp_table_size = int(128) + elif (wo_ram > 64000): wo_innodb_instance = int(64) tmp_table_size = int(256) data = dict( From f34885fc4a1ee20fa53e3f032a2bc6adea2a74ba Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:53:19 +0200 Subject: [PATCH 18/22] Also purge mysql-common --- wo/cli/plugins/stack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 9def7d3..72f6c79 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -601,7 +601,8 @@ class WOStackController(CementBaseController): # MariaDB if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") - apt_packages = apt_packages + ['mariadb-server'] + apt_packages = apt_packages + ['mariadb-server', 'mysql-common', + 'mariadb-client'] # mysqlclient if pargs.mysqlclient: @@ -820,7 +821,8 @@ class WOStackController(CementBaseController): # MariaDB if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") - apt_packages = apt_packages + ['mariadb-server'] + apt_packages = apt_packages + ['mariadb-server', 'mysql-common', + 'mariadb-client'] # mysqlclient if pargs.mysqlclient: From 3696afc03e66e7cf4291b06d79d72233a0d480ef Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 19:57:12 +0200 Subject: [PATCH 19/22] [skip travis] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c302c2..ad83701 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Improve Let's Encrypt certificate issuance logging informations +- MariaDB configuration & optimization is now rendered from a template (can be protected against overwriting with .custom) #### Fixed @@ -22,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - subdomain detection with complex suffixes like com.br - Fix mariadb install/upgrade when running mariadb-10.1 - Fix mariadb install/upgrade on raspbian and debian 8 +- Fix mariadb tuning wrong pool_instance calculation ### v3.9.8.9 - 2019-09-03 From 50cc8a4d094ef91b403e63277141b6b95884c336 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 20:36:15 +0200 Subject: [PATCH 20/22] Sort import, create class DMN --- wo/cli/plugins/clean.py | 14 +- wo/cli/plugins/debug.py | 26 +- wo/cli/plugins/import_slow_log.py | 3 +- wo/cli/plugins/info.py | 594 +++++++++++++++--------------- wo/cli/plugins/log.py | 1 + wo/cli/plugins/maintenance.py | 1 + wo/cli/plugins/models.py | 3 +- wo/cli/plugins/secure.py | 16 +- wo/cli/plugins/site.py | 28 +- wo/cli/plugins/site_functions.py | 4 +- wo/cli/plugins/sitedb.py | 11 +- wo/cli/plugins/stack_pref.py | 1 - wo/cli/plugins/stack_services.py | 1 + wo/cli/plugins/stack_upgrade.py | 1 + wo/cli/plugins/sync.py | 1 + wo/cli/plugins/update.py | 1 + wo/core/addswap.py | 10 +- wo/core/apt_repo.py | 5 +- wo/core/aptget.py | 11 +- wo/core/checkfqdn.py | 3 +- wo/core/cron.py | 3 +- wo/core/domainvalidate.py | 83 ++--- wo/core/download.py | 5 +- wo/core/extract.py | 3 +- wo/core/fileutils.py | 5 +- wo/core/git.py | 6 +- wo/core/logwatch.py | 5 +- wo/core/mysql.py | 8 +- wo/core/nginxhashbucket.py | 5 +- wo/core/sendmail.py | 6 +- wo/core/services.py | 1 + wo/core/shellexec.py | 3 +- wo/core/template.py | 4 +- wo/core/variables.py | 7 +- 34 files changed, 458 insertions(+), 421 deletions(-) diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index a8e0052..a1b1231 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -1,14 +1,16 @@ """Clean Plugin for WordOps.""" -from wo.core.shellexec import WOShellExec -from wo.core.aptget import WOAptGet -from wo.core.services import WOService -from wo.core.logging import Log -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook import os import urllib.request +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose + +from wo.core.aptget import WOAptGet +from wo.core.logging import Log +from wo.core.services import WOService +from wo.core.shellexec import WOShellExec + def wo_clean_hook(app): pass diff --git a/wo/cli/plugins/debug.py b/wo/cli/plugins/debug.py index 391b3fc..9c2def0 100644 --- a/wo/cli/plugins/debug.py +++ b/wo/cli/plugins/debug.py @@ -1,21 +1,23 @@ """Debug Plugin for WordOps""" -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.aptget import WOAptGet -from wo.core.shellexec import WOShellExec -from wo.core.mysql import WOMysql -from wo.core.services import WOService -from wo.core.logging import Log -from wo.cli.plugins.site_functions import logwatch -from wo.core.variables import WOVariables -from wo.core.fileutils import WOFileUtils -from pynginxconfig import NginxConfig -import os import configparser import glob +import os import signal +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from pynginxconfig import NginxConfig + +from wo.cli.plugins.site_functions import logwatch +from wo.core.aptget import WOAptGet +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.mysql import WOMysql +from wo.core.services import WOService +from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables + def wo_debug_hook(app): pass diff --git a/wo/cli/plugins/import_slow_log.py b/wo/cli/plugins/import_slow_log.py index 6e51b51..bc13ff8 100644 --- a/wo/cli/plugins/import_slow_log.py +++ b/wo/cli/plugins/import_slow_log.py @@ -1,5 +1,6 @@ -from cement.core.controller import CementBaseController, expose from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose + from wo.core.logging import Log diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index efaf767..0a6a145 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -1,296 +1,298 @@ -"""WOInfo Plugin for WordOps""" - -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from pynginxconfig import NginxConfig -from wo.core.aptget import WOAptGet -from wo.core.shellexec import WOShellExec -from wo.core.logging import Log -import os -import configparser - - -def wo_info_hook(app): - pass - - -class WOInfoController(CementBaseController): - class Meta: - label = 'info' - stacked_on = 'base' - stacked_type = 'nested' - description = ('Display configuration information related to Nginx,' - ' PHP and MySQL') - arguments = [ - (['--mysql'], - dict(help='Get MySQL configuration information', - action='store_true')), - (['--php'], - dict(help='Get PHP 7.2 configuration information', - action='store_true')), - (['--php73'], - dict(help='Get PHP 7.3 configuration information', - action='store_true')), - (['--nginx'], - dict(help='Get Nginx configuration information', - action='store_true')), - ] - usage = "wo info [options]" - - @expose(hide=True) - def info_nginx(self): - """Display Nginx information""" - version = os.popen("/usr/sbin/nginx -v 2>&1 | " - "awk -F '/' '{print $2}' | " - "awk -F ' ' '{print $1}' | tr '\n' ' '").read() - allow = os.popen("grep ^allow /etc/nginx/common/acl.conf | " - "cut -d' ' -f2 | cut -d';' -f1 | tr '\n' ' '").read() - nc = NginxConfig() - nc.loadf('/etc/nginx/nginx.conf') - user = nc.get('user')[1] - worker_processes = nc.get('worker_processes')[1] - worker_connections = nc.get([('events',), 'worker_connections'])[1] - keepalive_timeout = nc.get([('http',), 'keepalive_timeout'])[1] - fastcgi_read_timeout = nc.get([('http',), - 'fastcgi_read_timeout'])[1] - client_max_body_size = nc.get([('http',), - 'client_max_body_size'])[1] - data = dict(version=version, allow=allow, user=user, - worker_processes=worker_processes, - keepalive_timeout=keepalive_timeout, - worker_connections=worker_connections, - fastcgi_read_timeout=fastcgi_read_timeout, - client_max_body_size=client_max_body_size) - self.app.render((data), 'info_nginx.mustache') - - @expose(hide=True) - def info_php(self): - """Display PHP information""" - version = os.popen("/usr/bin/php7.2 -v 2>/dev/null | " - "head -n1 | cut -d' ' -f2 |" - " cut -d'+' -f1 | tr -d '\n'").read - config = configparser.ConfigParser() - config.read('/etc/{0}/fpm/php.ini'.format("php/7.2")) - expose_php = config['PHP']['expose_php'] - memory_limit = config['PHP']['memory_limit'] - post_max_size = config['PHP']['post_max_size'] - upload_max_filesize = config['PHP']['upload_max_filesize'] - max_execution_time = config['PHP']['max_execution_time'] - - config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - www_listen = config['www']['listen'] - www_ping_path = config['www']['ping.path'] - www_pm_status_path = config['www']['pm.status_path'] - www_pm = config['www']['pm'] - www_pm_max_requests = config['www']['pm.max_requests'] - www_pm_max_children = config['www']['pm.max_children'] - www_pm_start_servers = config['www']['pm.start_servers'] - www_pm_min_spare_servers = config['www']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www']['pm.max_spare_servers'] - www_request_terminate_time = (config['www'] - ['request_terminate_timeout']) - try: - www_xdebug = (config['www']['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) - except Exception as e: - Log.debug(self, "{0}".format(e)) - www_xdebug = 'off' - - config.read('/etc/{0}/fpm/pool.d/debug.conf'.format("php/7.2")) - debug_listen = config['debug']['listen'] - debug_ping_path = config['debug']['ping.path'] - debug_pm_status_path = config['debug']['pm.status_path'] - debug_pm = config['debug']['pm'] - debug_pm_max_requests = config['debug']['pm.max_requests'] - debug_pm_max_children = config['debug']['pm.max_children'] - debug_pm_start_servers = config['debug']['pm.start_servers'] - debug_pm_min_spare_servers = config['debug']['pm.min_spare_servers'] - debug_pm_max_spare_servers = config['debug']['pm.max_spare_servers'] - debug_request_terminate = (config['debug'] - ['request_terminate_timeout']) - try: - debug_xdebug = (config['debug']['php_admin_flag[xdebug.profiler_' - 'enable_trigger]']) - except Exception as e: - Log.debug(self, "{0}".format(e)) - debug_xdebug = 'off' - - data = dict(version=version, expose_php=expose_php, - memory_limit=memory_limit, post_max_size=post_max_size, - upload_max_filesize=upload_max_filesize, - max_execution_time=max_execution_time, - www_listen=www_listen, www_ping_path=www_ping_path, - www_pm_status_path=www_pm_status_path, www_pm=www_pm, - www_pm_max_requests=www_pm_max_requests, - www_pm_max_children=www_pm_max_children, - www_pm_start_servers=www_pm_start_servers, - www_pm_min_spare_servers=www_pm_min_spare_servers, - www_pm_max_spare_servers=www_pm_max_spare_servers, - www_request_terminate_timeout=www_request_terminate_time, - www_xdebug_profiler_enable_trigger=www_xdebug, - debug_listen=debug_listen, debug_ping_path=debug_ping_path, - debug_pm_status_path=debug_pm_status_path, - debug_pm=debug_pm, - debug_pm_max_requests=debug_pm_max_requests, - debug_pm_max_children=debug_pm_max_children, - debug_pm_start_servers=debug_pm_start_servers, - debug_pm_min_spare_servers=debug_pm_min_spare_servers, - debug_pm_max_spare_servers=debug_pm_max_spare_servers, - debug_request_terminate_timeout=debug_request_terminate, - debug_xdebug_profiler_enable_trigger=debug_xdebug) - self.app.render((data), 'info_php.mustache') - - @expose(hide=True) - def info_php73(self): - """Display PHP information""" - version = os.popen("/usr/bin/php7.3 -v 2>/dev/null | " - "head -n1 | cut -d' ' -f2 |" - " cut -d'+' -f1 | tr -d '\n'").read - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/php.ini') - expose_php = config['PHP']['expose_php'] - memory_limit = config['PHP']['memory_limit'] - post_max_size = config['PHP']['post_max_size'] - upload_max_filesize = config['PHP']['upload_max_filesize'] - max_execution_time = config['PHP']['max_execution_time'] - - config.read('/etc/php/7.3/fpm/pool.d/www.conf') - www_listen = config['www']['listen'] - www_ping_path = config['www']['ping.path'] - www_pm_status_path = config['www']['pm.status_path'] - www_pm = config['www']['pm'] - www_pm_max_requests = config['www']['pm.max_requests'] - www_pm_max_children = config['www']['pm.max_children'] - www_pm_start_servers = config['www']['pm.start_servers'] - www_pm_min_spare_servers = config['www']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www']['pm.max_spare_servers'] - www_request_terminate_time = (config['www'] - ['request_terminate_timeout']) - try: - www_xdebug = (config['www']['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) - except Exception as e: - Log.debug(self, "{0}".format(e)) - www_xdebug = 'off' - - config.read('/etc/php/7.3/fpm/pool.d/debug.conf') - debug_listen = config['debug']['listen'] - debug_ping_path = config['debug']['ping.path'] - debug_pm_status_path = config['debug']['pm.status_path'] - debug_pm = config['debug']['pm'] - debug_pm_max_requests = config['debug']['pm.max_requests'] - debug_pm_max_children = config['debug']['pm.max_children'] - debug_pm_start_servers = config['debug']['pm.start_servers'] - debug_pm_min_spare_servers = config['debug']['pm.min_spare_servers'] - debug_pm_max_spare_servers = config['debug']['pm.max_spare_servers'] - debug_request_terminate = (config['debug'] - ['request_terminate_timeout']) - try: - debug_xdebug = (config['debug']['php_admin_flag[xdebug.profiler_' - 'enable_trigger]']) - except Exception as e: - Log.debug(self, "{0}".format(e)) - debug_xdebug = 'off' - - data = dict(version=version, expose_php=expose_php, - memory_limit=memory_limit, post_max_size=post_max_size, - upload_max_filesize=upload_max_filesize, - max_execution_time=max_execution_time, - www_listen=www_listen, www_ping_path=www_ping_path, - www_pm_status_path=www_pm_status_path, www_pm=www_pm, - www_pm_max_requests=www_pm_max_requests, - www_pm_max_children=www_pm_max_children, - www_pm_start_servers=www_pm_start_servers, - www_pm_min_spare_servers=www_pm_min_spare_servers, - www_pm_max_spare_servers=www_pm_max_spare_servers, - www_request_terminate_timeout=www_request_terminate_time, - www_xdebug_profiler_enable_trigger=www_xdebug, - debug_listen=debug_listen, debug_ping_path=debug_ping_path, - debug_pm_status_path=debug_pm_status_path, - debug_pm=debug_pm, - debug_pm_max_requests=debug_pm_max_requests, - debug_pm_max_children=debug_pm_max_children, - debug_pm_start_servers=debug_pm_start_servers, - debug_pm_min_spare_servers=debug_pm_min_spare_servers, - debug_pm_max_spare_servers=debug_pm_max_spare_servers, - debug_request_terminate_timeout=debug_request_terminate, - debug_xdebug_profiler_enable_trigger=debug_xdebug) - self.app.render((data), 'info_php.mustache') - - @expose(hide=True) - def info_mysql(self): - """Display MySQL information""" - version = os.popen("/usr/bin/mysql -V | awk '{print($5)}' | " - "cut -d ',' " - "-f1 | tr -d '\n'").read() - host = "localhost" - port = os.popen("/usr/bin/mysql -e \"show variables\" | " - "/bin/grep ^port | awk " - "'{print($2)}' | tr -d '\n'").read() - wait_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " - "^wait_timeout | awk '{print($2)}' | " - "tr -d '\n'").read() - interactive_timeout = os.popen("/usr/bin/mysql -e " - "\"show variables\" | grep " - "^interactive_timeout | awk " - "'{print($2)}' | tr -d '\n'").read() - max_used_connections = os.popen("/usr/bin/mysql -e " - "\"show global status\" | " - "grep Max_used_connections | awk " - "'{print($2)}' | tr -d '\n'").read() - datadir = os.popen("/usr/bin/mysql -e \"show variables\" | " - "/bin/grep datadir | awk" - " '{print($2)}' | tr -d '\n'").read() - socket = os.popen("/usr/bin/mysql -e \"show variables\" | " - "/bin/grep \"^socket\" | " - "awk '{print($2)}' | tr -d '\n'").read() - data = dict(version=version, host=host, port=port, - wait_timeout=wait_timeout, - interactive_timeout=interactive_timeout, - max_used_connections=max_used_connections, - datadir=datadir, socket=socket) - self.app.render((data), 'info_mysql.mustache') - - @expose(hide=True) - def default(self): - """default function for info""" - if (not self.app.pargs.nginx and not self.app.pargs.php and - not self.app.pargs.mysql and not self.app.pargs.php73): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - if WOAptGet.is_installed(self, 'php7.3-fpm'): - self.app.pargs.php73 = True - - if self.app.pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-wo')): - self.info_nginx() - else: - Log.error(self, "Nginx is not installed") - - if self.app.pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): - self.info_php() - else: - Log.error(self, "PHP 7.2 is not installed") - - if self.app.pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): - self.info_php73() - else: - Log.error(self, "PHP 7.3 is not installed") - - if self.app.pargs.mysql: - if WOShellExec.cmd_exec(self, "/usr/bin/mysqladmin ping"): - self.info_mysql() - else: - Log.error(self, "MySQL is not installed") - - -def load(app): - # register the plugin class.. this only happens if the plugin is enabled - handler.register(WOInfoController) - - # register a hook (function) to run after arguments are parsed. - hook.register('post_argument_parsing', wo_info_hook) +"""WOInfo Plugin for WordOps""" + +import configparser +import os + +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from pynginxconfig import NginxConfig + +from wo.core.aptget import WOAptGet +from wo.core.logging import Log +from wo.core.shellexec import WOShellExec + + +def wo_info_hook(app): + pass + + +class WOInfoController(CementBaseController): + class Meta: + label = 'info' + stacked_on = 'base' + stacked_type = 'nested' + description = ('Display configuration information related to Nginx,' + ' PHP and MySQL') + arguments = [ + (['--mysql'], + dict(help='Get MySQL configuration information', + action='store_true')), + (['--php'], + dict(help='Get PHP 7.2 configuration information', + action='store_true')), + (['--php73'], + dict(help='Get PHP 7.3 configuration information', + action='store_true')), + (['--nginx'], + dict(help='Get Nginx configuration information', + action='store_true')), + ] + usage = "wo info [options]" + + @expose(hide=True) + def info_nginx(self): + """Display Nginx information""" + version = os.popen("/usr/sbin/nginx -v 2>&1 | " + "awk -F '/' '{print $2}' | " + "awk -F ' ' '{print $1}' | tr '\n' ' '").read() + allow = os.popen("grep ^allow /etc/nginx/common/acl.conf | " + "cut -d' ' -f2 | cut -d';' -f1 | tr '\n' ' '").read() + nc = NginxConfig() + nc.loadf('/etc/nginx/nginx.conf') + user = nc.get('user')[1] + worker_processes = nc.get('worker_processes')[1] + worker_connections = nc.get([('events',), 'worker_connections'])[1] + keepalive_timeout = nc.get([('http',), 'keepalive_timeout'])[1] + fastcgi_read_timeout = nc.get([('http',), + 'fastcgi_read_timeout'])[1] + client_max_body_size = nc.get([('http',), + 'client_max_body_size'])[1] + data = dict(version=version, allow=allow, user=user, + worker_processes=worker_processes, + keepalive_timeout=keepalive_timeout, + worker_connections=worker_connections, + fastcgi_read_timeout=fastcgi_read_timeout, + client_max_body_size=client_max_body_size) + self.app.render((data), 'info_nginx.mustache') + + @expose(hide=True) + def info_php(self): + """Display PHP information""" + version = os.popen("/usr/bin/php7.2 -v 2>/dev/null | " + "head -n1 | cut -d' ' -f2 |" + " cut -d'+' -f1 | tr -d '\n'").read + config = configparser.ConfigParser() + config.read('/etc/{0}/fpm/php.ini'.format("php/7.2")) + expose_php = config['PHP']['expose_php'] + memory_limit = config['PHP']['memory_limit'] + post_max_size = config['PHP']['post_max_size'] + upload_max_filesize = config['PHP']['upload_max_filesize'] + max_execution_time = config['PHP']['max_execution_time'] + + config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) + www_listen = config['www']['listen'] + www_ping_path = config['www']['ping.path'] + www_pm_status_path = config['www']['pm.status_path'] + www_pm = config['www']['pm'] + www_pm_max_requests = config['www']['pm.max_requests'] + www_pm_max_children = config['www']['pm.max_children'] + www_pm_start_servers = config['www']['pm.start_servers'] + www_pm_min_spare_servers = config['www']['pm.min_spare_servers'] + www_pm_max_spare_servers = config['www']['pm.max_spare_servers'] + www_request_terminate_time = (config['www'] + ['request_terminate_timeout']) + try: + www_xdebug = (config['www']['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) + except Exception as e: + Log.debug(self, "{0}".format(e)) + www_xdebug = 'off' + + config.read('/etc/{0}/fpm/pool.d/debug.conf'.format("php/7.2")) + debug_listen = config['debug']['listen'] + debug_ping_path = config['debug']['ping.path'] + debug_pm_status_path = config['debug']['pm.status_path'] + debug_pm = config['debug']['pm'] + debug_pm_max_requests = config['debug']['pm.max_requests'] + debug_pm_max_children = config['debug']['pm.max_children'] + debug_pm_start_servers = config['debug']['pm.start_servers'] + debug_pm_min_spare_servers = config['debug']['pm.min_spare_servers'] + debug_pm_max_spare_servers = config['debug']['pm.max_spare_servers'] + debug_request_terminate = (config['debug'] + ['request_terminate_timeout']) + try: + debug_xdebug = (config['debug']['php_admin_flag[xdebug.profiler_' + 'enable_trigger]']) + except Exception as e: + Log.debug(self, "{0}".format(e)) + debug_xdebug = 'off' + + data = dict(version=version, expose_php=expose_php, + memory_limit=memory_limit, post_max_size=post_max_size, + upload_max_filesize=upload_max_filesize, + max_execution_time=max_execution_time, + www_listen=www_listen, www_ping_path=www_ping_path, + www_pm_status_path=www_pm_status_path, www_pm=www_pm, + www_pm_max_requests=www_pm_max_requests, + www_pm_max_children=www_pm_max_children, + www_pm_start_servers=www_pm_start_servers, + www_pm_min_spare_servers=www_pm_min_spare_servers, + www_pm_max_spare_servers=www_pm_max_spare_servers, + www_request_terminate_timeout=www_request_terminate_time, + www_xdebug_profiler_enable_trigger=www_xdebug, + debug_listen=debug_listen, debug_ping_path=debug_ping_path, + debug_pm_status_path=debug_pm_status_path, + debug_pm=debug_pm, + debug_pm_max_requests=debug_pm_max_requests, + debug_pm_max_children=debug_pm_max_children, + debug_pm_start_servers=debug_pm_start_servers, + debug_pm_min_spare_servers=debug_pm_min_spare_servers, + debug_pm_max_spare_servers=debug_pm_max_spare_servers, + debug_request_terminate_timeout=debug_request_terminate, + debug_xdebug_profiler_enable_trigger=debug_xdebug) + self.app.render((data), 'info_php.mustache') + + @expose(hide=True) + def info_php73(self): + """Display PHP information""" + version = os.popen("/usr/bin/php7.3 -v 2>/dev/null | " + "head -n1 | cut -d' ' -f2 |" + " cut -d'+' -f1 | tr -d '\n'").read + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/php.ini') + expose_php = config['PHP']['expose_php'] + memory_limit = config['PHP']['memory_limit'] + post_max_size = config['PHP']['post_max_size'] + upload_max_filesize = config['PHP']['upload_max_filesize'] + max_execution_time = config['PHP']['max_execution_time'] + + config.read('/etc/php/7.3/fpm/pool.d/www.conf') + www_listen = config['www']['listen'] + www_ping_path = config['www']['ping.path'] + www_pm_status_path = config['www']['pm.status_path'] + www_pm = config['www']['pm'] + www_pm_max_requests = config['www']['pm.max_requests'] + www_pm_max_children = config['www']['pm.max_children'] + www_pm_start_servers = config['www']['pm.start_servers'] + www_pm_min_spare_servers = config['www']['pm.min_spare_servers'] + www_pm_max_spare_servers = config['www']['pm.max_spare_servers'] + www_request_terminate_time = (config['www'] + ['request_terminate_timeout']) + try: + www_xdebug = (config['www']['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) + except Exception as e: + Log.debug(self, "{0}".format(e)) + www_xdebug = 'off' + + config.read('/etc/php/7.3/fpm/pool.d/debug.conf') + debug_listen = config['debug']['listen'] + debug_ping_path = config['debug']['ping.path'] + debug_pm_status_path = config['debug']['pm.status_path'] + debug_pm = config['debug']['pm'] + debug_pm_max_requests = config['debug']['pm.max_requests'] + debug_pm_max_children = config['debug']['pm.max_children'] + debug_pm_start_servers = config['debug']['pm.start_servers'] + debug_pm_min_spare_servers = config['debug']['pm.min_spare_servers'] + debug_pm_max_spare_servers = config['debug']['pm.max_spare_servers'] + debug_request_terminate = (config['debug'] + ['request_terminate_timeout']) + try: + debug_xdebug = (config['debug']['php_admin_flag[xdebug.profiler_' + 'enable_trigger]']) + except Exception as e: + Log.debug(self, "{0}".format(e)) + debug_xdebug = 'off' + + data = dict(version=version, expose_php=expose_php, + memory_limit=memory_limit, post_max_size=post_max_size, + upload_max_filesize=upload_max_filesize, + max_execution_time=max_execution_time, + www_listen=www_listen, www_ping_path=www_ping_path, + www_pm_status_path=www_pm_status_path, www_pm=www_pm, + www_pm_max_requests=www_pm_max_requests, + www_pm_max_children=www_pm_max_children, + www_pm_start_servers=www_pm_start_servers, + www_pm_min_spare_servers=www_pm_min_spare_servers, + www_pm_max_spare_servers=www_pm_max_spare_servers, + www_request_terminate_timeout=www_request_terminate_time, + www_xdebug_profiler_enable_trigger=www_xdebug, + debug_listen=debug_listen, debug_ping_path=debug_ping_path, + debug_pm_status_path=debug_pm_status_path, + debug_pm=debug_pm, + debug_pm_max_requests=debug_pm_max_requests, + debug_pm_max_children=debug_pm_max_children, + debug_pm_start_servers=debug_pm_start_servers, + debug_pm_min_spare_servers=debug_pm_min_spare_servers, + debug_pm_max_spare_servers=debug_pm_max_spare_servers, + debug_request_terminate_timeout=debug_request_terminate, + debug_xdebug_profiler_enable_trigger=debug_xdebug) + self.app.render((data), 'info_php.mustache') + + @expose(hide=True) + def info_mysql(self): + """Display MySQL information""" + version = os.popen("/usr/bin/mysql -V | awk '{print($5)}' | " + "cut -d ',' " + "-f1 | tr -d '\n'").read() + host = "localhost" + port = os.popen("/usr/bin/mysql -e \"show variables\" | " + "/bin/grep ^port | awk " + "'{print($2)}' | tr -d '\n'").read() + wait_timeout = os.popen("/usr/bin/mysql -e \"show variables\" | grep " + "^wait_timeout | awk '{print($2)}' | " + "tr -d '\n'").read() + interactive_timeout = os.popen("/usr/bin/mysql -e " + "\"show variables\" | grep " + "^interactive_timeout | awk " + "'{print($2)}' | tr -d '\n'").read() + max_used_connections = os.popen("/usr/bin/mysql -e " + "\"show global status\" | " + "grep Max_used_connections | awk " + "'{print($2)}' | tr -d '\n'").read() + datadir = os.popen("/usr/bin/mysql -e \"show variables\" | " + "/bin/grep datadir | awk" + " '{print($2)}' | tr -d '\n'").read() + socket = os.popen("/usr/bin/mysql -e \"show variables\" | " + "/bin/grep \"^socket\" | " + "awk '{print($2)}' | tr -d '\n'").read() + data = dict(version=version, host=host, port=port, + wait_timeout=wait_timeout, + interactive_timeout=interactive_timeout, + max_used_connections=max_used_connections, + datadir=datadir, socket=socket) + self.app.render((data), 'info_mysql.mustache') + + @expose(hide=True) + def default(self): + """default function for info""" + if (not self.app.pargs.nginx and not self.app.pargs.php and + not self.app.pargs.mysql and not self.app.pargs.php73): + self.app.pargs.nginx = True + self.app.pargs.php = True + self.app.pargs.mysql = True + if WOAptGet.is_installed(self, 'php7.3-fpm'): + self.app.pargs.php73 = True + + if self.app.pargs.nginx: + if (WOAptGet.is_installed(self, 'nginx-custom') or + WOAptGet.is_installed(self, 'nginx-wo')): + self.info_nginx() + else: + Log.error(self, "Nginx is not installed") + + if self.app.pargs.php: + if WOAptGet.is_installed(self, 'php7.2-fpm'): + self.info_php() + else: + Log.error(self, "PHP 7.2 is not installed") + + if self.app.pargs.php73: + if WOAptGet.is_installed(self, 'php7.3-fpm'): + self.info_php73() + else: + Log.error(self, "PHP 7.3 is not installed") + + if self.app.pargs.mysql: + if WOShellExec.cmd_exec(self, "/usr/bin/mysqladmin ping"): + self.info_mysql() + else: + Log.error(self, "MySQL is not installed") + + +def load(app): + # register the plugin class.. this only happens if the plugin is enabled + handler.register(WOInfoController) + + # register a hook (function) to run after arguments are parsed. + hook.register('post_argument_parsing', wo_info_hook) diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index d080368..d2d2fe3 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -6,6 +6,7 @@ import os from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.cli.plugins.site_functions import logwatch from wo.core.fileutils import WOFileUtils from wo.core.logging import Log diff --git a/wo/cli/plugins/maintenance.py b/wo/cli/plugins/maintenance.py index ad65841..8d45967 100644 --- a/wo/cli/plugins/maintenance.py +++ b/wo/cli/plugins/maintenance.py @@ -2,6 +2,7 @@ from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.core.aptget import WOAptGet from wo.core.logging import Log diff --git a/wo/cli/plugins/models.py b/wo/cli/plugins/models.py index a609c67..382aab7 100644 --- a/wo/cli/plugins/models.py +++ b/wo/cli/plugins/models.py @@ -1,4 +1,5 @@ -from sqlalchemy import Column, DateTime, String, Integer, Boolean, func +from sqlalchemy import Boolean, Column, DateTime, Integer, String, func + from wo.core.database import Base diff --git a/wo/cli/plugins/secure.py b/wo/cli/plugins/secure.py index 119a31b..e8ca67e 100644 --- a/wo/cli/plugins/secure.py +++ b/wo/cli/plugins/secure.py @@ -1,13 +1,15 @@ -from cement.core.controller import CementBaseController, expose +import getpass +import random +import string + from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose + +from wo.core.git import WOGit +from wo.core.logging import Log +from wo.core.services import WOService from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables -from wo.core.logging import Log -from wo.core.git import WOGit -from wo.core.services import WOService -import string -import random -import getpass def wo_secure_hook(app): diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 5d420cc..2050fad 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -10,7 +10,7 @@ from cement.core.controller import CementBaseController, expose from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import (addNewSite, deleteSiteInfo, getAllsites, getSiteInfo, updateSiteInfo) -from wo.core.domainvalidate import GetDomainlevel, ValidateDomain +from wo.core.domainvalidate import DMN from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log @@ -57,7 +57,7 @@ class WOSiteController(CementBaseController): pargs.site_name = pargs.site_name.strip() # validate domain name - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) # check if site exists if not check_domain_exists(self, wo_domain): @@ -94,7 +94,7 @@ class WOSiteController(CementBaseController): Log.debug(self, str(e)) Log.error(self, 'could not input site name') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) # check if site exists if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -134,8 +134,8 @@ class WOSiteController(CementBaseController): Log.debug(self, str(e)) Log.error(self, 'could not input site name') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) - (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) + (wo_domain_type, wo_root_domain) = DMN.getdomainlevel(self, wo_domain) wo_db_name = '' wo_db_user = '' wo_db_pass = '' @@ -186,7 +186,7 @@ class WOSiteController(CementBaseController): def log(self): pargs = self.app.pargs pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) wo_site_webroot = getSiteInfo(self, wo_domain).site_path if not check_domain_exists(self, wo_domain): @@ -208,7 +208,7 @@ class WOSiteController(CementBaseController): Log.error(self, 'could not input site name') # TODO Write code for wo site edit command here pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -239,7 +239,7 @@ class WOSiteController(CementBaseController): Log.error(self, 'Unable to read input, please try again') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -280,7 +280,7 @@ class WOSiteEditController(CementBaseController): Log.error(self, 'Unable to read input, Please try again') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -425,7 +425,7 @@ class WOSiteCreateController(CementBaseController): Log.error(self, "Unable to input site name, Please try again!") pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) if not wo_domain.strip(): Log.error(self, "Invalid domain name, " "Provide valid domain name") @@ -718,7 +718,7 @@ class WOSiteCreateController(CementBaseController): "`tail /var/log/wo/wordops.log` and please try again") if pargs.letsencrypt: - (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain) + (wo_domain_type, wo_root_domain) = DMN.getdomainlevel(self, wo_domain) data['letsencrypt'] = True letsencrypt = True if data['letsencrypt'] is True: @@ -931,7 +931,7 @@ class WOSiteUpdateController(CementBaseController): Log.error(self, 'Unable to input site name, Please try again!') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) wo_site_webroot = WOVariables.wo_webroot + wo_domain check_site = getSiteInfo(self, wo_domain) @@ -1129,7 +1129,7 @@ class WOSiteUpdateController(CementBaseController): pargs.php73 = False if pargs.letsencrypt: - (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain) + (wo_domain_type, wo_root_domain) = DMN.getdomainlevel(self, wo_domain) if pargs.letsencrypt == 'on': data['letsencrypt'] = True letsencrypt = True @@ -1858,7 +1858,7 @@ class WOSiteDeleteController(CementBaseController): Log.error(self, 'could not input site name') pargs.site_name = pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + (wo_domain, wo_www_domain) = DMN.validatedomain(self, pargs.site_name) wo_db_name = '' wo_prompt = '' wo_nginx_prompt = '' diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 7afdc8a..80bbb1a 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1,8 +1,8 @@ import getpass import glob +import json import os import random -import json import re import string import subprocess @@ -10,13 +10,13 @@ from subprocess import CalledProcessError from wo.cli.plugins.sitedb import getSiteInfo from wo.cli.plugins.stack import WOStackController +from wo.cli.plugins.stack_pref import post_pref from wo.core.aptget import WOAptGet from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.services import WOService -from wo.cli.plugins.stack_pref import post_pref from wo.core.shellexec import CommandExecutionError, WOShellExec from wo.core.sslutils import SSL from wo.core.variables import WOVariables diff --git a/wo/cli/plugins/sitedb.py b/wo/cli/plugins/sitedb.py index 9e21e13..2ed7aa2 100644 --- a/wo/cli/plugins/sitedb.py +++ b/wo/cli/plugins/sitedb.py @@ -1,10 +1,11 @@ -from sqlalchemy import Column, DateTime, String, Integer, Boolean -from sqlalchemy import ForeignKey, func -from sqlalchemy.orm import relationship, backref +from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Integer, String, + func) from sqlalchemy.ext.declarative import declarative_base -from wo.core.logging import Log -from wo.core.database import db_session +from sqlalchemy.orm import backref, relationship + from wo.cli.plugins.models import SiteDB +from wo.core.database import db_session +from wo.core.logging import Log def addNewSite(self, site, stype, cache, path, diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 0e61cf7..b131c13 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -14,7 +14,6 @@ from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.checkfqdn import check_fqdn_ip from wo.core.cron import WOCron -from wo.core.domainvalidate import GetDomainlevel from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 695ee1d..0fd60ee 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -2,6 +2,7 @@ import os from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.core.aptget import WOAptGet from wo.core.logging import Log from wo.core.services import WOService diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index ebe8284..1f6e2b2 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -3,6 +3,7 @@ import shutil from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.cli.plugins.stack_pref import post_pref, pre_pref from wo.core.aptget import WOAptGet from wo.core.download import WODownload diff --git a/wo/cli/plugins/sync.py b/wo/cli/plugins/sync.py index 5039a03..b2f1d91 100644 --- a/wo/cli/plugins/sync.py +++ b/wo/cli/plugins/sync.py @@ -2,6 +2,7 @@ import glob from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.cli.plugins.sitedb import getAllsites, updateSiteInfo from wo.core.fileutils import WOFileUtils from wo.core.logging import Log diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index d9f557d..0372853 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -3,6 +3,7 @@ import time from cement.core import handler, hook from cement.core.controller import CementBaseController, expose + from wo.core.download import WODownload from wo.core.logging import Log diff --git a/wo/core/addswap.py b/wo/core/addswap.py index ad0c14c..04dcfd1 100644 --- a/wo/core/addswap.py +++ b/wo/core/addswap.py @@ -1,11 +1,13 @@ """WordOps Swap Creation""" -from wo.core.shellexec import WOShellExec -from wo.core.fileutils import WOFileUtils -from wo.core.aptget import WOAptGet -from wo.core.logging import Log import os + import psutil +from wo.core.aptget import WOAptGet +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.shellexec import WOShellExec + class WOSwap(): """Manage Swap""" diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index 44f53df..1715639 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -1,8 +1,9 @@ """WordOps packages repository operations""" +import os + +from wo.core.logging import Log from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables -from wo.core.logging import Log -import os class WORepo(): diff --git a/wo/core/aptget.py b/wo/core/aptget.py index 3e3fe53..3bff57e 100644 --- a/wo/core/aptget.py +++ b/wo/core/aptget.py @@ -1,11 +1,12 @@ """WordOps package installation using apt-get module.""" -import apt -import sys import subprocess -from wo.core.logging import Log +import sys + +from sh import ErrorReturnCode, apt_get + +import apt from wo.core.apt_repo import WORepo -from sh import apt_get -from sh import ErrorReturnCode +from wo.core.logging import Log class WOAptGet(): diff --git a/wo/core/checkfqdn.py b/wo/core/checkfqdn.py index 933c020..8b240f8 100644 --- a/wo/core/checkfqdn.py +++ b/wo/core/checkfqdn.py @@ -1,6 +1,7 @@ +import requests + from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables -import requests def check_fqdn(self, wo_host): diff --git a/wo/core/cron.py b/wo/core/cron.py index fd4ec70..adfabf7 100644 --- a/wo/core/cron.py +++ b/wo/core/cron.py @@ -1,5 +1,6 @@ -from wo.core.shellexec import WOShellExec from wo.core.logging import Log +from wo.core.shellexec import WOShellExec + """ Set CRON on LINUX system. diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index 667fc21..a787480 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -3,49 +3,50 @@ import os from urllib.parse import urlparse -def ValidateDomain(url): - """ - This function returns domain name removing http:// and https:// - returns domain name only with or without www as user provided. - """ +class DOMN: - # Check if http:// or https:// present remove it if present - domain_name = url.split('/') - if 'http:' in domain_name or 'https:' in domain_name: - domain_name = domain_name[2] - else: - domain_name = domain_name[0] - www_domain_name = domain_name.split('.') - final_domain = '' - if www_domain_name[0] == 'www': - final_domain = '.'.join(www_domain_name[1:]) - else: - final_domain = domain_name + def validatedomain(self, url): + """ + This function returns domain name removing http:// and https:// + returns domain name only with or without www as user provided. + """ - return (final_domain, domain_name) + # Check if http:// or https:// present remove it if present + domain_name = url.split('/') + if 'http:' in domain_name or 'https:' in domain_name: + domain_name = domain_name[2] + else: + domain_name = domain_name[0] + www_domain_name = domain_name.split('.') + final_domain = '' + if www_domain_name[0] == 'www': + final_domain = '.'.join(www_domain_name[1:]) + else: + final_domain = domain_name + return (final_domain, domain_name) -def GetDomainlevel(domain): - """ - This function returns the domain type : domain, subdomain, - """ - domain_name = domain.lower().strip().split('.') - if domain_name[0] == 'www': - domain_name = domain_name[1:] - domain_type = '' - if os.path.isfile("/var/lib/wo/public_suffix_list.dat"): - # Read mode opens a file for reading only. - suffix_file = open( - "/var/lib/wo/public_suffix_list.dat", encoding='utf-8', ) - # Read all the lines into a list. - for domain_suffix in suffix_file: - if (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): - domain_type = 'domain' - root_domain = ('.'.join(domain_name[0:])) - break - else: - domain_type = 'subdomain' - root_domain = ('.'.join(domain_name[1:])) - suffix_file.close() + def getdomainlevel(self, domain): + """ + This function returns the domain type : domain, subdomain, + """ + domain_name = domain.lower().strip().split('.') + if domain_name[0] == 'www': + domain_name = domain_name[1:] + domain_type = '' + if os.path.isfile("/var/lib/wo/public_suffix_list.dat"): + # Read mode opens a file for reading only. + suffix_file = open( + "/var/lib/wo/public_suffix_list.dat", encoding='utf-8', ) + # Read all the lines into a list. + for domain_suffix in suffix_file: + if (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])): + domain_type = 'domain' + root_domain = ('.'.join(domain_name[0:])) + break + else: + domain_type = 'subdomain' + root_domain = ('.'.join(domain_name[1:])) + suffix_file.close() - return (domain_type, root_domain) + return (domain_type, root_domain) diff --git a/wo/core/download.py b/wo/core/download.py index d81b277..53abbfd 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -1,7 +1,8 @@ """WordOps download core classes.""" -import urllib.request -import urllib.error import os +import urllib.error +import urllib.request + from wo.core.logging import Log diff --git a/wo/core/extract.py b/wo/core/extract.py index 0564b9b..2effab0 100644 --- a/wo/core/extract.py +++ b/wo/core/extract.py @@ -1,6 +1,7 @@ """WordOps Extract Core """ -import tarfile import os +import tarfile + from wo.core.logging import Log diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index cf44ab3..0cf8bcb 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -1,8 +1,9 @@ """WordOps file utils core classes.""" -import shutil +import fileinput import os import pwd -import fileinput +import shutil + from wo.core.logging import Log diff --git a/wo/core/git.py b/wo/core/git.py index 32f919e..96a9638 100644 --- a/wo/core/git.py +++ b/wo/core/git.py @@ -1,8 +1,10 @@ """WordOps GIT module""" -from sh import git, ErrorReturnCode -from wo.core.logging import Log import os +from sh import ErrorReturnCode, git + +from wo.core.logging import Log + class WOGit: """Intialization of core variables""" diff --git a/wo/core/logwatch.py b/wo/core/logwatch.py index 6440bdd..0d51440 100644 --- a/wo/core/logwatch.py +++ b/wo/core/logwatch.py @@ -3,10 +3,11 @@ Real time log files watcher supporting log rotation. """ -import os -import time import errno +import os import stat +import time + from wo.core.logging import Log diff --git a/wo/core/mysql.py b/wo/core/mysql.py index e168e85..2e962b6 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -1,8 +1,10 @@ """WordOps MySQL core classes.""" -import pymysql -from pymysql import connections, DatabaseError, Error -from os.path import expanduser import os +from os.path import expanduser + +import pymysql +from pymysql import DatabaseError, Error, connections + from wo.core.logging import Log from wo.core.variables import WOVariables diff --git a/wo/core/nginxhashbucket.py b/wo/core/nginxhashbucket.py index 921fe1f..9145812 100644 --- a/wo/core/nginxhashbucket.py +++ b/wo/core/nginxhashbucket.py @@ -1,10 +1,11 @@ """WordOps Hash Bucket Calculator""" -from wo.core.fileutils import WOFileUtils +import fileinput import math import os -import fileinput import subprocess +from wo.core.fileutils import WOFileUtils + def hashbucket(self): # Check Nginx Hashbucket error diff --git a/wo/core/sendmail.py b/wo/core/sendmail.py index 59c330f..9a60602 100644 --- a/wo/core/sendmail.py +++ b/wo/core/sendmail.py @@ -1,10 +1,10 @@ -import smtplib import os -from email.mime.multipart import MIMEMultipart +import smtplib +from email import encoders from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate -from email import encoders def WOSendMail(send_from, send_to, subject, text, files, server="localhost", diff --git a/wo/core/services.py b/wo/core/services.py index 350a84a..5d4a1bd 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -1,5 +1,6 @@ """WordOps Service Manager""" import subprocess + from wo.core.logging import Log diff --git a/wo/core/shellexec.py b/wo/core/shellexec.py index 9cfd191..979affb 100644 --- a/wo/core/shellexec.py +++ b/wo/core/shellexec.py @@ -1,7 +1,8 @@ """WordOps Shell Functions""" -from wo.core.logging import Log import subprocess +from wo.core.logging import Log + class CommandExecutionError(Exception): """custom Exception for command execution""" diff --git a/wo/core/template.py b/wo/core/template.py index a4057d9..0d50224 100644 --- a/wo/core/template.py +++ b/wo/core/template.py @@ -1,6 +1,8 @@ -from wo.core.logging import Log import os +from wo.core.logging import Log + + """ Render Templates """ diff --git a/wo/core/variables.py b/wo/core/variables.py index 96563b8..10349b5 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -1,9 +1,10 @@ """WordOps core variable module""" -import distro -import socket import configparser -import os import datetime +import os +import socket + +import distro class WOVariables(): From bb334c09fdd4d458c451c016b5402b3286563048 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 20:39:27 +0200 Subject: [PATCH 21/22] Fix missing () --- wo/core/domainvalidate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index a787480..f8dc5c3 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -3,7 +3,7 @@ import os from urllib.parse import urlparse -class DOMN: +class DOMN(): def validatedomain(self, url): """ From 278a4f956356a055601ba60dea66f22fa98537e8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 4 Sep 2019 20:41:23 +0200 Subject: [PATCH 22/22] Fix typo --- wo/core/domainvalidate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index f8dc5c3..ff7ddd3 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -3,7 +3,7 @@ import os from urllib.parse import urlparse -class DOMN(): +class DMN(): def validatedomain(self, url): """