From ccba38d1c8da3c2a97c5d57fc7181994335be05d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 00:20:09 +0200 Subject: [PATCH 001/102] Simplify letsencrypt --- wo/cli/plugins/site_functions.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 4181afe..032ae6f 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -39,7 +39,7 @@ def pre_run_checks(self): Log.debug(self, "checking NGINX configuration ...") FNULL = open('/dev/null', 'w') subprocess.check_call(["/usr/sbin/nginx", "-t"], stdout=FNULL, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT) except CalledProcessError as e: Log.debug(self, "{0}".format(str(e))) raise SiteError("nginx configuration check failed.") @@ -90,7 +90,7 @@ def setupdomain(self, data): Log.debug(self, "Checking generated nginx conf, please wait...") FNULL = open('/dev/null', 'w') subprocess.check_call(["/usr/sbin/nginx", "-t"], stdout=FNULL, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT) Log.info(self, "[" + Log.ENDC + "Done" + Log.OKBLUE + "]") except CalledProcessError as e: Log.debug(self, "{0}".format(str(e))) @@ -1382,6 +1382,8 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, else: keylenght = "{0}".format(self.app.config.get('letsencrypt', 'keylength')) + wo_acme_exec = ("/etc/letsencrypt/acme.sh --config-home " + "'/etc/letsencrypt/config'") if wo_dns: acme_mode = "--dns {0}".format(wo_acme_dns) validation_mode = "DNS with {0}".format(wo_acme_dns) @@ -1394,9 +1396,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, if subdomain: Log.info(self, "Issuing subdomain SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} {1} " "-k {2} -f" @@ -1406,9 +1406,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, elif wildcard: Log.info(self, "Issuing Wildcard SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d *.{0} --dns {1} " "-k {2} -f" @@ -1418,9 +1416,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, else: Log.info(self, "Issuing domain SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d www.{0} {1} " "-k {2} -f" From 69b66e941e0567d7cac78817d1ba50b148810410 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 00:23:04 +0200 Subject: [PATCH 002/102] simplify travis --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 593ae87..ae182c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,9 @@ before_install: - rm -rf ~/.gnupg before_script: - - sudo rm -rf /etc/mysql/ + - sudo rm -rf /etc/mysql - sudo bash -c 'echo example.com > /etc/hostname' - - sudo apt-get -qq purge mysql* graphviz* redis* - - sudo apt-get -qq autoremove --purge + - sudo apt-get -qq autoremove mysql* graphviz* redis* --purge script: - lsb_release -a From 2090336ddebf0e6214e91d06ad22f0b6b197bcaf Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:27:27 +0200 Subject: [PATCH 003/102] Add package configuration with stack upgrade --- install | 4 ++-- wo/cli/plugins/stack_upgrade.py | 29 +++++++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/install b/install index d0d2de2..08452d3 100755 --- a/install +++ b/install @@ -147,9 +147,9 @@ if [ -z "$wo_force_install" ]; then wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." exit 100 else - check_wo_linux_distro=$(lsb_release -sc | grep -E "trusty|xenial|bionic|disco|jessie|stretch|buster") + check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster") if [ -z "$check_wo_linux_distro" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04/19.04 LTS, Debian 8.x/9.x/10.x and Raspbian 9.x" + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/19.04 LTS, Debian 9.x/10.x and Raspbian 9.x" exit 100 fi fi diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0344c09..5af52b8 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -1,17 +1,19 @@ -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.logging import Log -from wo.core.variables import WOVariables -from wo.core.aptget import WOAptGet -from wo.core.apt_repo import WORepo -from wo.core.services import WOService -from wo.core.fileutils import WOFileUtils -from wo.core.shellexec import WOShellExec -from wo.core.extract import WOExtract -from wo.core.download import WODownload import os import shutil +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.cli.plugins.stack import WOStackController +from wo.core.apt_repo import WORepo +from wo.core.aptget import WOAptGet +from wo.core.download import WODownload +from wo.core.extract import WOExtract +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.services import WOService +from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables + class WOStackUpgradeController(CementBaseController): class Meta: @@ -83,6 +85,9 @@ class WOStackUpgradeController(CementBaseController): # All package update apt_packages = [] packages = [] + empty_packages = [] + stack = WOStackController() + stack.app = self.app if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and (not self.app.pargs.php) and (not self.app.pargs.mysql) and @@ -187,7 +192,7 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) - + stack.post_pref(apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') From 8e0384d7e111b5ffecf9c212d1ee2fcbd152290d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:28:37 +0200 Subject: [PATCH 004/102] update travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ae182c5..3e9437a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,8 @@ script: - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf + - sudo wo stack upgrade --nginx + - sudo wo stack upgrade --php - sudo wo stack upgrade --netdata --no-prompt - sudo wo stack upgrade --phpmyadmin --no-prompt - sudo wo stack upgrade --composer --no-prompt From cd62916cfca8d8c935d84c44e3e6d7dc8c920b10 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:56:33 +0200 Subject: [PATCH 005/102] Add stack_pref --- wo/cli/plugins/stack.py | 1336 +----------------------------- wo/cli/plugins/stack_pref.py | 1357 +++++++++++++++++++++++++++++++ wo/cli/plugins/stack_upgrade.py | 8 +- 3 files changed, 1364 insertions(+), 1337 deletions(-) create mode 100644 wo/cli/plugins/stack_pref.py diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index a1ccaf0..563f6b9 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1,7 +1,7 @@ """Stack Plugin for WordOps""" -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose +from cement.core import handler, hook import codecs import configparser @@ -20,6 +20,7 @@ from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_migrate import WOStackMigrateController from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController +from wo.cli.plugins.stack_pref import pre_pref, post_pref from wo.core.addswap import WOSwap from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet @@ -93,1335 +94,6 @@ class WOStackController(CementBaseController): """default action of wo stack command""" self.app.args.print_help() - @expose(hide=True) - def pre_pref(self, apt_packages): - """Pre settings to do before installation packages""" - - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - # add mariadb repository excepted on raspbian and ubuntu 19.04 - if (not WOVariables.wo_distro == 'raspbian'): - Log.info(self, "Adding repository for MySQL, please wait...") - mysql_pref = ("Package: *\nPin: origin " - "sfo1.mirrors.digitalocean.com" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'MariaDB.pref', 'w') as mysql_pref_file: - mysql_pref_file.write(mysql_pref) - WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVariables.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - WORepo.add_key(self, '0xF1656F24C74CD1D8', - keyserver="keyserver.ubuntu.com") - # generate random 24 characters root password - chars = ''.join(random.sample(string.ascii_letters, 24)) - # configure MySQL non-interactive install - if (not WOVariables.wo_distro == 'raspbian'): - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - else: - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") - # generate my.cnf root credentials - mysql_config = """ - [client] - user = root - password = {chars} - """.format(chars=chars) - config = configparser.ConfigParser() - config.read_string(mysql_config) - Log.debug(self, 'Writting configuration into MySQL file') - conf_path = "/etc/mysql/conf.d/my.cnf" - os.makedirs(os.path.dirname(conf_path), exist_ok=True) - with open(conf_path, encoding='utf-8', - mode='w') as configfile: - config.write(configfile) - Log.debug(self, 'Setting my.cnf permission') - WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) - - # add nginx repository - if set(WOVariables.wo_nginx).issubset(set(apt_packages)): - if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for NGINX, please wait...") - WORepo.add(self, ppa=WOVariables.wo_nginx_repo) - Log.debug(self, 'Adding ppa for Nginx') - else: - Log.info(self, "Adding repository for NGINX, please wait...") - WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) - Log.debug(self, 'Adding repository for Nginx') - WORepo.add_key(self, WOVariables.wo_nginx_key) - - # add php repository - if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or - set(WOVariables.wo_php).issubset(set(apt_packages))): - if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for PHP, please wait...") - Log.debug(self, 'Adding ppa for PHP') - WORepo.add(self, ppa=WOVariables.wo_php_repo) - else: - Log.info(self, "Adding repository for PHP, please wait...") - # Add repository for php - if (WOVariables.wo_platform_codename == 'buster'): - php_pref = ("Package: *\nPin: origin " - "packages.sury.org" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'PHP.pref', 'w') as php_pref_file: - php_pref_file.write(php_pref) - Log.debug(self, 'Adding repo_url of php for debian') - WORepo.add(self, repo_url=WOVariables.wo_php_repo) - Log.debug(self, 'Adding deb.sury GPG key') - WORepo.add_key(self, WOVariables.wo_php_key) - # add redis repository - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - Log.info(self, "Adding repository for Redis, please wait...") - if WOVariables.wo_distro == 'ubuntu': - Log.debug(self, 'Adding ppa for redis') - WORepo.add(self, ppa=WOVariables.wo_redis_repo) - else: - Log.debug(self, 'Adding repo_url of redis for debian') - - @expose(hide=True) - def post_pref(self, apt_packages, packages): - """Post activity after installation of packages""" - if (apt_packages): - - # Nginx configuration - if set(WOVariables.wo_nginx).issubset(set(apt_packages)): - - # Fix for white screen death with NGINX PLUS - if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', - 'SCRIPT_FILENAME'): - with open('/etc/nginx/fastcgi_params', - encoding='utf-8', mode='a') as wo_nginx: - wo_nginx.write('fastcgi_param \tSCRIPT_FILENAME ' - '\t$request_filename;\n') - - if os.path.isfile('/etc/nginx/nginx.conf'): - data = dict(php="9000", debug="9001", - php7="9070", debug7="9170") - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/upstream.conf') - wo_nginx = open('/etc/nginx/conf.d/upstream.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'upstream.mustache', out=wo_nginx) - wo_nginx.close() - - data = dict(phpconf=True if - WOAptGet.is_installed(self, 'php7.2-fpm') - else False) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/stub_status.conf') - wo_nginx = open('/etc/nginx/conf.d/stub_status.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'stub_status.mustache', out=wo_nginx) - wo_nginx.close() - - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/webp.conf') - wo_nginx = open('/etc/nginx/conf.d/webp.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'webp.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/' - 'map-wp-fastcgi-cache.conf') - wo_nginx = open('/etc/nginx/conf.d/' - 'map-wp-fastcgi-cache.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'map-wp.mustache', - out=wo_nginx) - wo_nginx.close() - - # Setup Nginx common directory - if not os.path.exists('/etc/nginx/common'): - Log.debug(self, 'Creating directory' - '/etc/nginx/common') - os.makedirs('/etc/nginx/common') - - if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) - - # Common Configuration - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsubdir.conf') - wo_nginx = open('/etc/nginx/common/wpsubdir.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsubdir.mustache', - out=wo_nginx) - wo_nginx.close() - - # PHP 7.2 conf - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php72.conf') - wo_nginx = open('/etc/nginx/common/php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php72.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc.mustache', - out=wo_nginx) - wo_nginx.close() - - # PHP 7.3 conf - if os.path.isdir("/etc/nginx/common"): - data = dict() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php73.conf') - wo_nginx = open('/etc/nginx/common/php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php73.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - # create redis conf - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - with open("/etc/nginx/common/release", - "a") as release_file: - release_file.write("v{0}" - .format(WOVariables.wo_version)) - release_file.close() - - # Following files should not be overwrited - - if not os.path.isfile('/etc/nginx/common/acl.conf'): - data = dict(webroot=WOVariables.wo_webroot) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/acl.conf') - wo_nginx = open('/etc/nginx/common/acl.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'acl.mustache', - out=wo_nginx) - wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/blockips.conf') - wo_nginx = open('/etc/nginx/conf.d/blockips.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'blockips.mustache', out=wo_nginx) - wo_nginx.close() - - if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/fastcgi.conf') - wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'fastcgi.mustache', out=wo_nginx) - wo_nginx.close() - - # add redis cache format if not already done - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d" - "/redis.conf")): - with open("/etc/nginx/conf.d/" - "redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - # Nginx-Plus does not have nginx - # package structure like this - # So creating directories - if not os.path.exists('/etc/nginx/sites-available'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-available') - - if not os.path.exists('/etc/nginx/sites-enabled'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-enabled') - - # 22222 port settings - if not os.path.isfile("/etc/nginx/sites-available/22222"): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/sites-available/' - '22222') - wo_nginx = open('/etc/nginx/sites-available/22222', - encoding='utf-8', mode='w') - self.app.render((data), '22222.mustache', - out=wo_nginx) - wo_nginx.close() - - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) - try: - WOShellExec.cmd_exec(self, "printf \"WordOps:" - "$(openssl passwd -crypt " - "{password} 2> /dev/null)\n\"" - "> /etc/nginx/htpasswd-wo " - "2>/dev/null" - .format(password=passwd)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to save HTTP Auth") - - # Create Symbolic link for 22222 - WOFileUtils.create_symlink(self, ['/etc/nginx/' - 'sites-available/' - '22222', - '/etc/nginx/' - 'sites-enabled/' - '22222']) - # Create log and cert folder and softlinks - if not os.path.exists('{0}22222/logs' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/logs " - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/logs' - .format(WOVariables.wo_webroot)) - - if not os.path.exists('{0}22222/cert' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/cert" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/cert' - .format(WOVariables.wo_webroot)) - - if not os.path.exists('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/conf/nginx" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)) - - WOFileUtils.create_symlink(self, ['/var/log/nginx/' - '22222.access.log', - '{0}22222/' - 'logs/access.log' - .format(WOVariables.wo_webroot)] - ) - - WOFileUtils.create_symlink(self, ['/var/log/nginx/' - '22222.error.log', - '{0}22222/' - 'logs/error.log' - .format(WOVariables.wo_webroot)] - ) - - try: - WOShellExec.cmd_exec(self, "openssl genrsa -out " - "{0}22222/cert/22222.key 2048" - .format(WOVariables.wo_webroot)) - WOShellExec.cmd_exec(self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key {0}22222/cert/22222.key " - "-out {0}22222/cert/" - "22222.csr" - .format(WOVariables.wo_webroot)) - - WOFileUtils.mvfile(self, "{0}22222/cert/22222.key" - .format(WOVariables.wo_webroot), - "{0}22222/cert/" - "22222.key.org" - .format(WOVariables.wo_webroot)) - - WOShellExec.cmd_exec(self, "openssl rsa -in " - "{0}22222/cert/" - "22222.key.org -out " - "{0}22222/cert/22222.key" - .format(WOVariables.wo_webroot)) - - WOShellExec.cmd_exec(self, "openssl x509 -req -days " - "3652 -in {0}22222/cert/" - "22222.csr -signkey {0}" - "22222/cert/22222.key -out " - "{0}22222/cert/22222.crt" - .format(WOVariables.wo_webroot)) - - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate HTTPS " - "certificate for 22222") - server_ip = requests.get('http://v4.wordops.eu') - - if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' - .format(WOVariables.wo_webroot)): - - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "a") as php_file: - php_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") - - # Nginx Configation into GIT - WOGit.add(self, - ["/etc/nginx"], msg="Adding Nginx into Git") - WOService.reload_service(self, 'nginx') - - if set(["nginx"]).issubset(set(apt_packages)): - - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) - else: - WOService.restart_service(self, 'nginx') - - # create nginx configuration for redis - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - if os.path.isdir('/etc/nginx/common'): - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}\n") - - if os.path.isfile("/etc/nginx/nginx.conf"): - if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): - with open("/etc/nginx/conf.d/redis.conf", - "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local]" - " '\n '$http_host" - " \"$request\" " - "$status $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - if set(WOVariables.wo_php).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.2/'): - Log.debug(self, 'Creating directory /var/log/php/7.2/') - os.makedirs('/var/log/php/7.2/') - - # Parse etc/php/7.2/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file " - "/etc/php/7.2/fpm/php.ini") - config.read('/etc/php/7.2/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.2/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.2/fpm/php.ini") - config.write(configfile) - - # Parse /etc/php/7.2/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.2-fpm.pid", - error_log="/var/log/php/7.2/fpm.log", - include="/etc/php/7.2/fpm/pool.d/*.conf") - Log.debug(self, "writting php7.2 configuration into " - "/etc/php/7.2/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() - - # Parse /etc/php/7.2/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.2/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php72-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writing PHP 7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www.conf") - config.write(configfile) - - with open("/etc/php/7.2/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") - - # Generate /etc/php/7.2/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php72-two-fpm.sock' - with open('/etc/php/7.2/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www-two.conf") - config.write(confifile) - - # Generate /etc/php/7.2/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9172' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.2/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/debug.conf") - config.write(confifile) - - with open("/etc/php/7.2/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") - - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.2/mods-available/" - "xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.2/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") - - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") - - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) - - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.2-fpm') - - # PHP7.3 configuration - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.3/'): - Log.debug(self, 'Creating directory /var/log/php/7.3/') - os.makedirs('/var/log/php/7.3/') - - # Parse etc/php/7.3/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file /etc/php/7.3/" - "fpm/php.ini") - config.read('/etc/php/7.3/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.3/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.3/fpm/php.ini") - config.write(configfile) - - # Parse /etc/php/7.3/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.3-fpm.pid", - error_log="/var/log/php7.3-fpm.log", - include="/etc/php/7.3/fpm/pool.d/*.conf") - Log.debug(self, "writting php 7.3 configuration into " - "/etc/php/7.3/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() - - # Parse /etc/php/7.3/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.3/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php73-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www.conf") - config.write(configfile) - - with open("/etc/php/7.3/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") - - # Generate /etc/php/7.3/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php73-two-fpm.sock' - with open('/etc/php/7.3/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www-two.conf") - config.write(confifile) - - # Generate /etc/php/7.3/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9173' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.3/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/debug.conf") - config.write(confifile) - - with open("/etc/php/7.3/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") - - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.3/mods-available" - "/xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.3/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") - - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") - - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) - - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.3-fpm') - - # create mysql config if it doesn't exist - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - if not os.path.isfile("/etc/mysql/my.cnf"): - config = ("[mysqld]\nwait_timeout = 30\n" - "interactive_timeout=60\nperformance_schema = 0" - "\nquery_cache_type = 1") - config_file = open("/etc/mysql/my.cnf", - encoding='utf-8', mode='w') - config_file.write(config) - config_file.close() - - WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) - WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' - '> /dev/null 2>&1', - comment='MySQL optimization cronjob ' - 'added by WordOps') - WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - WOService.reload_service(self, 'mysql') - - # create fail2ban configuration files - if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): - if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - data = dict() - Log.debug(self, "Setting up fail2ban jails configuration") - fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban.mustache', - out=fail2ban_config) - fail2ban_config.close() - - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'wo-wordpress.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-wp.mustache', - out=fail2ban_config) - fail2ban_config.close() - - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'nginx-forbidden.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-forbidden.mustache', - out=fail2ban_config) - fail2ban_config.close() - WOGit.add(self, ["/etc/fail2ban"], - msg="Adding Fail2ban into Git") - WOService.reload_service(self, 'fail2ban') - - # Proftpd configuration - if set(["proftpd-basic"]).issubset(set(apt_packages)): - if os.path.isfile("/etc/proftpd/proftpd.conf"): - Log.debug(self, "Setting up Proftpd configuration") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# DefaultRoot", - "DefaultRoot") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# RequireValidShell", - "RequireValidShell") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# PassivePorts " - " " - "49152 65534", - "PassivePorts " - " " - " 49000 50000") - # proftpd TLS configuration - if not os.path.isdir("/etc/proftpd/ssl"): - WOFileUtils.mkdir(self, "/etc/proftpd/ssl") - - try: - WOShellExec.cmd_exec(self, "openssl genrsa -out " - "/etc/proftpd/ssl/proftpd.key 2048") - WOShellExec.cmd_exec(self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key /etc/proftpd/ssl/proftpd.key " - "-out /etc/proftpd/ssl/proftpd.csr") - WOFileUtils.mvfile(self, "/etc/proftpd/ssl/proftpd.key", - "/etc/proftpd/ssl/proftpd.key.org") - WOShellExec.cmd_exec(self, "openssl rsa -in " - "/etc/proftpd/ssl/proftpd.key.org " - "-out /etc/proftpd/ssl/proftpd.key") - WOShellExec.cmd_exec(self, "openssl x509 -req -days " - "3652 -in /etc/proftpd/ssl/proftpd.csr " - "-signkey /etc/proftpd/ssl/proftpd.key " - " -out /etc/proftpd/ssl/proftpd.crt") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate SSL " - "certificate for Proftpd") - WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.key", 0o700) - WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.crt", 0o700) - data = dict() - Log.debug(self, 'Writting the proftpd configuration to ' - 'file /etc/proftpd/tls.conf') - wo_proftpdconf = open('/etc/proftpd/tls.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'proftpd-tls.mustache', - out=wo_proftpdconf) - wo_proftpdconf.close() - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "#Include /etc/proftpd/tls.conf", - "Include /etc/proftpd/tls.conf") - WOService.restart_service(self, 'proftpd') - - # add rule for proftpd with UFW - if WOAptGet.is_installed(self, 'ufw'): - try: - WOShellExec.cmd_exec(self, "ufw allow " - "49000:50000/tcp") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Unable to add UFW rule") - - if os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - with open("/etc/fail2ban/jail.d/custom.conf", - encoding='utf-8', mode='a') as f2bproftpd: - f2bproftpd.write("\n\n[proftpd]\nenabled = true\n") - WOService.reload_service(self, 'fail2ban') - - WOGit.add(self, ["/etc/proftpd"], - msg="Adding ProFTPd into Git") - WOService.reload_service(self, 'proftpd') - - if (packages): - if any('/usr/local/bin/wp' == x[1] for x in packages): - Log.debug(self, "Setting Privileges" - " to /usr/local/bin/wp file ") - WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - - if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] - for x in packages): - WOExtract.extract( - self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') - Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' - 'location /var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/db" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', - '{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)) - shutil.copyfile('{0}22222/htdocs/db/pma' - '/config.sample.inc.php' - .format(WOVariables.wo_webroot), - '{0}22222/htdocs/db/pma/config.inc.php' - .format(WOVariables.wo_webroot)) - Log.debug(self, 'Setting Blowfish Secret Key ' - 'FOR COOKIE AUTH to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - blowfish_key = ''.join([random.choice - (string.ascii_letters + - string.digits) - for n in range(25)]) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'blowfish_secret\']" - " = \'\';", - "$cfg[\'blowfish_secret\']" - " = \'{0}\';" - .format(blowfish_key)) - Log.debug(self, 'Setting HOST Server For Mysql to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'Servers\'][$i][\'host\']" - " = \'localhost\';", "$cfg" - "[\'Servers\'][$i][\'host\'] = \'{0}\';" - .format(WOVariables.wo_mysql_host)) - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # composer install and phpmyadmin update - if any('/var/lib/wo/tmp/composer-install' == x[1] - for x in packages): - Log.info(self, "Installing composer, please wait...") - WOShellExec.cmd_exec(self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") - shutil.copyfile('/var/lib/wo/tmp/composer.phar', - '/usr/local/bin/composer') - WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " - "update -n --no-dev -d " - "/var/www/22222/htdocs/db/pma/") - - # netdata install - if any('/var/lib/wo/tmp/kickstart.sh' == x[1] - for x in packages): - if ((not os.path.exists('/opt/netdata')) and - (not os.path.exists('/etc/netdata'))): - Log.info(self, "Installing Netdata, please wait...") - WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" - "kickstart.sh " - "--dont-wait") - # disable mail notifications - WOFileUtils.searchreplace(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - 'SEND_EMAIL="YES"', - 'SEND_EMAIL="NO"') - # make changes persistant - WOFileUtils.copyfile(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - "/opt/netdata/etc/netdata/" - "health_alarm_notify.conf") - # check if mysql credentials are available - if os.path.isfile('/etc/mysql/conf.d/my.cnf'): - try: - WOMysql.execute(self, - "create user " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "grant usage on *.* to " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "flush privileges;", - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.info( - self, "fail to setup mysql user for netdata") - WOService.restart_service(self, 'netdata') - - # WordOps Dashboard - if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] - for x in packages): - if not os.path.isfile('{0}22222/htdocs/index.php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting wo-dashboard.tar.gz " - "to location {0}22222/htdocs/" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/' - 'wo-dashboard.tar.gz', - '{0}22222/htdocs' - .format(WOVariables.wo_webroot)) - wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " - "grep -oP \"dev [^[:space:]]+ \" " - "| cut -d ' ' -f 2").read() - if (wo_wan != 'eth0' and wo_wan != ''): - WOFileUtils.searchreplace(self, - "{0}22222/htdocs/index.php" - .format(WOVariables.wo_webroot), - "eth0", - "{0}".format(wo_wan)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # Extplorer FileManager - if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting explorer.tar.gz " - "to location {0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', - '/var/lib/wo/tmp/') - shutil.move('/var/lib/wo/tmp/extplorer-{0}' - .format(WOVariables.wo_extplorer), - '{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # webgrind - if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file webgrind.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/webgrind.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directroy " - "{0}22222/htdocs/php" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/webgrind-master/', - '{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "/usr/local/bin/dot", "/usr/bin/dot") - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "Europe/Copenhagen", - WOVariables.wo_timezone) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "90", "100") - - Log.debug(self, "Setting Privileges of webroot permission to " - "{0}22222/htdocs/php/webgrind/ file " - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - # anemometer - if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file anemometer.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/anemometer.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory") - os.makedirs('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/Anemometer-master', - '{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)) - chars = ''.join(random.sample(string.ascii_letters, 8)) - try: - WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' - '/anemometer/install.sql' - .format(WOVariables.wo_webroot)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("Unable to import Anemometer database") - - WOMysql.execute(self, 'grant select on' - ' *.* to \'anemometer\'' - '@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get - ('mysql', - 'grant-host'), - chars)) - Log.debug(self, "grant all on slow-query-log.*" - " to anemometer@root_user" - " IDENTIFIED BY password ") - WOMysql.execute(self, 'grant all on slow_query_log.* to' - '\'anemometer\'@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get( - 'mysql', 'grant-host'), - chars), - errormsg="cannot grant priviledges", - log=False) - - # Custom Anemometer configuration - Log.debug(self, "configration Anemometer") - data = dict(host=WOVariables.wo_mysql_host, port='3306', - user='anemometer', password=chars) - wo_anemometer = open('{0}22222/htdocs/db/anemometer' - '/conf/config.inc.php' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') - self.app.render((data), 'anemometer.mustache', - out=wo_anemometer) - wo_anemometer.close() - - if any('/usr/bin/pt-query-advisor' == x[1] - for x in packages): - WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) - - # phpredisadmin - if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/cache/' - 'redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/cache/redis" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - if os.path.isfile("/usr/local/bin/composer"): - WOShellExec.cmd_exec(self, "sudo -u www-data -H " - "composer " - "create-project -n -s dev " - "erik-dubbelboer/php-redis-admin " - "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin ") - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/cache/file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - @expose(help="Install packages") def install(self, packages=[], apt_packages=[], disp_msg=True): """Start installation of packages""" @@ -1719,7 +391,7 @@ class WOStackController(CementBaseController): if (apt_packages) or (packages): Log.debug(self, "Calling pre_pref") - self.pre_pref(apt_packages) + pre_pref(self, apt_packages) if (apt_packages): meminfo = (os.popen('cat /proc/meminfo ' '| grep MemTotal').read()).split(":") @@ -1735,7 +407,7 @@ class WOStackController(CementBaseController): Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) Log.debug(self, "Calling post_pref") - self.post_pref(apt_packages, packages) + post_pref(self, apt_packages, packages) if 'redis-server' in apt_packages: # set redis.conf parameter # set maxmemory 10% for ram below 512MB and 20% for others diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py new file mode 100644 index 0000000..9b7a9ed --- /dev/null +++ b/wo/cli/plugins/stack_pref.py @@ -0,0 +1,1357 @@ +import codecs +import configparser +import os +import pwd +import random +import shutil +import string +import re +import requests + +import psutil +# from pynginxconfig import NginxConfig +from wo.cli.plugins.site_functions import * +from wo.cli.plugins.sitedb import * +from wo.cli.plugins.stack_services import WOStackStatusController +from wo.core.addswap import WOSwap +from wo.core.apt_repo import WORepo +from wo.core.aptget import WOAptGet +from wo.core.cron import WOCron +from wo.core.download import WODownload +from wo.core.extract import WOExtract +from wo.core.fileutils import WOFileUtils +from wo.core.git import WOGit +from wo.core.logging import Log +from wo.core.mysql import WOMysql +from wo.core.services import WOService +from wo.core.shellexec import CommandExecutionError, WOShellExec +from wo.core.variables import WOVariables + + +def pre_pref(self, apt_packages): + """Pre settings to do before installation packages""" + + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + # add mariadb repository excepted on raspbian and ubuntu 19.04 + if (not WOVariables.wo_distro == 'raspbian'): + Log.info(self, "Adding repository for MySQL, please wait...") + mysql_pref = ("Package: *\nPin: origin " + "sfo1.mirrors.digitalocean.com" + "\nPin-Priority: 1000\n") + with open('/etc/apt/preferences.d/' + 'MariaDB.pref', 'w') as mysql_pref_file: + mysql_pref_file.write(mysql_pref) + WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) + Log.debug(self, 'Adding key for {0}' + .format(WOVariables.wo_mysql_repo)) + WORepo.add_key(self, '0xcbcb082a1bb943db', + keyserver="keyserver.ubuntu.com") + WORepo.add_key(self, '0xF1656F24C74CD1D8', + keyserver="keyserver.ubuntu.com") + # generate random 24 characters root password + chars = ''.join(random.sample(string.ascii_letters, 24)) + # configure MySQL non-interactive install + if (not WOVariables.wo_distro == 'raspbian'): + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + else: + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to initialize MySQL package") + # generate my.cnf root credentials + mysql_config = """ + [client] + user = root + password = {chars} + """.format(chars=chars) + config = configparser.ConfigParser() + config.read_string(mysql_config) + Log.debug(self, 'Writting configuration into MySQL file') + conf_path = "/etc/mysql/conf.d/my.cnf" + os.makedirs(os.path.dirname(conf_path), exist_ok=True) + with open(conf_path, encoding='utf-8', + mode='w') as configfile: + config.write(configfile) + Log.debug(self, 'Setting my.cnf permission') + WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) + + # add nginx repository + if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + if (WOVariables.wo_distro == 'ubuntu'): + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, ppa=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding ppa for Nginx') + else: + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding repository for Nginx') + WORepo.add_key(self, WOVariables.wo_nginx_key) + + # add php repository + if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or + set(WOVariables.wo_php).issubset(set(apt_packages))): + if (WOVariables.wo_distro == 'ubuntu'): + Log.info(self, "Adding repository for PHP, please wait...") + Log.debug(self, 'Adding ppa for PHP') + WORepo.add(self, ppa=WOVariables.wo_php_repo) + else: + Log.info(self, "Adding repository for PHP, please wait...") + # Add repository for php + if (WOVariables.wo_platform_codename == 'buster'): + php_pref = ("Package: *\nPin: origin " + "packages.sury.org" + "\nPin-Priority: 1000\n") + with open('/etc/apt/preferences.d/' + 'PHP.pref', 'w') as php_pref_file: + php_pref_file.write(php_pref) + Log.debug(self, 'Adding repo_url of php for debian') + WORepo.add(self, repo_url=WOVariables.wo_php_repo) + Log.debug(self, 'Adding deb.sury GPG key') + WORepo.add_key(self, WOVariables.wo_php_key) + # add redis repository + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + Log.info(self, "Adding repository for Redis, please wait...") + if WOVariables.wo_distro == 'ubuntu': + Log.debug(self, 'Adding ppa for redis') + WORepo.add(self, ppa=WOVariables.wo_redis_repo) + else: + Log.debug(self, 'Adding repo_url of redis for debian') + + +def post_pref(self, apt_packages, packages): + """Post activity after installation of packages""" + if (apt_packages): + + # Nginx configuration + if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + + # Fix for white screen death with NGINX PLUS + if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', + 'SCRIPT_FILENAME'): + with open('/etc/nginx/fastcgi_params', + encoding='utf-8', mode='a') as wo_nginx: + wo_nginx.write('fastcgi_param \tSCRIPT_FILENAME ' + '\t$request_filename;\n') + + if os.path.isfile('/etc/nginx/nginx.conf'): + data = dict(php="9000", debug="9001", + php7="9070", debug7="9170") + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/upstream.conf') + wo_nginx = open('/etc/nginx/conf.d/upstream.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'upstream.mustache', out=wo_nginx) + wo_nginx.close() + + data = dict(phpconf=True if + WOAptGet.is_installed(self, 'php7.2-fpm') + else False) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/stub_status.conf') + wo_nginx = open('/etc/nginx/conf.d/stub_status.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'stub_status.mustache', out=wo_nginx) + wo_nginx.close() + + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/webp.conf') + wo_nginx = open('/etc/nginx/conf.d/webp.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'webp.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/' + 'map-wp-fastcgi-cache.conf') + wo_nginx = open('/etc/nginx/conf.d/' + 'map-wp-fastcgi-cache.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'map-wp.mustache', + out=wo_nginx) + wo_nginx.close() + + # Setup Nginx common directory + if not os.path.exists('/etc/nginx/common'): + Log.debug(self, 'Creating directory' + '/etc/nginx/common') + os.makedirs('/etc/nginx/common') + + if os.path.exists('/etc/nginx/common'): + data = dict(webroot=WOVariables.wo_webroot) + + # Common Configuration + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/locations-wo.conf') + wo_nginx = open('/etc/nginx/common/locations-wo.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'locations.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsubdir.conf') + wo_nginx = open('/etc/nginx/common/wpsubdir.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsubdir.mustache', + out=wo_nginx) + wo_nginx.close() + + # PHP 7.2 conf + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php72.conf') + wo_nginx = open('/etc/nginx/common/php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php72.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() + + # PHP 7.3 conf + if os.path.isdir("/etc/nginx/common"): + data = dict() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php73.conf') + wo_nginx = open('/etc/nginx/common/php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php73.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + # create redis conf + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php73.conf') + wo_nginx = open('/etc/nginx/common/redis-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + with open("/etc/nginx/common/release", + "a") as release_file: + release_file.write("v{0}" + .format(WOVariables.wo_version)) + release_file.close() + + # Following files should not be overwrited + + if not os.path.isfile('/etc/nginx/common/acl.conf'): + data = dict(webroot=WOVariables.wo_webroot) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/acl.conf') + wo_nginx = open('/etc/nginx/common/acl.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'acl.mustache', + out=wo_nginx) + wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/blockips.conf') + wo_nginx = open('/etc/nginx/conf.d/blockips.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'blockips.mustache', out=wo_nginx) + wo_nginx.close() + + if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/fastcgi.conf') + wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'fastcgi.mustache', out=wo_nginx) + wo_nginx.close() + + # add redis cache format if not already done + if (os.path.isfile("/etc/nginx/nginx.conf") and + not os.path.isfile("/etc/nginx/conf.d" + "/redis.conf")): + with open("/etc/nginx/conf.d/" + "redis.conf", "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local] '\n" + "'$http_host \"$request\" $status" + " $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") + + # Nginx-Plus does not have nginx + # package structure like this + # So creating directories + if not os.path.exists('/etc/nginx/sites-available'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-available') + + if not os.path.exists('/etc/nginx/sites-enabled'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-enabled') + + # 22222 port settings + if not os.path.isfile("/etc/nginx/sites-available/22222"): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/sites-available/' + '22222') + wo_nginx = open('/etc/nginx/sites-available/22222', + encoding='utf-8', mode='w') + self.app.render((data), '22222.mustache', + out=wo_nginx) + wo_nginx.close() + + passwd = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(16)]) + try: + WOShellExec.cmd_exec(self, "printf \"WordOps:" + "$(openssl passwd -crypt " + "{password} 2> /dev/null)\n\"" + "> /etc/nginx/htpasswd-wo " + "2>/dev/null" + .format(password=passwd)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to save HTTP Auth") + + # Create Symbolic link for 22222 + WOFileUtils.create_symlink(self, ['/etc/nginx/' + 'sites-available/' + '22222', + '/etc/nginx/' + 'sites-enabled/' + '22222']) + # Create log and cert folder and softlinks + if not os.path.exists('{0}22222/logs' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/logs " + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/logs' + .format(WOVariables.wo_webroot)) + + if not os.path.exists('{0}22222/cert' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/cert" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/cert' + .format(WOVariables.wo_webroot)) + + if not os.path.exists('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/conf/nginx" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)) + + WOFileUtils.create_symlink(self, ['/var/log/nginx/' + '22222.access.log', + '{0}22222/' + 'logs/access.log' + .format(WOVariables.wo_webroot)] + ) + + WOFileUtils.create_symlink(self, ['/var/log/nginx/' + '22222.error.log', + '{0}22222/' + 'logs/error.log' + .format(WOVariables.wo_webroot)] + ) + + try: + WOShellExec.cmd_exec(self, "openssl genrsa -out " + "{0}22222/cert/22222.key 2048" + .format(WOVariables.wo_webroot)) + WOShellExec.cmd_exec(self, "openssl req -new -batch " + "-subj /commonName=localhost/ " + "-key {0}22222/cert/22222.key " + "-out {0}22222/cert/" + "22222.csr" + .format(WOVariables.wo_webroot)) + + WOFileUtils.mvfile(self, "{0}22222/cert/22222.key" + .format(WOVariables.wo_webroot), + "{0}22222/cert/" + "22222.key.org" + .format(WOVariables.wo_webroot)) + + WOShellExec.cmd_exec(self, "openssl rsa -in " + "{0}22222/cert/" + "22222.key.org -out " + "{0}22222/cert/22222.key" + .format(WOVariables.wo_webroot)) + + WOShellExec.cmd_exec(self, "openssl x509 -req -days " + "3652 -in {0}22222/cert/" + "22222.csr -signkey {0}" + "22222/cert/22222.key -out " + "{0}22222/cert/22222.crt" + .format(WOVariables.wo_webroot)) + + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error( + self, "Failed to generate HTTPS " + "certificate for 22222") + server_ip = requests.get('http://v4.wordops.eu') + + if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' + .format(WOVariables.wo_webroot)): + + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "a") as php_file: + php_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") + + # Nginx Configation into GIT + WOGit.add(self, + ["/etc/nginx"], msg="Adding Nginx into Git") + WOService.reload_service(self, 'nginx') + + if set(["nginx"]).issubset(set(apt_packages)): + + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) + else: + WOService.restart_service(self, 'nginx') + + # create nginx configuration for redis + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if os.path.isdir('/etc/nginx/common'): + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + + if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): + if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" + "upstream.conf", + "redis"): + with open("/etc/nginx/conf.d/upstream.conf", + "a") as redis_file: + redis_file.write("upstream redis {\n" + " server 127.0.0.1:6379;\n" + " keepalive 10;\n}\n") + + if os.path.isfile("/etc/nginx/nginx.conf"): + if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): + with open("/etc/nginx/conf.d/redis.conf", + "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local]" + " '\n '$http_host" + " \"$request\" " + "$status $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") + + if set(WOVariables.wo_php).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.2/'): + Log.debug(self, 'Creating directory /var/log/php/7.2/') + os.makedirs('/var/log/php/7.2/') + + # Parse etc/php/7.2/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file " + "/etc/php/7.2/fpm/php.ini") + config.read('/etc/php/7.2/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.2/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.2/fpm/php.ini") + config.write(configfile) + + # Parse /etc/php/7.2/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.2-fpm.pid", + error_log="/var/log/php/7.2/fpm.log", + include="/etc/php/7.2/fpm/pool.d/*.conf") + Log.debug(self, "writting php7.2 configuration into " + "/etc/php/7.2/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() + + # Parse /etc/php/7.2/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.2/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php72-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writing PHP 7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www.conf") + config.write(configfile) + + with open("/etc/php/7.2/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") + + # Generate /etc/php/7.2/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php72-two-fpm.sock' + with open('/etc/php/7.2/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www-two.conf") + config.write(confifile) + + # Generate /etc/php/7.2/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9172' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.2/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/debug.conf") + config.write(confifile) + + with open("/etc/php/7.2/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") + + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.2/mods-available/" + "xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.2/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") + + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") + + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) + + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.2-fpm') + + # PHP7.3 configuration + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.3/'): + Log.debug(self, 'Creating directory /var/log/php/7.3/') + os.makedirs('/var/log/php/7.3/') + + # Parse etc/php/7.3/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file /etc/php/7.3/" + "fpm/php.ini") + config.read('/etc/php/7.3/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.3/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.3/fpm/php.ini") + config.write(configfile) + + # Parse /etc/php/7.3/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.3-fpm.pid", + error_log="/var/log/php7.3-fpm.log", + include="/etc/php/7.3/fpm/pool.d/*.conf") + Log.debug(self, "writting php 7.3 configuration into " + "/etc/php/7.3/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() + + # Parse /etc/php/7.3/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.3/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php73-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www.conf") + config.write(configfile) + + with open("/etc/php/7.3/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") + + # Generate /etc/php/7.3/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php73-two-fpm.sock' + with open('/etc/php/7.3/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www-two.conf") + config.write(confifile) + + # Generate /etc/php/7.3/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9173' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.3/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/debug.conf") + config.write(confifile) + + with open("/etc/php/7.3/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") + + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.3/mods-available" + "/xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.3/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") + + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") + + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) + + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.3-fpm') + + # create mysql config if it doesn't exist + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if not os.path.isfile("/etc/mysql/my.cnf"): + config = ("[mysqld]\nwait_timeout = 30\n" + "interactive_timeout=60\nperformance_schema = 0" + "\nquery_cache_type = 1") + config_file = open("/etc/mysql/my.cnf", + encoding='utf-8', mode='w') + config_file.write(config) + config_file.close() + + WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' + '> /dev/null 2>&1', + comment='MySQL optimization cronjob ' + 'added by WordOps') + WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") + WOService.reload_service(self, 'mysql') + + # create fail2ban configuration files + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=fail2ban_config) + fail2ban_config.close() + + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=fail2ban_config) + fail2ban_config.close() + + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'nginx-forbidden.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-forbidden.mustache', + out=fail2ban_config) + fail2ban_config.close() + WOGit.add(self, ["/etc/fail2ban"], + msg="Adding Fail2ban into Git") + WOService.reload_service(self, 'fail2ban') + + # Proftpd configuration + if set(["proftpd-basic"]).issubset(set(apt_packages)): + if os.path.isfile("/etc/proftpd/proftpd.conf"): + Log.debug(self, "Setting up Proftpd configuration") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# DefaultRoot", + "DefaultRoot") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# RequireValidShell", + "RequireValidShell") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# PassivePorts " + " " + "49152 65534", + "PassivePorts " + " " + " 49000 50000") + # proftpd TLS configuration + if not os.path.isdir("/etc/proftpd/ssl"): + WOFileUtils.mkdir(self, "/etc/proftpd/ssl") + + try: + WOShellExec.cmd_exec(self, "openssl genrsa -out " + "/etc/proftpd/ssl/proftpd.key 2048") + WOShellExec.cmd_exec(self, "openssl req -new -batch " + "-subj /commonName=localhost/ " + "-key /etc/proftpd/ssl/proftpd.key " + "-out /etc/proftpd/ssl/proftpd.csr") + WOFileUtils.mvfile(self, "/etc/proftpd/ssl/proftpd.key", + "/etc/proftpd/ssl/proftpd.key.org") + WOShellExec.cmd_exec(self, "openssl rsa -in " + "/etc/proftpd/ssl/proftpd.key.org " + "-out /etc/proftpd/ssl/proftpd.key") + WOShellExec.cmd_exec(self, "openssl x509 -req -days " + "3652 -in /etc/proftpd/ssl/proftpd.csr " + "-signkey /etc/proftpd/ssl/proftpd.key " + " -out /etc/proftpd/ssl/proftpd.crt") + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error( + self, "Failed to generate SSL " + "certificate for Proftpd") + WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.key", 0o700) + WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.crt", 0o700) + data = dict() + Log.debug(self, 'Writting the proftpd configuration to ' + 'file /etc/proftpd/tls.conf') + wo_proftpdconf = open('/etc/proftpd/tls.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'proftpd-tls.mustache', + out=wo_proftpdconf) + wo_proftpdconf.close() + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "#Include /etc/proftpd/tls.conf", + "Include /etc/proftpd/tls.conf") + WOService.restart_service(self, 'proftpd') + + # add rule for proftpd with UFW + if WOAptGet.is_installed(self, 'ufw'): + try: + WOShellExec.cmd_exec(self, "ufw allow " + "49000:50000/tcp") + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to add UFW rule") + + if os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + with open("/etc/fail2ban/jail.d/custom.conf", + encoding='utf-8', mode='a') as f2bproftpd: + f2bproftpd.write("\n\n[proftpd]\nenabled = true\n") + WOService.reload_service(self, 'fail2ban') + + WOGit.add(self, ["/etc/proftpd"], + msg="Adding ProFTPd into Git") + WOService.reload_service(self, 'proftpd') + + if (packages): + if any('/usr/local/bin/wp' == x[1] for x in packages): + Log.debug(self, "Setting Privileges" + " to /usr/local/bin/wp file ") + WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + + if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] + for x in packages): + WOExtract.extract( + self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' + 'location /var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/db" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', + '{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)) + shutil.copyfile('{0}22222/htdocs/db/pma' + '/config.sample.inc.php' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/db/pma/config.inc.php' + .format(WOVariables.wo_webroot)) + Log.debug(self, 'Setting Blowfish Secret Key ' + 'FOR COOKIE AUTH to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + blowfish_key = ''.join([random.choice + (string.ascii_letters + + string.digits) + for n in range(25)]) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'blowfish_secret\']" + " = \'\';", + "$cfg[\'blowfish_secret\']" + " = \'{0}\';" + .format(blowfish_key)) + Log.debug(self, 'Setting HOST Server For Mysql to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'Servers\'][$i][\'host\']" + " = \'localhost\';", "$cfg" + "[\'Servers\'][$i][\'host\'] = \'{0}\';" + .format(WOVariables.wo_mysql_host)) + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # composer install and phpmyadmin update + if any('/var/lib/wo/tmp/composer-install' == x[1] + for x in packages): + Log.info(self, "Installing composer, please wait...") + WOShellExec.cmd_exec(self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") + shutil.copyfile('/var/lib/wo/tmp/composer.phar', + '/usr/local/bin/composer') + WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) + Log.info(self, "Updating phpMyAdmin, please wait...") + WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " + "update -n --no-dev -d " + "/var/www/22222/htdocs/db/pma/") + + # netdata install + if any('/var/lib/wo/tmp/kickstart.sh' == x[1] + for x in packages): + if ((not os.path.exists('/opt/netdata')) and + (not os.path.exists('/etc/netdata'))): + Log.info(self, "Installing Netdata, please wait...") + WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" + "kickstart.sh " + "--dont-wait") + # disable mail notifications + WOFileUtils.searchreplace(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + 'SEND_EMAIL="YES"', + 'SEND_EMAIL="NO"') + # make changes persistant + WOFileUtils.copyfile(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + "/opt/netdata/etc/netdata/" + "health_alarm_notify.conf") + # check if mysql credentials are available + if os.path.isfile('/etc/mysql/conf.d/my.cnf'): + try: + WOMysql.execute(self, + "create user " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "grant usage on *.* to " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "flush privileges;", + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.info( + self, "fail to setup mysql user for netdata") + WOService.restart_service(self, 'netdata') + + # WordOps Dashboard + if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] + for x in packages): + if not os.path.isfile('{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting wo-dashboard.tar.gz " + "to location {0}22222/htdocs/" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/' + 'wo-dashboard.tar.gz', + '{0}22222/htdocs' + .format(WOVariables.wo_webroot)) + wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " + "grep -oP \"dev [^[:space:]]+ \" " + "| cut -d ' ' -f 2").read() + if (wo_wan != 'eth0' and wo_wan != ''): + WOFileUtils.searchreplace(self, + "{0}22222/htdocs/index.php" + .format(WOVariables.wo_webroot), + "eth0", + "{0}".format(wo_wan)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs" + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # Extplorer FileManager + if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting explorer.tar.gz " + "to location {0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', + '/var/lib/wo/tmp/') + shutil.move('/var/lib/wo/tmp/extplorer-{0}' + .format(WOVariables.wo_extplorer), + '{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # webgrind + if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file webgrind.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/webgrind.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directroy " + "{0}22222/htdocs/php" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/webgrind-master/', + '{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "/usr/local/bin/dot", "/usr/bin/dot") + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "Europe/Copenhagen", + WOVariables.wo_timezone) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "90", "100") + + Log.debug(self, "Setting Privileges of webroot permission to " + "{0}22222/htdocs/php/webgrind/ file " + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + # anemometer + if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file anemometer.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/anemometer.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory") + os.makedirs('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/Anemometer-master', + '{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)) + chars = ''.join(random.sample(string.ascii_letters, 8)) + try: + WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' + '/anemometer/install.sql' + .format(WOVariables.wo_webroot)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + raise SiteError("Unable to import Anemometer database") + + WOMysql.execute(self, 'grant select on' + ' *.* to \'anemometer\'' + '@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get + ('mysql', + 'grant-host'), + chars)) + Log.debug(self, "grant all on slow-query-log.*" + " to anemometer@root_user" + " IDENTIFIED BY password ") + WOMysql.execute(self, 'grant all on slow_query_log.* to' + '\'anemometer\'@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get( + 'mysql', 'grant-host'), + chars), + errormsg="cannot grant priviledges", + log=False) + + # Custom Anemometer configuration + Log.debug(self, "configration Anemometer") + data = dict(host=WOVariables.wo_mysql_host, port='3306', + user='anemometer', password=chars) + wo_anemometer = open('{0}22222/htdocs/db/anemometer' + '/conf/config.inc.php' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') + self.app.render((data), 'anemometer.mustache', + out=wo_anemometer) + wo_anemometer.close() + + if any('/usr/bin/pt-query-advisor' == x[1] + for x in packages): + WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + + # phpredisadmin + if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/cache/' + 'redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/cache/redis" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + if os.path.isfile("/usr/local/bin/composer"): + WOShellExec.cmd_exec(self, "sudo -u www-data -H " + "composer " + "create-project -n -s dev " + "erik-dubbelboer/php-redis-admin " + "/var/www/22222/htdocs/cache" + "/redis/phpRedisAdmin ") + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/cache/file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 5af52b8..9edad10 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -1,9 +1,8 @@ import os import shutil -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose -from wo.cli.plugins.stack import WOStackController +from cement.core import handler, hook from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.download import WODownload @@ -13,6 +12,7 @@ from wo.core.logging import Log from wo.core.services import WOService from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables +from wo.cli.plugins.stack_pref import post_pref class WOStackUpgradeController(CementBaseController): @@ -86,8 +86,6 @@ class WOStackUpgradeController(CementBaseController): apt_packages = [] packages = [] empty_packages = [] - stack = WOStackController() - stack.app = self.app if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and (not self.app.pargs.php) and (not self.app.pargs.mysql) and @@ -192,7 +190,7 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) - stack.post_pref(apt_packages, empty_packages) + post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') From d58d334f8722b44cdfe2f9176f328437e8ef4c69 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 05:11:13 +0200 Subject: [PATCH 006/102] Fix travis non-interactive --- .travis.yml | 4 ++-- CHANGELOG.md | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e9437a..b3ca2ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,8 +47,8 @@ script: - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - - sudo wo stack upgrade --nginx - - sudo wo stack upgrade --php + - sudo wo stack upgrade --nginx --no-prompt + - sudo wo stack upgrade --php --no-prompt - sudo wo stack upgrade --netdata --no-prompt - sudo wo stack upgrade --phpmyadmin --no-prompt - sudo wo stack upgrade --composer --no-prompt diff --git a/CHANGELOG.md b/CHANGELOG.md index ba005be..d5dfb72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- APT Packages configuration step with `wo stack upgrade` to apply new configurations + +#### Changed + +- Moving package configuration in a new plugin stack_pref.py + ### v3.9.7 - 2019-08-02 #### Added From 316768a934487d74fb016c1d7cc39a823c990741 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 06:57:39 +0200 Subject: [PATCH 007/102] Fix stack pref --- wo/cli/plugins/stack_pref.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 9b7a9ed..2667045 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -9,7 +9,6 @@ import re import requests import psutil -# from pynginxconfig import NginxConfig from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_services import WOStackStatusController From 2bdb552a1fbe293956afc684a1923e68c8171c0c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 07:49:08 +0200 Subject: [PATCH 008/102] Fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3ca2ea..5fc0384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ script: - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis - sudo wp --allow-root --info - - sudo wo info + - sudo wo info || tail -a /var/log/wo/wordops.log - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - sudo wo stack upgrade --nginx --no-prompt From dbc2984a31a9424c68855604082684f3473b5128 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 07:49:35 +0200 Subject: [PATCH 009/102] Set back xenial --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fc0384..58e4c7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: bionic +dist: xenial language: bash @@ -44,7 +44,7 @@ script: - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis - sudo wp --allow-root --info - - sudo wo info || tail -a /var/log/wo/wordops.log + - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - sudo wo stack upgrade --nginx --no-prompt From 992174de37badcebe41a4ff90f43c1481b3a229a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 09:45:08 +0200 Subject: [PATCH 010/102] Reindent stack_pref --- wo/cli/plugins/stack_pref.py | 1156 +++++++++++++++++----------------- 1 file changed, 580 insertions(+), 576 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2667045..1e71bb2 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -228,241 +228,241 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - # Setup Nginx common directory - if not os.path.exists('/etc/nginx/common'): - Log.debug(self, 'Creating directory' - '/etc/nginx/common') - os.makedirs('/etc/nginx/common') + # Setup Nginx common directory + if not os.path.exists('/etc/nginx/common'): + Log.debug(self, 'Creating directory' + '/etc/nginx/common') + os.makedirs('/etc/nginx/common') - if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) + if os.path.exists('/etc/nginx/common'): + data = dict(webroot=WOVariables.wo_webroot) - # Common Configuration - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', - out=wo_nginx) - wo_nginx.close() + # Common Configuration + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/locations-wo.conf') + wo_nginx = open('/etc/nginx/common/locations-wo.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'locations.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsubdir.conf') - wo_nginx = open('/etc/nginx/common/wpsubdir.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsubdir.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsubdir.conf') + wo_nginx = open('/etc/nginx/common/wpsubdir.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsubdir.mustache', + out=wo_nginx) + wo_nginx.close() - # PHP 7.2 conf - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php72.conf') - wo_nginx = open('/etc/nginx/common/php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php.mustache', - out=wo_nginx) - wo_nginx.close() + # PHP 7.2 conf + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php72.conf') + wo_nginx = open('/etc/nginx/common/php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php72.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php72.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() - # PHP 7.3 conf - if os.path.isdir("/etc/nginx/common"): - data = dict() + # PHP 7.3 conf + if os.path.isdir("/etc/nginx/common"): + data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php73.conf') - wo_nginx = open('/etc/nginx/common/php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php73.conf') + wo_nginx = open('/etc/nginx/common/php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php73.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php73.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon-php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc-php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc-php7.mustache', + out=wo_nginx) + wo_nginx.close() - # create redis conf - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() + # create redis conf + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', - out=wo_nginx) - wo_nginx.close() + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php73.conf') + wo_nginx = open('/etc/nginx/common/redis-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis-php7.mustache', + out=wo_nginx) + wo_nginx.close() - with open("/etc/nginx/common/release", - "a") as release_file: - release_file.write("v{0}" - .format(WOVariables.wo_version)) - release_file.close() + with open("/etc/nginx/common/release", + "a") as release_file: + release_file.write("v{0}" + .format(WOVariables.wo_version)) + release_file.close() - # Following files should not be overwrited + # Following files should not be overwrited - if not os.path.isfile('/etc/nginx/common/acl.conf'): - data = dict(webroot=WOVariables.wo_webroot) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/acl.conf') - wo_nginx = open('/etc/nginx/common/acl.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'acl.mustache', - out=wo_nginx) - wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/blockips.conf') - wo_nginx = open('/etc/nginx/conf.d/blockips.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'blockips.mustache', out=wo_nginx) - wo_nginx.close() + if not os.path.isfile('/etc/nginx/common/acl.conf'): + data = dict(webroot=WOVariables.wo_webroot) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/acl.conf') + wo_nginx = open('/etc/nginx/common/acl.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'acl.mustache', + out=wo_nginx) + wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/blockips.conf') + wo_nginx = open('/etc/nginx/conf.d/blockips.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'blockips.mustache', out=wo_nginx) + wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/fastcgi.conf') - wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'fastcgi.mustache', out=wo_nginx) - wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/fastcgi.conf') + wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'fastcgi.mustache', out=wo_nginx) + wo_nginx.close() - # add redis cache format if not already done - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d" - "/redis.conf")): - with open("/etc/nginx/conf.d/" - "redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") + # add redis cache format if not already done + if (os.path.isfile("/etc/nginx/nginx.conf") and + not os.path.isfile("/etc/nginx/conf.d" + "/redis.conf")): + with open("/etc/nginx/conf.d/" + "redis.conf", "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local] '\n" + "'$http_host \"$request\" $status" + " $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") # Nginx-Plus does not have nginx # package structure like this # So creating directories - if not os.path.exists('/etc/nginx/sites-available'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-available') + if not os.path.exists('/etc/nginx/sites-available'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-available') - if not os.path.exists('/etc/nginx/sites-enabled'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-enabled') + if not os.path.exists('/etc/nginx/sites-enabled'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-enabled') - # 22222 port settings - if not os.path.isfile("/etc/nginx/sites-available/22222"): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/sites-available/' - '22222') - wo_nginx = open('/etc/nginx/sites-available/22222', - encoding='utf-8', mode='w') - self.app.render((data), '22222.mustache', - out=wo_nginx) - wo_nginx.close() + # 22222 port settings + if not os.path.isfile("/etc/nginx/sites-available/22222"): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/sites-available/' + '22222') + wo_nginx = open('/etc/nginx/sites-available/22222', + encoding='utf-8', mode='w') + self.app.render((data), '22222.mustache', + out=wo_nginx) + wo_nginx.close() - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) - try: - WOShellExec.cmd_exec(self, "printf \"WordOps:" - "$(openssl passwd -crypt " - "{password} 2> /dev/null)\n\"" - "> /etc/nginx/htpasswd-wo " - "2>/dev/null" - .format(password=passwd)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to save HTTP Auth") + passwd = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(16)]) + try: + WOShellExec.cmd_exec(self, "printf \"WordOps:" + "$(openssl passwd -crypt " + "{password} 2> /dev/null)\n\"" + "> /etc/nginx/htpasswd-wo " + "2>/dev/null" + .format(password=passwd)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to save HTTP Auth") # Create Symbolic link for 22222 - WOFileUtils.create_symlink(self, ['/etc/nginx/' - 'sites-available/' - '22222', - '/etc/nginx/' - 'sites-enabled/' - '22222']) - # Create log and cert folder and softlinks - if not os.path.exists('{0}22222/logs' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/logs " - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/logs' - .format(WOVariables.wo_webroot)) + WOFileUtils.create_symlink(self, ['/etc/nginx/' + 'sites-available/' + '22222', + '/etc/nginx/' + 'sites-enabled/' + '22222']) + # Create log and cert folder and softlinks + if not os.path.exists('{0}22222/logs' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/logs " + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/logs' + .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/cert' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/cert" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/cert' - .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/cert' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/cert" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/cert' + .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/conf/nginx" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/conf/nginx" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)) WOFileUtils.create_symlink(self, ['/var/log/nginx/' '22222.access.log', @@ -525,75 +525,74 @@ def post_pref(self, apt_packages, packages): "ssl_certificate_key " "/var/www/22222/cert/22222.key;\n") - # Nginx Configation into GIT - WOGit.add(self, - ["/etc/nginx"], msg="Adding Nginx into Git") - WOService.reload_service(self, 'nginx') + # Nginx Configation into GIT + WOGit.add(self, + ["/etc/nginx"], msg="Adding Nginx into Git") + WOService.reload_service(self, 'nginx') - if set(["nginx"]).issubset(set(apt_packages)): + if set(["nginx"]).issubset(set(apt_packages)): + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) + else: + WOService.restart_service(self, 'nginx') - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) - else: - WOService.restart_service(self, 'nginx') + # create nginx configuration for redis + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if os.path.isdir('/etc/nginx/common'): + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() - # create nginx configuration for redis - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - if os.path.isdir('/etc/nginx/common'): - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() + if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): + if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" + "upstream.conf", + "redis"): + with open("/etc/nginx/conf.d/upstream.conf", + "a") as redis_file: + redis_file.write("upstream redis {\n" + " server 127.0.0.1:6379;\n" + " keepalive 10;\n}\n") - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}\n") + if os.path.isfile("/etc/nginx/nginx.conf"): + if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): + with open("/etc/nginx/conf.d/redis.conf", + "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local]" + " '\n '$http_host" + " \"$request\" " + "$status $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") - if os.path.isfile("/etc/nginx/nginx.conf"): - if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): - with open("/etc/nginx/conf.d/redis.conf", - "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local]" - " '\n '$http_host" - " \"$request\" " - "$status $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - if set(WOVariables.wo_php).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.2/'): - Log.debug(self, 'Creating directory /var/log/php/7.2/') - os.makedirs('/var/log/php/7.2/') + if set(WOVariables.wo_php).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.2/'): + Log.debug(self, 'Creating directory /var/log/php/7.2/') + os.makedirs('/var/log/php/7.2/') # Parse etc/php/7.2/fpm/php.ini config = configparser.ConfigParser() @@ -752,12 +751,12 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.2-fpm') - # PHP7.3 configuration - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.3/'): - Log.debug(self, 'Creating directory /var/log/php/7.3/') - os.makedirs('/var/log/php/7.3/') + # PHP7.3 configuration + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.3/'): + Log.debug(self, 'Creating directory /var/log/php/7.3/') + os.makedirs('/var/log/php/7.3/') # Parse etc/php/7.3/fpm/php.ini config = configparser.ConfigParser() @@ -916,16 +915,16 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.3-fpm') - # create mysql config if it doesn't exist - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - if not os.path.isfile("/etc/mysql/my.cnf"): - config = ("[mysqld]\nwait_timeout = 30\n" - "interactive_timeout=60\nperformance_schema = 0" - "\nquery_cache_type = 1") - config_file = open("/etc/mysql/my.cnf", - encoding='utf-8', mode='w') - config_file.write(config) - config_file.close() + # create mysql config if it doesn't exist + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if not os.path.isfile("/etc/mysql/my.cnf"): + config = ("[mysqld]\nwait_timeout = 30\n" + "interactive_timeout=60\nperformance_schema = 0" + "\nquery_cache_type = 1") + config_file = open("/etc/mysql/my.cnf", + encoding='utf-8', mode='w') + config_file.write(config) + config_file.close() WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' @@ -935,32 +934,32 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") WOService.reload_service(self, 'mysql') - # create fail2ban configuration files - if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): - if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - data = dict() - Log.debug(self, "Setting up fail2ban jails configuration") - fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban.mustache', - out=fail2ban_config) - fail2ban_config.close() + # create fail2ban configuration files + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=fail2ban_config) + fail2ban_config.close() - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'wo-wordpress.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-wp.mustache', - out=fail2ban_config) - fail2ban_config.close() + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=fail2ban_config) + fail2ban_config.close() - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'nginx-forbidden.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-forbidden.mustache', - out=fail2ban_config) - fail2ban_config.close() + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'nginx-forbidden.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-forbidden.mustache', + out=fail2ban_config) + fail2ban_config.close() WOGit.add(self, ["/etc/fail2ban"], msg="Adding Fail2ban into Git") WOService.reload_service(self, 'fail2ban') @@ -1045,63 +1044,63 @@ def post_pref(self, apt_packages, packages): msg="Adding ProFTPd into Git") WOService.reload_service(self, 'proftpd') - if (packages): - if any('/usr/local/bin/wp' == x[1] for x in packages): - Log.debug(self, "Setting Privileges" - " to /usr/local/bin/wp file ") - WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + if (packages): + if any('/usr/local/bin/wp' == x[1] for x in packages): + Log.debug(self, "Setting Privileges" + " to /usr/local/bin/wp file ") + WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] - for x in packages): - WOExtract.extract( - self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') - Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' - 'location /var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/db" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/db' + if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] + for x in packages): + WOExtract.extract( + self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' + 'location /var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/db" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', + '{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)) + shutil.copyfile('{0}22222/htdocs/db/pma' + '/config.sample.inc.php' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/db/pma/config.inc.php' .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', - '{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)) - shutil.copyfile('{0}22222/htdocs/db/pma' - '/config.sample.inc.php' - .format(WOVariables.wo_webroot), - '{0}22222/htdocs/db/pma/config.inc.php' - .format(WOVariables.wo_webroot)) - Log.debug(self, 'Setting Blowfish Secret Key ' - 'FOR COOKIE AUTH to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - blowfish_key = ''.join([random.choice - (string.ascii_letters + - string.digits) - for n in range(25)]) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'blowfish_secret\']" - " = \'\';", - "$cfg[\'blowfish_secret\']" - " = \'{0}\';" - .format(blowfish_key)) - Log.debug(self, 'Setting HOST Server For Mysql to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'Servers\'][$i][\'host\']" - " = \'localhost\';", "$cfg" - "[\'Servers\'][$i][\'host\'] = \'{0}\';" - .format(WOVariables.wo_mysql_host)) + Log.debug(self, 'Setting Blowfish Secret Key ' + 'FOR COOKIE AUTH to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + blowfish_key = ''.join([random.choice + (string.ascii_letters + + string.digits) + for n in range(25)]) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'blowfish_secret\']" + " = \'\';", + "$cfg[\'blowfish_secret\']" + " = \'{0}\';" + .format(blowfish_key)) + Log.debug(self, 'Setting HOST Server For Mysql to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'Servers\'][$i][\'host\']" + " = \'localhost\';", "$cfg" + "[\'Servers\'][$i][\'host\'] = \'{0}\';" + .format(WOVariables.wo_mysql_host)) Log.debug(self, 'Setting Privileges of webroot permission to ' '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), @@ -1109,74 +1108,79 @@ def post_pref(self, apt_packages, packages): WOVariables.wo_php_user, recursive=True) - # composer install and phpmyadmin update - if any('/var/lib/wo/tmp/composer-install' == x[1] - for x in packages): - Log.info(self, "Installing composer, please wait...") - WOShellExec.cmd_exec(self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") - shutil.copyfile('/var/lib/wo/tmp/composer.phar', - '/usr/local/bin/composer') - WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " - "update -n --no-dev -d " - "/var/www/22222/htdocs/db/pma/") + # composer install and phpmyadmin update + if any('/var/lib/wo/tmp/composer-install' == x[1] + for x in packages): + Log.info(self, "Installing composer, please wait...") + WOShellExec.cmd_exec(self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") + shutil.copyfile('/var/lib/wo/tmp/composer.phar', + '/usr/local/bin/composer') + WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) + Log.info(self, "Updating phpMyAdmin, please wait...") + WOShellExec.cmd_exec(self, "/usr/local/bin/composer update --no-plugins --no-scripts " + "-n --no-dev -d " + "/var/www/22222/htdocs/db/pma/") + WOFileUtils.chown(self, '{0}22222/htdocs/db/pma' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) - # netdata install - if any('/var/lib/wo/tmp/kickstart.sh' == x[1] - for x in packages): - if ((not os.path.exists('/opt/netdata')) and - (not os.path.exists('/etc/netdata'))): - Log.info(self, "Installing Netdata, please wait...") - WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" - "kickstart.sh " - "--dont-wait") - # disable mail notifications - WOFileUtils.searchreplace(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - 'SEND_EMAIL="YES"', - 'SEND_EMAIL="NO"') - # make changes persistant - WOFileUtils.copyfile(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - "/opt/netdata/etc/netdata/" - "health_alarm_notify.conf") - # check if mysql credentials are available - if os.path.isfile('/etc/mysql/conf.d/my.cnf'): - try: - WOMysql.execute(self, - "create user " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "grant usage on *.* to " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "flush privileges;", - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.info( - self, "fail to setup mysql user for netdata") - WOService.restart_service(self, 'netdata') + # netdata install + if any('/var/lib/wo/tmp/kickstart.sh' == x[1] + for x in packages): + if ((not os.path.exists('/opt/netdata')) and + (not os.path.exists('/etc/netdata'))): + Log.info(self, "Installing Netdata, please wait...") + WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" + "kickstart.sh " + "--dont-wait") + # disable mail notifications + WOFileUtils.searchreplace(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + 'SEND_EMAIL="YES"', + 'SEND_EMAIL="NO"') + # make changes persistant + WOFileUtils.copyfile(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + "/opt/netdata/etc/netdata/" + "health_alarm_notify.conf") + # check if mysql credentials are available + if os.path.isfile('/etc/mysql/conf.d/my.cnf'): + try: + WOMysql.execute(self, + "create user " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "grant usage on *.* to " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "flush privileges;", + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.info( + self, "fail to setup mysql user for netdata") + WOService.restart_service(self, 'netdata') - # WordOps Dashboard - if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] - for x in packages): - if not os.path.isfile('{0}22222/htdocs/index.php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting wo-dashboard.tar.gz " - "to location {0}22222/htdocs/" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/' - 'wo-dashboard.tar.gz', - '{0}22222/htdocs' - .format(WOVariables.wo_webroot)) + # WordOps Dashboard + if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] + for x in packages): + if not os.path.isfile('{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting wo-dashboard.tar.gz " + "to location {0}22222/htdocs/" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/' + 'wo-dashboard.tar.gz', + '{0}22222/htdocs' + .format(WOVariables.wo_webroot)) wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " "grep -oP \"dev [^[:space:]]+ \" " "| cut -d ' ' -f 2").read() @@ -1195,162 +1199,162 @@ def post_pref(self, apt_packages, packages): WOVariables.wo_php_user, recursive=True) - # Extplorer FileManager - if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting explorer.tar.gz " - "to location {0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', - '/var/lib/wo/tmp/') - shutil.move('/var/lib/wo/tmp/extplorer-{0}' - .format(WOVariables.wo_extplorer), - '{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # webgrind - if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file webgrind.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/webgrind.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directroy " - "{0}22222/htdocs/php" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/webgrind-master/', - '{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "/usr/local/bin/dot", "/usr/bin/dot") - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "Europe/Copenhagen", - WOVariables.wo_timezone) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "90", "100") - - Log.debug(self, "Setting Privileges of webroot permission to " - "{0}22222/htdocs/php/webgrind/ file " + # Extplorer FileManager + if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting explorer.tar.gz " + "to location {0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', + '/var/lib/wo/tmp/') + shutil.move('/var/lib/wo/tmp/extplorer-{0}' + .format(WOVariables.wo_extplorer), + '{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs/files" .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222' .format(WOVariables.wo_webroot), WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) - # anemometer - if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file anemometer.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/anemometer.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory") - os.makedirs('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/Anemometer-master', - '{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)) - chars = ''.join(random.sample(string.ascii_letters, 8)) - try: - WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' - '/anemometer/install.sql' - .format(WOVariables.wo_webroot)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("Unable to import Anemometer database") - WOMysql.execute(self, 'grant select on' - ' *.* to \'anemometer\'' - '@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get - ('mysql', - 'grant-host'), - chars)) - Log.debug(self, "grant all on slow-query-log.*" - " to anemometer@root_user" - " IDENTIFIED BY password ") - WOMysql.execute(self, 'grant all on slow_query_log.* to' - '\'anemometer\'@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get( - 'mysql', 'grant-host'), - chars), - errormsg="cannot grant priviledges", - log=False) - - # Custom Anemometer configuration - Log.debug(self, "configration Anemometer") - data = dict(host=WOVariables.wo_mysql_host, port='3306', - user='anemometer', password=chars) - wo_anemometer = open('{0}22222/htdocs/db/anemometer' - '/conf/config.inc.php' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') - self.app.render((data), 'anemometer.mustache', - out=wo_anemometer) - wo_anemometer.close() - - if any('/usr/bin/pt-query-advisor' == x[1] - for x in packages): - WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) - - # phpredisadmin - if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/cache/' - 'redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/cache/redis" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - if os.path.isfile("/usr/local/bin/composer"): - WOShellExec.cmd_exec(self, "sudo -u www-data -H " - "composer " - "create-project -n -s dev " - "erik-dubbelboer/php-redis-admin " - "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin ") - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/cache/file ' + # webgrind + if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file webgrind.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/webgrind.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directroy " + "{0}22222/htdocs/php" .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/webgrind-master/', + '{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "/usr/local/bin/dot", "/usr/bin/dot") + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "Europe/Copenhagen", + WOVariables.wo_timezone) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "90", "100") + + Log.debug(self, "Setting Privileges of webroot permission to " + "{0}22222/htdocs/php/webgrind/ file " + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + # anemometer + if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file anemometer.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/anemometer.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory") + os.makedirs('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/Anemometer-master', + '{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)) + chars = ''.join(random.sample(string.ascii_letters, 8)) + try: + WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' + '/anemometer/install.sql' + .format(WOVariables.wo_webroot)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + raise SiteError("Unable to import Anemometer database") + + WOMysql.execute(self, 'grant select on' + ' *.* to \'anemometer\'' + '@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get + ('mysql', + 'grant-host'), + chars)) + Log.debug(self, "grant all on slow-query-log.*" + " to anemometer@root_user" + " IDENTIFIED BY password ") + WOMysql.execute(self, 'grant all on slow_query_log.* to' + '\'anemometer\'@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get( + 'mysql', 'grant-host'), + chars), + errormsg="cannot grant priviledges", + log=False) + + # Custom Anemometer configuration + Log.debug(self, "configration Anemometer") + data = dict(host=WOVariables.wo_mysql_host, port='3306', + user='anemometer', password=chars) + wo_anemometer = open('{0}22222/htdocs/db/anemometer' + '/conf/config.inc.php' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') + self.app.render((data), 'anemometer.mustache', + out=wo_anemometer) + wo_anemometer.close() + + if any('/usr/bin/pt-query-advisor' == x[1] + for x in packages): + WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + + # phpredisadmin + if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/cache/' + 'redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/cache/redis" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222' .format(WOVariables.wo_webroot), WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) + if os.path.isfile("/usr/local/bin/composer"): + WOShellExec.cmd_exec(self, "sudo -u www-data -H " + "composer " + "create-project -n -s dev " + "erik-dubbelboer/php-redis-admin " + "/var/www/22222/htdocs/cache" + "/redis/phpRedisAdmin ") + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/cache/file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) From e84fed489edb1c44b54ffaf70d729e8686fbc1f6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 11:46:45 +0200 Subject: [PATCH 011/102] Another fix for stack_pref --- wo/cli/plugins/stack_pref.py | 40 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 1e71bb2..01b3642 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1,22 +1,16 @@ import codecs import configparser import os -import pwd import random import shutil import string -import re import requests -import psutil from wo.cli.plugins.site_functions import * -from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_services import WOStackStatusController -from wo.core.addswap import WOSwap from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.cron import WOCron -from wo.core.download import WODownload from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit @@ -530,23 +524,23 @@ def post_pref(self, apt_packages, packages): ["/etc/nginx"], msg="Adding Nginx into Git") WOService.reload_service(self, 'nginx') - if set(["nginx"]).issubset(set(apt_packages)): - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) + if set(["nginx"]).issubset(set(apt_packages)): + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) else: WOService.restart_service(self, 'nginx') From 915b3f41f813032fa6e7a2f21c7e90beb6f6bbf0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:25:00 +0200 Subject: [PATCH 012/102] Fix php upgrade --- CHANGELOG.md | 1 + wo/cli/plugins/stack_pref.py | 574 +++++++++++++++++------------------ 2 files changed, 288 insertions(+), 287 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5dfb72..6e12027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Moving package configuration in a new plugin stack_pref.py +- Set WordOps backend password length from 16 to 24 ### v3.9.7 - 2019-08-02 diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 01b3642..23d652a 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -412,16 +412,16 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) + randompass = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(24)]) try: WOShellExec.cmd_exec(self, "printf \"WordOps:" "$(openssl passwd -crypt " "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo " "2>/dev/null" - .format(password=passwd)) + .format(password=randompass)) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Failed to save HTTP Auth") @@ -589,161 +589,161 @@ def post_pref(self, apt_packages, packages): os.makedirs('/var/log/php/7.2/') # Parse etc/php/7.2/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file " + config = configparser.ConfigParser() + Log.debug(self, "configuring php file " + "/etc/php/7.2/fpm/php.ini") + config.read('/etc/php/7.2/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.2/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " "/etc/php/7.2/fpm/php.ini") - config.read('/etc/php/7.2/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.2/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.2/fpm/php.ini") - config.write(configfile) + config.write(configfile) - # Parse /etc/php/7.2/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.2-fpm.pid", - error_log="/var/log/php/7.2/fpm.log", + # Parse /etc/php/7.2/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.2-fpm.pid", + error_log="/var/log/php/7.2/fpm.log", include="/etc/php/7.2/fpm/pool.d/*.conf") - Log.debug(self, "writting php7.2 configuration into " - "/etc/php/7.2/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() + Log.debug(self, "writting php7.2 configuration into " + "/etc/php/7.2/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() - # Parse /etc/php/7.2/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.2/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php72-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writing PHP 7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www.conf") - config.write(configfile) + # Parse /etc/php/7.2/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.2/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php72-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writing PHP 7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www.conf") + config.write(configfile) - with open("/etc/php/7.2/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") + with open("/etc/php/7.2/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") - # Generate /etc/php/7.2/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php72-two-fpm.sock' - with open('/etc/php/7.2/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www-two.conf") - config.write(confifile) + # Generate /etc/php/7.2/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php72-two-fpm.sock' + with open('/etc/php/7.2/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www-two.conf") + config.write(confifile) - # Generate /etc/php/7.2/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9172' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.2/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/debug.conf") - config.write(confifile) + # Generate /etc/php/7.2/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9172' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.2/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/debug.conf") + config.write(confifile) - with open("/etc/php/7.2/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") + with open("/etc/php/7.2/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.2/mods-available/" - "xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.2/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.2/mods-available/" + "xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.2/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.2-fpm') + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.2-fpm') # PHP7.3 configuration if set(WOVariables.wo_php73).issubset(set(apt_packages)): @@ -752,162 +752,162 @@ def post_pref(self, apt_packages, packages): Log.debug(self, 'Creating directory /var/log/php/7.3/') os.makedirs('/var/log/php/7.3/') - # Parse etc/php/7.3/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file /etc/php/7.3/" - "fpm/php.ini") - config.read('/etc/php/7.3/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.3/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.3/fpm/php.ini") - config.write(configfile) + # Parse etc/php/7.3/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file /etc/php/7.3/" + "fpm/php.ini") + config.read('/etc/php/7.3/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.3/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.3/fpm/php.ini") + config.write(configfile) - # Parse /etc/php/7.3/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.3-fpm.pid", + # Parse /etc/php/7.3/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.3-fpm.pid", error_log="/var/log/php7.3-fpm.log", include="/etc/php/7.3/fpm/pool.d/*.conf") - Log.debug(self, "writting php 7.3 configuration into " - "/etc/php/7.3/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() + Log.debug(self, "writting php 7.3 configuration into " + "/etc/php/7.3/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() - # Parse /etc/php/7.3/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.3/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php73-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www.conf") - config.write(configfile) + # Parse /etc/php/7.3/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.3/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php73-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www.conf") + config.write(configfile) - with open("/etc/php/7.3/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") + with open("/etc/php/7.3/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") - # Generate /etc/php/7.3/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php73-two-fpm.sock' - with open('/etc/php/7.3/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www-two.conf") - config.write(confifile) + # Generate /etc/php/7.3/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php73-two-fpm.sock' + with open('/etc/php/7.3/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www-two.conf") + config.write(confifile) - # Generate /etc/php/7.3/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9173' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.3/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/debug.conf") - config.write(confifile) + # Generate /etc/php/7.3/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9173' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.3/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/debug.conf") + config.write(confifile) - with open("/etc/php/7.3/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") + with open("/etc/php/7.3/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.3/mods-available" - "/xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.3/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.3/mods-available" + "/xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.3/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.3-fpm') + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.3-fpm') # create mysql config if it doesn't exist if set(WOVariables.wo_mysql).issubset(set(apt_packages)): From 65af41eb244c697efb76e319de5bf85ec357d890 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:32:07 +0200 Subject: [PATCH 013/102] Fix php72 upgrade --- wo/cli/plugins/stack_upgrade.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 9edad10..0d2c3ea 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -190,6 +190,10 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) + if set(WOVariables.wo_php).issubset(set(apt_packages)): + WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") + WOFileUtils.rm(self, "/etc/php/7.2/fpm/" + "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): From 3ab5f74c862dde83b464b5a3b7e712092731dfb0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:38:50 +0200 Subject: [PATCH 014/102] Remove outdated part of wo upgrade --- wo/cli/plugins/stack_upgrade.py | 55 +++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0d2c3ea..bafeeff 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -31,7 +31,9 @@ class WOStackUpgradeController(CementBaseController): (['--nginx'], dict(help='Upgrade Nginx stack', action='store_true')), (['--php'], - dict(help='Upgrade PHP stack', action='store_true')), + dict(help='Upgrade PHP 7.2 stack', action='store_true')), + (['--php73'], + dict(help='Upgrade PHP 7.3 stack', action='store_true')), (['--mysql'], dict(help='Upgrade MySQL stack', action='store_true')), (['--wpcli'], @@ -49,37 +51,6 @@ class WOStackUpgradeController(CementBaseController): action='store_true')), ] - @expose(hide=True) - def upgrade_php56(self): - if WOVariables.wo_distro == "ubuntu": - if os.path.isfile("/etc/apt/sources.list.d/ondrej-php5-5_6-{0}." - "list".format(WOVariables.wo_platform_codename)): - Log.error(self, "Unable to find PHP 5.5") - else: - if not(os.path.isfile(WOVariables.wo_repo_file_path) and - WOFileUtils.grep(self, WOVariables.wo_repo_file_path, - "php55")): - Log.error(self, "Unable to find PHP 5.5") - - Log.info(self, "During PHP update process non nginx-cached" - " parts of your site may remain down.") - - # Check prompt - if (not self.app.pargs.no_prompt): - start_upgrade = input("Do you want to continue:[y/N]") - if start_upgrade != "Y" and start_upgrade != "y": - Log.error(self, "Not starting PHP package update") - - if WOVariables.wo_distro == "ubuntu": - WORepo.remove(self, ppa="ppa:ondrej/php5") - WORepo.add(self, ppa=WOVariables.wo_php_repo) - - Log.info(self, "Updating apt-cache, please wait...") - WOAptGet.update(self) - Log.info(self, "Installing packages, please wait ...") - WOAptGet.install(self, WOVariables.wo_php + - WOVariables.wo_php_extra) - @expose(hide=True) def default(self): # All package update @@ -88,7 +59,8 @@ class WOStackUpgradeController(CementBaseController): empty_packages = [] if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and - (not self.app.pargs.php) and (not self.app.pargs.mysql) and + (not self.app.pargs.php) and (not self.app.pargs.php73) and + (not self.app.pargs.mysql) and (not self.app.pargs.all) and (not self.app.pargs.wpcli) and (not self.app.pargs.netdata) and (not self.app.pargs.composer) and (not self.app.pargs.phpmyadmin) and @@ -106,7 +78,6 @@ class WOStackUpgradeController(CementBaseController): self.app.pargs.php = True self.app.pargs.mysql = True self.app.pargs.wpcli = True - self.app.pargs.netdata = True if self.app.pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): @@ -124,6 +95,16 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") + if self.app.pargs.php73: + if WOAptGet.is_installed(self, 'php7.3-fpm'): + if not WOAptGet.is_installed(self, 'php7.2-fpm'): + apt_packages = apt_packages + WOVariables.wo_php73 + \ + WOVariables.wo_php_extra + else: + apt_packages = apt_packages + WOVariables.wo_php73 + else: + Log.info(self, "PHP 7.3 is not installed") + if self.app.pargs.mysql: if WOAptGet.is_installed(self, 'mariadb-server'): apt_packages = apt_packages + WOVariables.wo_mysql @@ -194,12 +175,18 @@ class WOStackUpgradeController(CementBaseController): WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") WOFileUtils.rm(self, "/etc/php/7.2/fpm/" "pool.d/www-two.conf") + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOFileUtils.rm(self, "/etc/php/7.3/fpm/pool.d/www.conf") + WOFileUtils.rm(self, "/etc/php/7.3/fpm/" + "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') if set(WOVariables.wo_php).issubset(set(apt_packages)): WOService.restart_service(self, 'php7.2-fpm') + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOService.restart_service(self, 'php7.3-fpm') if set(WOVariables.wo_mysql).issubset(set(apt_packages)): WOService.restart_service(self, 'mysql') if set(WOVariables.wo_redis).issubset(set(apt_packages)): From 72a8800ebd90d84c55d57210e5ab056de48f7e3c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:50:12 +0200 Subject: [PATCH 015/102] Another fix for php7.x-fpm upgrade --- wo/cli/plugins/stack_upgrade.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index bafeeff..dbcb074 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -169,16 +169,12 @@ class WOStackUpgradeController(CementBaseController): if len(apt_packages): # apt-get update WOAptGet.update(self) + if set(WOVariables.wo_php).issubset(set(apt_packages)): + WOAptGet.purge(self, ['php7.2-fpm']) + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOAptGet.purge(self, ['php7.3-fpm']) # Update packages WOAptGet.install(self, apt_packages) - if set(WOVariables.wo_php).issubset(set(apt_packages)): - WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") - WOFileUtils.rm(self, "/etc/php/7.2/fpm/" - "pool.d/www-two.conf") - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - WOFileUtils.rm(self, "/etc/php/7.3/fpm/pool.d/www.conf") - WOFileUtils.rm(self, "/etc/php/7.3/fpm/" - "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): From 7d8577eb17a80e995f99196b92487925759e0837 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 13:05:56 +0200 Subject: [PATCH 016/102] Fix passwd variable --- setup.py | 3 ++- wo/cli/plugins/stack_migrate.py | 4 ---- wo/cli/plugins/stack_pref.py | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 14a12e6..dff04ab 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup(name='wo', classifiers=[], keywords='', author='WordOps', - author_email='core@wordops.net', + author_email='contact@wordops.io', url='https://wordops.net', license='MIT', packages=find_packages(exclude=['ez_setup', 'examples', 'tests', @@ -85,6 +85,7 @@ setup(name='wo', 'psutil == 3.1.1', 'sh', 'SQLAlchemy', + 'requests == 2.22.0', ], data_files=[('/etc/wo', ['config/wo.conf']), ('/etc/wo/plugins.d', conf), diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index a81d2b4..c1a84c6 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -79,10 +79,6 @@ class WOStackMigrateController(CementBaseController): # Install MariaDB apt_packages = WOVariables.wo_mysql - # If PHP is installed then install php7.2-mysql - if WOAptGet.is_installed(self, "php7.2-fpm"): - apt_packages = apt_packages + ["php7.2-mysql"] - Log.info(self, "Updating apt-cache, hang on...") WOAptGet.update(self) Log.info(self, "Installing MariaDB, hang on...") diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 23d652a..c4f3863 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -412,7 +412,7 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - randompass = ''.join([random.choice + passwd = ''.join([random.choice (string.ascii_letters + string.digits) for n in range(24)]) try: @@ -421,7 +421,7 @@ def post_pref(self, apt_packages, packages): "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo " "2>/dev/null" - .format(password=randompass)) + .format(password=passwd)) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Failed to save HTTP Auth") From 3729d27fb29043825f16cb525540ae6efdd7570a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 13:11:24 +0200 Subject: [PATCH 017/102] Fix package purge --- wo/cli/plugins/stack_upgrade.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index dbcb074..a834857 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -170,9 +170,11 @@ class WOStackUpgradeController(CementBaseController): # apt-get update WOAptGet.update(self) if set(WOVariables.wo_php).issubset(set(apt_packages)): - WOAptGet.purge(self, ['php7.2-fpm']) + WOAptGet.remove(self, ['php7.2-fpm'], + auto=False, purge=True) if set(WOVariables.wo_php73).issubset(set(apt_packages)): - WOAptGet.purge(self, ['php7.3-fpm']) + WOAptGet.remove(self, ['php7.3-fpm'], + auto=False, purge=True) # Update packages WOAptGet.install(self, apt_packages) post_pref(self, apt_packages, empty_packages) From 6f6df150683fe8e0f250fdc0ad5e673c31117c66 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 14:22:20 +0200 Subject: [PATCH 018/102] add more mysql settings --- wo/cli/plugins/stack.py | 68 ---------------------- wo/cli/plugins/stack_pref.py | 107 ++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 77 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 563f6b9..7c75c30 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -465,74 +465,6 @@ class WOStackController(CementBaseController): "maxmemory-policy " "allkeys-lru") WOService.restart_service(self, 'redis-server') - if 'mariadb-server' in apt_packages: - # setting innodb memory usage - wo_ram = psutil.virtual_memory().total / (1024 * 1024) - wo_ram_innodb = int(wo_ram*0.3) - wo_ram_log_buffer = int(wo_ram_innodb*0.25) - wo_ram_log_size = int(wo_ram_log_buffer*0.5) - # replacing default values - if os.path.isfile("/etc/mysql/my.cnf"): - Log.debug(self, "Tuning MySQL configuration") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_buffer_pool_size = 256M", - "innodb_buffer_pool_size = {0}M" - .format(wo_ram_innodb)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_log_buffer_size = 8M", - "innodb_log_buffer_size = {0}M" - .format(wo_ram_log_buffer)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "#innodb_log_file_size = 50M", - "innodb_log_file_size = {0}M" - .format(wo_ram_log_size)) - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "wait_timeout " - "= 600", - "wait_timeout " - "= 120") - # disabling mariadb binlog - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "log_bin " - "= /var/log/mysql/" - "mariadb-bin", - "#log_bin " - " = /var/log/" - "mysql/mariadb-bin") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - 'log_bin_index ' - "= /var/log/mysql/" - "mariadb-bin.index", - "#log_bin_index " - "= /var/log/mysql/" - "mariadb-bin.index") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "expire_logs_days = 10", - "#expire_logs_days " - "= 10") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "max_binlog_size " - "= 100M", - "#max_binlog_size " - "= 100M") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_open_files =" - " 400", - "innodb_open_files =" - " 16000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_io_capacity =" - " 400", - "innodb_io_capacity =" - " 16000") - WOService.stop_service(self, 'mysql') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', - '/var/lib/mysql/ib_logfile0.bak') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', - '/var/lib/mysql/ib_logfile1.bak') - WOService.start_service(self, 'mysql') if disp_msg: if (self.msg): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index c4f3863..5f47d2f 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -5,6 +5,7 @@ import random import shutil import string import requests +import psutil from wo.cli.plugins.site_functions import * from wo.cli.plugins.stack_services import WOStackStatusController @@ -413,8 +414,8 @@ def post_pref(self, apt_packages, packages): wo_nginx.close() passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(24)]) + (string.ascii_letters + string.digits) + for n in range(24)]) try: WOShellExec.cmd_exec(self, "printf \"WordOps:" "$(openssl passwd -crypt " @@ -919,14 +920,102 @@ def post_pref(self, apt_packages, packages): encoding='utf-8', mode='w') config_file.write(config) config_file.close() + else: + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + wo_ram_innodb = int(wo_ram*0.3) + wo_ram_log_buffer = int(wo_ram_innodb*0.25) + wo_ram_log_size = int(wo_ram_log_buffer*0.5) + # replacing default values + Log.debug(self, "Tuning MySQL configuration") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_buffer_pool_size = 256M", + "innodb_buffer_pool_size = {0}M" + .format(wo_ram_innodb)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_log_buffer_size = 8M", + "innodb_log_buffer_size = {0}M" + .format(wo_ram_log_buffer)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#innodb_log_file_size = 50M", + "innodb_log_file_size = {0}M" + .format(wo_ram_log_size)) + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "wait_timeout " + "= 600", + "wait_timeout " + "= 120") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "skip-external-locking", + "skip-external-locking\n" + "skip-name-resolve = 1\n") + # disabling mariadb binlog + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "log_bin " + "= /var/log/mysql/" + "mariadb-bin", + "#log_bin " + " = /var/log/" + "mysql/mariadb-bin") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + 'log_bin_index ' + "= /var/log/mysql/" + "mariadb-bin.index", + "#log_bin_index " + "= /var/log/mysql/" + "mariadb-bin.index") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "expire_logs_days = 10", + "#expire_logs_days " + "= 10") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "max_binlog_size " + "= 100M", + "#max_binlog_size " + "= 100M") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_open_files =" + " 400", + "innodb_open_files =" + " 16000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_io_capacity =" + " 400", + "innodb_io_capacity =" + " 16000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "query_cache_size = 64M", + "query_cache_size = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#query_cache_type = DEMAND", + "query_cache_type = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "max_allowed_packet = 16M", + "max_allowed_packet = 64M") + if (wo_ram_innodb > 1000) and (wo_ram_innodb < 64000): + wo_innodb_instance = int(wo_ram_innodb/1000) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "# * Security Features", + "innodb_buffer_pool_instances " + "= {0}\n" + .format(wo_innodb_instance) + + "# * Security Features") - WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) - WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' - '> /dev/null 2>&1', - comment='MySQL optimization cronjob ' - 'added by WordOps') - WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - WOService.reload_service(self, 'mysql') + WOService.stop_service(self, 'mysql') + WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', + '/var/lib/mysql/ib_logfile0.bak') + WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', + '/var/lib/mysql/ib_logfile1.bak') + WOService.start_service(self, 'mysql') + + WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' + '> /dev/null 2>&1', + comment='MySQL optimization cronjob ' + 'added by WordOps') + WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") + WOService.restart_service(self, 'mysql') # create fail2ban configuration files if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): From 3211a7494094ebc8c8653706446c83fa5853ef55 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 14:40:48 +0200 Subject: [PATCH 019/102] more MariaDB optimizations --- wo/cli/plugins/stack_pref.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 5f47d2f..2098a62 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -949,6 +949,7 @@ def post_pref(self, apt_packages, packages): "skip-external-locking", "skip-external-locking\n" "skip-name-resolve = 1\n") + # disabling mariadb binlog WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", @@ -990,6 +991,12 @@ def post_pref(self, apt_packages, packages): WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "#query_cache_type = DEMAND", "query_cache_type = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#open-files-limit = 2000", + "open-files-limit = 10000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "table_open_cache = 400", + "table_open_cache = 16000") WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "max_allowed_packet = 16M", "max_allowed_packet = 64M") From 01e6eaecb01e2226951785d0018b55b40174da26 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 19:17:59 +0200 Subject: [PATCH 020/102] Implement `--wprocket` * cleanup templates by using variables * add configuration for wp-rocket * update mapping for wp-rocket --- wo/cli/plugins/site.py | 51 ++++++++++++++-- wo/cli/plugins/site_functions.py | 24 +++++--- wo/cli/plugins/stack_pref.py | 41 ++++++++----- wo/cli/templates/22222.mustache | 2 +- wo/cli/templates/locations-php7.mustache | 54 ----------------- wo/cli/templates/locations.mustache | 2 +- wo/cli/templates/map-wp.mustache | 7 ++- wo/cli/templates/php.mustache | 4 +- wo/cli/templates/php7.mustache | 10 ---- wo/cli/templates/redis-php7.mustache | 42 ------------- wo/cli/templates/redis.mustache | 4 +- wo/cli/templates/upstream.mustache | 2 +- wo/cli/templates/virtualconf-php7.mustache | 2 +- wo/cli/templates/virtualconf.mustache | 2 +- wo/cli/templates/webp.mustache | 2 +- wo/cli/templates/wpcommon-php7.mustache | 70 ---------------------- wo/cli/templates/wpcommon.mustache | 4 +- wo/cli/templates/wpfc-php7.mustache | 21 ------- wo/cli/templates/wpfc.mustache | 4 +- wo/cli/templates/wprocket.mustache | 21 +++++++ wo/cli/templates/wpsc-php7.mustache | 17 ------ wo/cli/templates/wpsc.mustache | 4 +- 22 files changed, 131 insertions(+), 259 deletions(-) delete mode 100644 wo/cli/templates/locations-php7.mustache delete mode 100644 wo/cli/templates/php7.mustache delete mode 100644 wo/cli/templates/redis-php7.mustache delete mode 100644 wo/cli/templates/wpcommon-php7.mustache delete mode 100644 wo/cli/templates/wpfc-php7.mustache create mode 100644 wo/cli/templates/wprocket.mustache delete mode 100644 wo/cli/templates/wpsc-php7.mustache diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index d5e5e59..20948e4 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -333,6 +333,9 @@ class WOSiteCreateController(CementBaseController): (['--wpsc'], dict(help="create WordPress single/multi site with wpsc cache", action='store_true')), + (['--wprocket'], + dict(help="create WordPress single/multi site with WP-Rocket", + action='store_true')), (['--wpredis'], dict(help="create WordPress single/multi site " "with redis cache", @@ -423,7 +426,8 @@ class WOSiteCreateController(CementBaseController): if stype == 'proxy': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['proxy'] = True data['host'] = host @@ -433,14 +437,16 @@ class WOSiteCreateController(CementBaseController): if self.app.pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=False, php73=True, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['basic'] = True if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) if stype == 'php': @@ -451,7 +457,8 @@ class WOSiteCreateController(CementBaseController): data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', wo_db_host='') @@ -479,6 +486,7 @@ class WOSiteCreateController(CementBaseController): if ((not self.app.pargs.wpfc) and (not self.app.pargs.wpsc) and + (not self.app.pargs.wprocket) and (not self.app.pargs.wpredis)): data['basic'] = True @@ -488,6 +496,7 @@ class WOSiteCreateController(CementBaseController): data['basic'] = False self.app.pargs.wpredis = True + # Check rerequired packages are installed or not wo_auth = site_package_check(self, stype) @@ -825,6 +834,8 @@ class WOSiteUpdateController(CementBaseController): dict(help="update to wpfc cache", action='store_true')), (['--wpsc'], dict(help="update to wpsc cache", action='store_true')), + (['--wprocket'], + dict(help="update to WP-Rocket cache", action='store_true')), (['--wpredis'], dict(help="update to redis cache", action='store_true')), (['-le', '--letsencrypt'], @@ -868,6 +879,7 @@ class WOSiteUpdateController(CementBaseController): if not (pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpsubdir or pargs.wpsubdomain or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpredis or pargs.letsencrypt or pargs.hsts or pargs.dns or pargs.force): Log.error(self, "Please provide options to update sites.") @@ -946,6 +958,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.password and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpsubdir or pargs.wpsubdomain or pargs.hsts)): try: @@ -958,6 +971,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.hsts and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpsubdir or pargs.wpsubdomain or pargs.password)): try: @@ -999,7 +1013,7 @@ class WOSiteUpdateController(CementBaseController): if stype == 'php': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, currsitetype=oldsitetype, currcachetype=oldcachetype) @@ -1007,7 +1021,7 @@ class WOSiteUpdateController(CementBaseController): data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', wo_db_host='', @@ -1061,21 +1075,31 @@ class WOSiteUpdateController(CementBaseController): data['wpfc'] = False data['wpsc'] = False data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpfc': data['basic'] = False data['wpfc'] = True data['wpsc'] = False data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpsc': data['basic'] = False data['wpfc'] = False data['wpsc'] = True data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpredis': data['basic'] = False data['wpfc'] = False data['wpsc'] = False data['wpredis'] = True + data['wprocket'] = False + elif oldcachetype == 'wprocket': + data['basic'] = False + data['wpfc'] = False + data['wpsc'] = False + data['wpredis'] = False + data['wprocket'] = True if pargs.php73 == 'on': data['php73'] = True @@ -1233,6 +1257,11 @@ class WOSiteUpdateController(CementBaseController): data['basic'] = False cache = 'wpredis' + if pargs.wprocket and data['currcachetype'] != 'wprocket': + data['wprocket'] = True + data['basic'] = False + cache = 'wprocket' + if (php73 is old_php73) and (stype == oldsitetype and cache == oldcachetype): return 1 @@ -1620,6 +1649,16 @@ class WOSiteUpdateController(CementBaseController): "`tail /var/log/wo/wordops.log` and please try again") return 1 + if oldcachetype == 'wprocket' and not data['wprocket']: + try: + uninstallwp_plugin(self, 'wp-rocket', data) + except SiteError as e: + Log.debug(self, str(e)) + Log.info(self, Log.FAIL + "Update site failed." + "Check the log for details: " + "`tail /var/log/wo/wordops.log` and please try again") + return 1 + # Service Nginx Reload if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 032ae6f..dd66040 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -897,12 +897,12 @@ def site_package_check(self, stype): if (os.path.isdir("/etc/nginx/common") and not os.path.isfile("/etc/nginx/common/locations-wo.conf")): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/locations-wo.conf') wo_nginx = open('/etc/nginx/common/locations-wo.conf', encoding='utf-8', mode='w') - self.app.render((data), 'locations-php7.mustache', + self.app.render((data), 'locations.mustache', out=wo_nginx) wo_nginx.close() @@ -910,7 +910,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/php73.conf') wo_nginx = open('/etc/nginx/common/php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', + self.app.render((data), 'php.mustache', out=wo_nginx) wo_nginx.close() @@ -918,7 +918,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpcommon-php73.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', + self.app.render((data), 'wpcommon.mustache', out=wo_nginx) wo_nginx.close() @@ -926,7 +926,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpfc-php73.conf') wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', + self.app.render((data), 'wpfc.mustache', out=wo_nginx) wo_nginx.close() @@ -934,18 +934,26 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpsc-php73.conf') wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wprocket-php73.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wprocket.mustache', out=wo_nginx) wo_nginx.close() if (os.path.isfile("/etc/nginx/nginx.conf") and not os.path.isfile("/etc/nginx/common/redis-php73.conf")): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') wo_nginx = open('/etc/nginx/common/redis-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', + self.app.render((data), 'redis.mustache', out=wo_nginx) wo_nginx.close() diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2098a62..df9bf9c 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -230,7 +230,7 @@ def post_pref(self, apt_packages, packages): os.makedirs('/etc/nginx/common') if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) + data = dict() # Common Configuration Log.debug(self, 'Writting the nginx configuration to ' @@ -248,7 +248,7 @@ def post_pref(self, apt_packages, packages): self.app.render((data), 'wpsubdir.mustache', out=wo_nginx) wo_nginx.close() - + data = dict(upstream="php72") # PHP 7.2 conf Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/php72.conf') @@ -258,6 +258,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/wpcommon-php72.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', @@ -282,15 +290,23 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wprocket-php72.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wprocket.mustache', + out=wo_nginx) + wo_nginx.close() + # PHP 7.3 conf if os.path.isdir("/etc/nginx/common"): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/php73.conf') wo_nginx = open('/etc/nginx/common/php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', + self.app.render((data), 'php.mustache', out=wo_nginx) wo_nginx.close() @@ -298,7 +314,7 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpcommon-php73.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', + self.app.render((data), 'wpcommon.mustache', out=wo_nginx) wo_nginx.close() @@ -306,7 +322,7 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpfc-php73.conf') wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', + self.app.render((data), 'wpfc.mustache', out=wo_nginx) wo_nginx.close() @@ -314,26 +330,23 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpsc-php73.conf') wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', + self.app.render((data), 'wpsc.mustache', out=wo_nginx) wo_nginx.close() - # create redis conf - data = dict() Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', + 'file /etc/nginx/common/wprocket-php73.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', + self.app.render((data), 'wprocket.mustache', out=wo_nginx) wo_nginx.close() - data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') wo_nginx = open('/etc/nginx/common/redis-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', + self.app.render((data), 'redis.mustache', out=wo_nginx) wo_nginx.close() diff --git a/wo/cli/templates/22222.mustache b/wo/cli/templates/22222.mustache index 3106ee7..e34fa36 100644 --- a/wo/cli/templates/22222.mustache +++ b/wo/cli/templates/22222.mustache @@ -1,4 +1,4 @@ -# WordOps admin NGINX CONFIGURATION - WO v3.9.5 +# WordOps admin NGINX CONFIGURATION - WO v3.9.7 server { diff --git a/wo/cli/templates/locations-php7.mustache b/wo/cli/templates/locations-php7.mustache deleted file mode 100644 index 267891c..0000000 --- a/wo/cli/templates/locations-php7.mustache +++ /dev/null @@ -1,54 +0,0 @@ -# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# Basic locations files -location = /favicon.ico { - access_log off; - log_not_found off; - expires max; -} -location = /robots.txt { -# Some WordPress plugin gererate robots.txt file -# Refer #340 issue - try_files $uri $uri/ /index.php$is_args$args @robots; - access_log off; - log_not_found off; -} -# fallback for robots.txt with default wordpress rules -location @robots { - return 200 "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n"; -} -# Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp)$ { - add_header "Access-Control-Allow-Origin" "*"; - access_log off; - log_not_found off; - expires max; -} -# Cache css & js files -location ~* \.(?:css(\.map)?|js(\.map)?)$ { - add_header "Access-Control-Allow-Origin" "*"; - access_log off; - log_not_found off; - expires 30d; -} -# Security settings for better privacy -# Deny hidden files -location ~ /\.(?!well-known\/) { - deny all; -} -# letsencrypt validation -location /.well-known/acme-challenge/ { - alias /var/www/html/.well-known/acme-challenge/; - allow all; -} -# Return 403 forbidden for readme.(txt|html) or license.(txt|html) or example.(txt|html) or other common git repository files -location ~* "/(^$|readme|license|example|README|LEGALNOTICE|INSTALLATION|CHANGELOG)\.(txt|html|md)" { - deny all; -} -# Deny backup extensions & log files and return 403 forbidden -location ~* "\.(old|orig|original|php#|php~|php_bak|save|swo|aspx?|tpl|sh|bash|bak?|cfg|cgi|dll|exe|git|hg|ini|jsp|log|mdb|out|sql|svn|swp|tar|rdf)$" { - deny all; -} -location ~* "/(=|\$&|_mm|(wp-)?config\.|cgi-|etc/passwd|muieblack)" { - deny all; -} diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index ba1d66f..26e8230 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -1,4 +1,4 @@ -# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.5 +# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # Basic locations files location = /favicon.ico { diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 5cdb3d9..f764020 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -1,4 +1,4 @@ -# NGINX CONFIGURATION FOR FASTCGI_CACHE EXCEPTION - WO v3.9.5 +# NGINX CONFIGURATION FOR FASTCGI_CACHE EXCEPTION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # do not cache xhtml request @@ -52,6 +52,11 @@ map $is_args $query_no_cache { "" 0; } +map $https $https_prefix { + default ""; + on "-https" +} + # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; diff --git a/wo/cli/templates/php.mustache b/wo/cli/templates/php.mustache index 59903a8..27c861d 100644 --- a/wo/cli/templates/php.mustache +++ b/wo/cli/templates/php.mustache @@ -1,4 +1,4 @@ -# PHP NGINX CONFIGURATION - WO v3.9.5 +# PHP NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE location / { try_files $uri $uri/ /index.php$is_args$args; @@ -6,5 +6,5 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } diff --git a/wo/cli/templates/php7.mustache b/wo/cli/templates/php7.mustache deleted file mode 100644 index 4005572..0000000 --- a/wo/cli/templates/php7.mustache +++ /dev/null @@ -1,10 +0,0 @@ -# PHP NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -location / { - try_files $uri $uri/ /index.php$is_args$args; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; -} diff --git a/wo/cli/templates/redis-php7.mustache b/wo/cli/templates/redis-php7.mustache deleted file mode 100644 index 5e21f7a..0000000 --- a/wo/cli/templates/redis-php7.mustache +++ /dev/null @@ -1,42 +0,0 @@ -# Redis NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf - -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - try_files $uri $uri/ /index.php$is_args$args; -} - -location /redis-fetch { - internal ; - set $redis_key $args; - redis_pass redis; -} -location /redis-store { - internal ; - set_unescape_uri $key $arg_key ; - redis2_query set $key $echo_request_body; - redis2_query expire $key 14400; - redis2_pass redis; -} - -location ~ \.php$ { - set $key "nginx-cache:$scheme$request_method$host$request_uri"; - try_files $uri =404; - - srcache_fetch_skip $skip_cache; - srcache_store_skip $skip_cache; - - srcache_response_cache_control off; - - set_escape_uri $escaped_key $key; - - srcache_fetch GET /redis-fetch $key; - srcache_store PUT /redis-store key=$escaped_key; - - more_set_headers 'X-SRCache-Fetch-Status $srcache_fetch_status'; - more_set_headers 'X-SRCache-Store-Status $srcache_store_status'; - - include fastcgi_params; - fastcgi_pass php73; -} diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index cdde0b1..dcbb0c1 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -1,4 +1,4 @@ -# Redis NGINX CONFIGURATION - WO v3.9.5 +# Redis NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf @@ -39,5 +39,5 @@ location ~ \.php$ { more_set_headers 'X-SRCache-Store-Status $srcache_store_status'; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } diff --git a/wo/cli/templates/upstream.mustache b/wo/cli/templates/upstream.mustache index 686a6a5..232dade 100644 --- a/wo/cli/templates/upstream.mustache +++ b/wo/cli/templates/upstream.mustache @@ -1,4 +1,4 @@ -# NGINX UPSTREAM CONFIGURATION - WO v3.9.5 +# NGINX UPSTREAM CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE #------------------------------- # PHP 5.6 diff --git a/wo/cli/templates/virtualconf-php7.mustache b/wo/cli/templates/virtualconf-php7.mustache index 625fd2a..56c1e96 100644 --- a/wo/cli/templates/virtualconf-php7.mustache +++ b/wo/cli/templates/virtualconf-php7.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}} + {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php73.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/virtualconf.mustache b/wo/cli/templates/virtualconf.mustache index d8610de..87d1100 100644 --- a/wo/cli/templates/virtualconf.mustache +++ b/wo/cli/templates/virtualconf.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}} + {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php72.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/webp.mustache b/wo/cli/templates/webp.mustache index 3ed187b..d53dc46 100644 --- a/wo/cli/templates/webp.mustache +++ b/wo/cli/templates/webp.mustache @@ -1,4 +1,4 @@ -# WEBP NGINX CONFIGURATION - WO v3.9.5 +# WEBP NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE map $http_accept $webp_suffix { diff --git a/wo/cli/templates/wpcommon-php7.mustache b/wo/cli/templates/wpcommon-php7.mustache deleted file mode 100644 index 85d2739..0000000 --- a/wo/cli/templates/wpcommon-php7.mustache +++ /dev/null @@ -1,70 +0,0 @@ -# WordPress COMMON SETTINGS - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# Limit access to avoid brute force attack -location = /wp-login.php { - limit_req zone=one burst=1 nodelay; - include fastcgi_params; - fastcgi_pass php73; -} -# Disable wp-config.txt -location = /wp-config.txt { - deny all; - access_log off; - log_not_found off; -} -location = /robots.txt { -# Some WordPress plugin gererate robots.txt file -# Refer #340 issue - try_files $uri $uri/ /index.php?$args @robots; - access_log off; - log_not_found off; -} -# fallback for robots.txt with default wordpress rules -location @robots { - return 200 "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n"; -} -# webp rewrite rules for jpg and png images -# try to load alternative image.png.webp before image.png -location /wp-content/uploads { - location ~ \.(png|jpe?g)$ { - add_header Vary "Accept-Encoding"; - add_header "Access-Control-Allow-Origin" "*"; - add_header Cache-Control "public, no-transform"; - access_log off; - log_not_found off; - expires max; - try_files $uri$webp_suffix $uri =404; - } - location ~ \.php$ { -#Prevent Direct Access Of PHP Files From Web Browsers - deny all; - } -} -# webp rewrite rules for EWWW testing image -location /wp-content/plugins/ewww-image-optimizer/images { - location ~ \.(png|jpe?g)$ { - add_header Vary "Accept-Encoding"; - add_header "Access-Control-Allow-Origin" "*"; - add_header Cache-Control "public, no-transform"; - access_log off; - log_not_found off; - expires max; - try_files $uri$webp_suffix $uri =404; - } - location ~ \.php$ { -#Prevent Direct Access Of PHP Files From Web Browsers - deny all; - } -} -# Deny access to any files with a .php extension in the uploads directory -# Works in sub-directory installs and also in multisite network -# Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban) -location ~* /(?:uploads|files)/.*\.php$ { - deny all; -} -# mitigate DoS attack CVE with WordPress script concatenation -# add the following line to wp-config.php -# define( 'CONCATENATE_SCRIPTS', false ); -location ~ \/wp-admin\/load-(scripts|styles).php { - deny all; -} diff --git a/wo/cli/templates/wpcommon.mustache b/wo/cli/templates/wpcommon.mustache index 7e6edb3..f646cd6 100644 --- a/wo/cli/templates/wpcommon.mustache +++ b/wo/cli/templates/wpcommon.mustache @@ -1,10 +1,10 @@ -# WordPress COMMON SETTINGS - WO v3.9.5 +# WordPress COMMON SETTINGS - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # Limit access to avoid brute force attack location = /wp-login.php { limit_req zone=one burst=1 nodelay; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } # Disable wp-config.txt location = /wp-config.txt { diff --git a/wo/cli/templates/wpfc-php7.mustache b/wo/cli/templates/wpfc-php7.mustache deleted file mode 100644 index 3667c40..0000000 --- a/wo/cli/templates/wpfc-php7.mustache +++ /dev/null @@ -1,21 +0,0 @@ -# WPFC NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf - -add_header X-fastcgi-cache $upstream_cache_status; -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - try_files $uri $uri/ /index.php$is_args$args; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; - fastcgi_cache_bypass $skip_cache; - fastcgi_no_cache $skip_cache; - fastcgi_cache WORDPRESS; -} -location ~ /purge(/.*) { - fastcgi_cache_purge WORDPRESS "$scheme$request_method$host$1"; - access_log off; -} diff --git a/wo/cli/templates/wpfc.mustache b/wo/cli/templates/wpfc.mustache index 447870f..5dbae7c 100644 --- a/wo/cli/templates/wpfc.mustache +++ b/wo/cli/templates/wpfc.mustache @@ -1,4 +1,4 @@ -# WPFC NGINX CONFIGURATION - WO v3.9.5 +# WPFC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf @@ -10,7 +10,7 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; fastcgi_cache_bypass $skip_cache; fastcgi_no_cache $skip_cache; fastcgi_cache WORDPRESS; diff --git a/wo/cli/templates/wprocket.mustache b/wo/cli/templates/wprocket.mustache new file mode 100644 index 0000000..e73f859 --- /dev/null +++ b/wo/cli/templates/wprocket.mustache @@ -0,0 +1,21 @@ +# WPSC NGINX CONFIGURATION - WO v3.9.7 +# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE +# $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf + +# Use cached or actual file if they exists, Otherwise pass request to WordPress +location / { + try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; +} +location ~ \.php$ { + try_files $uri =404; + include fastcgi_params; + fastcgi_pass {{upstream}}; +} + +location ~* /cache/wp-rocket/\.html$ { + etag on; + add_header Vary "Accept-Encoding, Cookie"; + access_log off; + log_not_found off; + expire 10h; +} \ No newline at end of file diff --git a/wo/cli/templates/wpsc-php7.mustache b/wo/cli/templates/wpsc-php7.mustache deleted file mode 100644 index 1fe48a6..0000000 --- a/wo/cli/templates/wpsc-php7.mustache +++ /dev/null @@ -1,17 +0,0 @@ -# WPSC NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf - -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - # If we add index.php?$args its break WooCommerce like plugins - # Ref: #330 - try_files /wp-content/cache/supercache/$http_host/$cache_uri/index.html $uri $uri/ /index.php; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; - # Following line is needed by WP Super Cache plugin - fastcgi_param SERVER_NAME $http_host; -} diff --git a/wo/cli/templates/wpsc.mustache b/wo/cli/templates/wpsc.mustache index cfe5070..ce5baa7 100644 --- a/wo/cli/templates/wpsc.mustache +++ b/wo/cli/templates/wpsc.mustache @@ -1,4 +1,4 @@ -# WPSC NGINX CONFIGURATION - WO v3.9.5 +# WPSC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf @@ -11,7 +11,7 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; # Following line is needed by WP Super Cache plugin fastcgi_param SERVER_NAME $http_host; } From 47ad8cd343d9e1fe3960516c05c27cec46447e3a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 19:29:04 +0200 Subject: [PATCH 021/102] add coverage with codacy --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58e4c7e..20a4f07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,13 @@ script: - unset LANG - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - - sudo apt-get install -qq --force-yes git python3-setuptools python3-dev python3-apt ccze tree + - sudo apt-get install -qq --force-yes git python3-setuptools python3-dev python3-apt python3-pip ccze tree + - export $CODACY_TOKEN - sudo bash install --travis + - python3 -m pip install coverage + - python3 -m pip install codacy-coverage + - coverage xml + - python-codacy-coverage -c $TRAVIS_COMMIT -d $TRAVIS_BUILD_DIR -r coverage.xml - sudo wo --help && sudo wo stack install && sudo wo stack install --proftpd - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc From ffb82d8d8bb070e528eba08306f724a2f3a44e7b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 21:48:14 +0200 Subject: [PATCH 022/102] Fix wprocket --- .travis.yml | 5 ----- wo/cli/plugins/site.py | 4 ++-- wo/cli/plugins/stack_pref.py | 14 ++++++++++++++ wo/cli/templates/map-wp.mustache | 10 +++++----- wo/cli/templates/nginx-core.mustache | 8 ++++---- wo/cli/templates/virtualconf-php7.mustache | 2 +- wo/cli/templates/virtualconf.mustache | 2 +- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20a4f07..1b45e50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,12 +30,7 @@ script: - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - sudo apt-get install -qq --force-yes git python3-setuptools python3-dev python3-apt python3-pip ccze tree - - export $CODACY_TOKEN - sudo bash install --travis - - python3 -m pip install coverage - - python3 -m pip install codacy-coverage - - coverage xml - - python-codacy-coverage -c $TRAVIS_COMMIT -d $TRAVIS_BUILD_DIR -r coverage.xml - sudo wo --help && sudo wo stack install && sudo wo stack install --proftpd - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 20948e4..ec63fd6 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1502,7 +1502,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - if ((oldcachetype in ['wpsc', 'basic', 'wpredis'] and + if ((oldcachetype in ['wpsc', 'basic', 'wpredis', 'wprocket'] and (data['wpfc'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpfc'])): try: @@ -1541,7 +1541,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - elif ((oldcachetype in ['wpsc', 'basic', 'wpfc'] and + elif ((oldcachetype in ['wpsc', 'basic', 'wpfc', 'wprocket'] and (data['wpredis'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpredis'])): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index df9bf9c..b7c84fb 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -173,6 +173,20 @@ def post_pref(self, apt_packages, packages): # Nginx configuration if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + # Nginx main configuration + if os.path.isfile('/etc/nginx/nginx.conf'): + if (WOVariables.wo_distro == 'ubuntu' or + WOVariables.wo_platform_codename == 'buster'): + data = dict(tls13=True) + else: + data = dict(tls13=False) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/nginx.conf') + wo_nginx = open('/etc/nginx/nginx.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'nginx-core.mustache', out=wo_nginx) + wo_nginx.close() # Fix for white screen death with NGINX PLUS if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index f764020..87be917 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -52,11 +52,6 @@ map $is_args $query_no_cache { "" 0; } -map $https $https_prefix { - default ""; - on "-https" -} - # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; @@ -68,3 +63,8 @@ map $skip_cache $cache_uri { 0 $request_uri; default 'null cache'; } + +map $https $https_prefix { + default ""; + on "-https" +} diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index dd4631e..4201f52 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -66,12 +66,12 @@ http { ssl_session_cache shared:SSL:50m; ssl_session_tickets off; ssl_prefer_server_ciphers on; - ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; - ssl_protocols TLSv1.2 TLSv1.3; + {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; + ssl_protocols TLSv1.2 TLSv1.3;{{/tls13}} ssl_ecdh_curve X25519:P-521:P-384:P-256; # Previous TLS v1.2 configuration - # ssl_protocols TLSv1.2; - # ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES; + {{^tls13}}ssl_protocols TLSv1.2; + ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES;{{/tls13}} # Common security headers more_set_headers "X-Frame-Options : SAMEORIGIN"; diff --git a/wo/cli/templates/virtualconf-php7.mustache b/wo/cli/templates/virtualconf-php7.mustache index 56c1e96..fa137cb 100644 --- a/wo/cli/templates/virtualconf-php7.mustache +++ b/wo/cli/templates/virtualconf-php7.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf{{/wprocket}} + {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf;{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php73.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/virtualconf.mustache b/wo/cli/templates/virtualconf.mustache index 87d1100..19a1afb 100644 --- a/wo/cli/templates/virtualconf.mustache +++ b/wo/cli/templates/virtualconf.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf{{/wprocket}} + {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf;{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php72.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} From e2b2db22b971585e41a065d4390f9da62ee4f230 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 00:31:47 +0200 Subject: [PATCH 023/102] Fix typo --- wo/cli/plugins/stack_pref.py | 2 +- wo/cli/templates/map-wp.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index b7c84fb..06bb82f 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -535,7 +535,6 @@ def post_pref(self, apt_packages, packages): Log.error( self, "Failed to generate HTTPS " "certificate for 22222") - server_ip = requests.get('http://v4.wordops.eu') if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' .format(WOVariables.wo_webroot)): @@ -553,6 +552,7 @@ def post_pref(self, apt_packages, packages): WOService.reload_service(self, 'nginx') if set(["nginx"]).issubset(set(apt_packages)): + server_ip = requests.get('http://v4.wordops.eu') print("WordOps backend configuration was successful\n" "You can access it on : https://{0}:22222" .format(server_ip)) diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 87be917..00d21da 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -66,5 +66,5 @@ map $skip_cache $cache_uri { map $https $https_prefix { default ""; - on "-https" + on "-https"; } From 13ebe2af3f8106aba5dc27a2a60c581b26971475 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 01:31:51 +0200 Subject: [PATCH 024/102] fix server_ip --- wo/cli/plugins/stack_pref.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 06bb82f..3726783 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -550,9 +550,8 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") WOService.reload_service(self, 'nginx') - + server_ip = requests.get('http://v4.wordops.eu') if set(["nginx"]).issubset(set(apt_packages)): - server_ip = requests.get('http://v4.wordops.eu') print("WordOps backend configuration was successful\n" "You can access it on : https://{0}:22222" .format(server_ip)) From d0799a08e710e625eeb71e38b108b65204c32053 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 01:49:13 +0200 Subject: [PATCH 025/102] export logs with travis --- .travis.yml | 2 ++ install | 4 ++-- setup.py | 2 +- wo/cli/plugins/site_functions.py | 9 --------- wo/core/variables.py | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b45e50..5bd7075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,5 +54,7 @@ script: - sudo wo stack upgrade --composer --no-prompt - sudo wo update --travis - sudo wo stack status + - sudo tar -I pigz -cf /var/log/wo + - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge diff --git a/install b/install index 08452d3..17c7243 100755 --- a/install +++ b/install @@ -7,10 +7,10 @@ # Copyright (c) 2019 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- -# Version 3.9.7 - 2019-08-02 +# Version 3.9.7.1 - 2019-08-06 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" -readonly wo_version_new="3.9.7" +readonly wo_version_new="3.9.7.1" # CONTENTS # --- # 1. VARIABLES AND DECLARATIONS diff --git a/setup.py b/setup.py index dff04ab..c05b21e 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if not os.path.isfile('/root/.gitconfig'): shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') setup(name='wo', - version='3.9.7', + version='3.9.7.1', description=long_description, long_description=long_description, classifiers=[], diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index dd66040..a37f7bb 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -957,15 +957,6 @@ def site_package_check(self, stype): out=wo_nginx) wo_nginx.close() - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/upstream.conf", - "php73"): - with open("/etc/nginx/conf.d/upstream.conf", "a") as php_file: - php_file.write("upstream php73 {\nserver" - "unix:/var/run/php/php73-fpm.sock;\n}\n" - "upstream debug73" - " {\nserver 127.0.0.1:9173;\n}\n") - return(stack.install(apt_packages=apt_packages, packages=packages, disp_msg=False)) diff --git a/wo/core/variables.py b/wo/core/variables.py index cf3474e..f565cff 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -10,7 +10,7 @@ class WOVariables(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.7" + wo_version = "3.9.7.1" # WordOps packages versions wo_wp_cli = "2.2.0" wo_adminer = "4.7.2" From 89ebbe9cd622e9105775aa857f4caa87750a0a1f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 11:37:03 +0200 Subject: [PATCH 026/102] Fix for redis and variable glob --- wo/cli/plugins/stack_pref.py | 18 +++++++++--------- wo/cli/plugins/sync.py | 1 + wo/cli/templates/redis.mustache | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 3726783..f6700e6 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -536,15 +536,15 @@ def post_pref(self, apt_packages, packages): self, "Failed to generate HTTPS " "certificate for 22222") - if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' - .format(WOVariables.wo_webroot)): + if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' + .format(WOVariables.wo_webroot)): - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "a") as php_file: - php_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "a") as php_file: + php_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") # Nginx Configation into GIT WOGit.add(self, @@ -574,7 +574,7 @@ def post_pref(self, apt_packages, packages): # create nginx configuration for redis if set(WOVariables.wo_redis).issubset(set(apt_packages)): if os.path.isdir('/etc/nginx/common'): - data = dict() + data = dict(upstream="php72") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php72.conf') wo_nginx = open('/etc/nginx/common/redis-php72.conf', diff --git a/wo/cli/plugins/sync.py b/wo/cli/plugins/sync.py index d646544..8495bec 100644 --- a/wo/cli/plugins/sync.py +++ b/wo/cli/plugins/sync.py @@ -4,6 +4,7 @@ from wo.core.fileutils import WOFileUtils from wo.cli.plugins.sitedb import * from wo.core.mysql import * from wo.core.logging import Log +import glob def wo_sync_hook(app): diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index dcbb0c1..88a49e0 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -10,7 +10,7 @@ location / { location /redis-fetch { internal ; set $redis_key $args; - redis_pass redis; + redis_pass redis; } location /redis-store { internal ; From 9e4cdbd05b0c795d19f2c1a8510ac882b32125d1 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:10:50 +0200 Subject: [PATCH 027/102] Additional fix for redis --- wo/cli/plugins/site_functions.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index a37f7bb..d3b5d1a 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -290,7 +290,7 @@ def setupwordpress(self, data): Log.debug(self, "Setting up wp-config file") if not data['multisite']: Log.debug(self, "Generating wp-config for WordPress Single site") - Log.debug(self, "bash -c \"php {0} --allow-root " + Log.debug(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' --dbuser=\'{2}\' " @@ -302,7 +302,7 @@ def setupwordpress(self, data): .format(data['wo_db_pass'], "\n\ndefine(\'WP_DEBUG\', false);")) try: - if WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root" + if WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root" .format(WOVariables.wo_wpcli_path) + " config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' " @@ -325,7 +325,7 @@ def setupwordpress(self, data): raise SiteError("generate wp-config failed for wp single site") else: Log.debug(self, "Generating wp-config for WordPress multisite") - Log.debug(self, "bash -c \"php {0} --allow-root " + Log.debug(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' --dbhost=\'{2}\' " @@ -340,7 +340,7 @@ def setupwordpress(self, data): " false);", "\n\ndefine(\'WP_DEBUG\', false);")) try: - if WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root" + if WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root" .format(WOVariables.wo_wpcli_path) + " config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' " @@ -365,36 +365,36 @@ def setupwordpress(self, data): try: - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_CACHE_KEY_SALT " "\'{0}:\'\"".format(wo_domain_name)) - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_MEMORY_LIMIT " "\'128M\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_MAX_MEMORY_LIMIT " "\'256M\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set CONCATENATE_SCRIPTS " "false --raw\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_POST_REVISIONS " "\'10\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set MEDIA_TRASH " "true --raw\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set EMPTY_TRASH_DAYS " "\'15\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_AUTO_UPDATE_CORE " "minor\"") @@ -456,14 +456,14 @@ def setupwordpress(self, data): if not data['multisite']: Log.debug(self, "Creating tables for WordPress Single site") - Log.debug(self, "php {0} --allow-root core install " + Log.debug(self, "{0} --allow-root core install " .format(WOVariables.wo_wpcli_path) + "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " .format(data['www_domain'], wo_wp_user) + "--admin_password= --admin_email=\'{1}\'" .format(wo_wp_pass, wo_wp_email)) try: - if WOShellExec.cmd_exec(self, "php {0} --allow-root core " + if WOShellExec.cmd_exec(self, "{0} --allow-root core " .format(WOVariables.wo_wpcli_path) + "install --url=\'{0}\' --title=\'{0}\' " "--admin_name=\'{1}\' " @@ -480,7 +480,7 @@ def setupwordpress(self, data): raise SiteError("setup WordPress tables failed for single site") else: Log.debug(self, "Creating tables for WordPress multisite") - Log.debug(self, "php {0} --allow-root " + Log.debug(self, "{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "core multisite-install " "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " @@ -491,7 +491,7 @@ def setupwordpress(self, data): subdomains='--subdomains' if not data['wpsubdir'] else '')) try: - if WOShellExec.cmd_exec(self, "php {0} --allow-root " + if WOShellExec.cmd_exec(self, "{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "core multisite-install " "--url=\'{0}\' --title=\'{0}\' " @@ -513,7 +513,7 @@ def setupwordpress(self, data): Log.debug(self, "Updating WordPress permalink") try: - WOShellExec.cmd_exec(self, " php {0} --allow-root " + WOShellExec.cmd_exec(self, " {0} --allow-root " .format(WOVariables.wo_wpcli_path) + "rewrite structure " "/%year%/%monthnum%/%day%/%postname%/") @@ -619,7 +619,7 @@ def installwp_plugin(self, plugin_name, data): .format(plugin_name)) WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) try: - WOShellExec.cmd_exec(self, "php {0} plugin " + WOShellExec.cmd_exec(self, "{0} plugin " .format(WOVariables.wo_wpcli_path) + "--allow-root install " "{0}".format(plugin_name)) @@ -628,7 +628,7 @@ def installwp_plugin(self, plugin_name, data): raise SiteError("plugin installation failed") try: - WOShellExec.cmd_exec(self, "php {0} plugin " + WOShellExec.cmd_exec(self, "{0} plugin " .format(WOVariables.wo_wpcli_path) + "--allow-root activate " "{0} {na}" @@ -856,7 +856,7 @@ def site_package_check(self, stype): if (os.path.isfile("/etc/nginx/nginx.conf") and not os.path.isfile("/etc/nginx/common/redis-php72.conf")): - data = dict() + data = dict(upstream="php72") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php72.conf') wo_nginx = open('/etc/nginx/common/redis-php72.conf', From e2bd566c2081ff6b902cea56de5fd6f8ed5125fc Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:42:00 +0200 Subject: [PATCH 028/102] Fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5bd7075..904b818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ script: - sudo wo stack upgrade --composer --no-prompt - sudo wo update --travis - sudo wo stack status - - sudo tar -I pigz -cf /var/log/wo + - sudo tar -I pigz -cf wordops.tar.gz /var/log/wo - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge From 07175a673782c1671ca6c6a47096dee8f028c524 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:56:24 +0200 Subject: [PATCH 029/102] Final fix for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 904b818..f484bef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,6 @@ script: - sudo wo update --travis - sudo wo stack status - sudo tar -I pigz -cf wordops.tar.gz /var/log/wo - - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" + - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "wordops.tar.gz") && echo "" | sudo tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge From 553cc226bdd6916a7e717ab87a18bdfbe5d8dadc Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 02:34:12 +0200 Subject: [PATCH 030/102] Fix WP download --- wo/cli/plugins/site_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d3b5d1a..d9233a1 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -240,7 +240,7 @@ def setupwordpress(self, data): # Random characters wo_random = (''.join(random.sample(string.ascii_uppercase + string.ascii_lowercase + - string.digits, 15))) + string.digits, 24))) wo_wp_prefix = '' # wo_wp_user = '' # wo_wp_pass = '' From 637cd1660a4334890965449ea4c28a73753c73e1 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 02:45:26 +0200 Subject: [PATCH 031/102] Reduce WO size --- wo/cli/plugins/clean.py | 17 +-- wo/cli/plugins/secure.py | 49 ++++---- wo/cli/plugins/stack_pref.py | 3 +- wo/cli/plugins/stack_services.py | 185 ++++++++++++++++--------------- wo/cli/plugins/stack_upgrade.py | 63 +++++------ wo/cli/plugins/update.py | 9 +- 6 files changed, 169 insertions(+), 157 deletions(-) diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 1a8643b..589bf80 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -37,22 +37,23 @@ class WOCleanController(CementBaseController): @expose(hide=True) def default(self): - if (not (self.app.pargs.all or self.app.pargs.fastcgi or - self.app.pargs.memcached or self.app.pargs.opcache or - self.app.pargs.redis)): + pargs = self.app.pargs + if (not (pargs.all or pargs.fastcgi or + pargs.memcached or pargs.opcache or + pargs.redis)): self.clean_fastcgi() - if self.app.pargs.all: + if pargs.all: self.clean_memcached() self.clean_fastcgi() self.clean_opcache() self.clean_redis() - if self.app.pargs.fastcgi: + if pargs.fastcgi: self.clean_fastcgi() - if self.app.pargs.memcached: + if pargs.memcached: self.clean_memcached() - if self.app.pargs.opcache: + if pargs.opcache: self.clean_opcache() - if self.app.pargs.redis: + if pargs.redis: self.clean_redis() @expose(hide=True) diff --git a/wo/cli/plugins/secure.py b/wo/cli/plugins/secure.py index c9a7de6..119a31b 100644 --- a/wo/cli/plugins/secure.py +++ b/wo/cli/plugins/secure.py @@ -35,31 +35,33 @@ class WOSecureController(CementBaseController): @expose(hide=True) def default(self): - if self.app.pargs.auth: + pargs = self.app.pargs + if pargs.auth: self.secure_auth() - if self.app.pargs.port: + if pargs.port: self.secure_port() - if self.app.pargs.ip: + if pargs.ip: self.secure_ip() @expose(hide=True) def secure_auth(self): """This function secures authentication""" + pargs = self.app.pargs passwd = ''.join([random.choice (string.ascii_letters + string.digits) for n in range(24)]) - if not self.app.pargs.user_input: + if not pargs.user_input: username = input("Provide HTTP authentication user " "name [{0}] :".format(WOVariables.wo_user)) - self.app.pargs.user_input = username + pargs.user_input = username if username == "": - self.app.pargs.user_input = WOVariables.wo_user - if not self.app.pargs.user_pass: + pargs.user_input = WOVariables.wo_user + if not pargs.user_pass: password = getpass.getpass("Provide HTTP authentication " "password [{0}] :".format(passwd)) - self.app.pargs.user_pass = password + pargs.user_pass = password if password == "": - self.app.pargs.user_pass = passwd + pargs.user_pass = passwd Log.debug(self, "printf username:" "$(openssl passwd -crypt " "password 2> /dev/null)\n\"" @@ -68,8 +70,8 @@ class WOSecureController(CementBaseController): "$(openssl passwd -crypt " "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo 2>/dev/null" - .format(username=self.app.pargs.user_input, - password=self.app.pargs.user_pass), + .format(username=pargs.user_input, + password=pargs.user_pass), log=False) WOGit.add(self, ["/etc/nginx"], msg="Adding changed secure auth into Git") @@ -77,41 +79,42 @@ class WOSecureController(CementBaseController): @expose(hide=True) def secure_port(self): """This function Secures port""" - if self.app.pargs.user_input: - while not self.app.pargs.user_input.isdigit(): + pargs = self.app.pargs + if pargs.user_input: + while not pargs.user_input.isdigit(): Log.info(self, "Please enter a valid port number ") - self.app.pargs.user_input = input("WordOps " + pargs.user_input = input("WordOps " "admin port [22222]:") - if not self.app.pargs.user_input: + if not pargs.user_input: port = input("WordOps admin port [22222]:") if port == "": - self.app.pargs.user_input = 22222 + pargs.user_input = 22222 while not port.isdigit() and port != "": Log.info(self, "Please Enter valid port number :") port = input("WordOps admin port [22222]:") - self.app.pargs.user_input = port + pargs.user_input = port WOShellExec.cmd_exec(self, "sed -i \"s/listen.*/listen " "{port} default_server ssl http2;/\" " "/etc/nginx/sites-available/22222" - .format(port=self.app.pargs.user_input)) + .format(port=pargs.user_input)) WOGit.add(self, ["/etc/nginx"], msg="Adding changed secure port into Git") if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") Log.info(self, "Successfully port changed {port}" - .format(port=self.app.pargs.user_input)) + .format(port=pargs.user_input)) @expose(hide=True) def secure_ip(self): """IP whitelisting""" - newlist = [] - if not self.app.pargs.user_input: + pargs = self.app.pargs + if not pargs.user_input: ip = input("Enter the comma separated IP addresses " "to white list [127.0.0.1]:") - self.app.pargs.user_input = ip + pargs.user_input = ip try: - user_ip = self.app.pargs.user_input.split(',') + user_ip = pargs.user_input.split(',') except Exception as e: Log.debug(self, "{0}".format(e)) user_ip = ['127.0.0.1'] diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index f6700e6..c308c24 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1235,7 +1235,8 @@ def post_pref(self, apt_packages, packages): '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "/usr/local/bin/composer update --no-plugins --no-scripts " + WOShellExec.cmd_exec(self, "/usr/local/bin/composer update " + "--no-plugins --no-scripts " "-n --no-dev -d " "/var/www/22222/htdocs/db/pma/") WOFileUtils.chown(self, '{0}22222/htdocs/db/pma' diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 56985b8..86796bc 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -18,24 +18,25 @@ class WOStackStatusController(CementBaseController): def start(self): """Start services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.redis or - self.app.pargs.fail2ban or - self.app.pargs.proftpd or - self.app.pargs.netdata): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.redis or + pargs.fail2ban or + pargs.proftpd or + pargs.netdata): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -45,13 +46,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -64,27 +65,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -98,26 +99,27 @@ class WOStackStatusController(CementBaseController): def stop(self): """Stop services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.fail2ban or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.fail2ban or + pargs.netdata or + pargs.proftpd or + pargs.redis): + pargs.nginx = True + pargs.php = True + pargs.mysql = True # nginx - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") # php7.2 - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -129,14 +131,14 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") # php7.3 - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") # mysql - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -150,28 +152,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") # redis - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -185,24 +187,25 @@ class WOStackStatusController(CementBaseController): def restart(self): """Restart services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -213,13 +216,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if ((WOAptGet.is_installed(self, 'mysql-server') or @@ -233,27 +236,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -267,24 +270,25 @@ class WOStackStatusController(CementBaseController): def status(self): """Status of services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -295,13 +299,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -314,27 +318,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -348,25 +352,26 @@ class WOStackStatusController(CementBaseController): def reload(self): """Reload service""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom') or WOAptGet.is_installed(self, 'nginx-mainline')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -377,13 +382,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -396,27 +401,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index a834857..5b84e22 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -57,35 +57,36 @@ class WOStackUpgradeController(CementBaseController): apt_packages = [] packages = [] empty_packages = [] + pargs = pargs = self.app.pargs - if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and - (not self.app.pargs.php) and (not self.app.pargs.php73) and - (not self.app.pargs.mysql) and - (not self.app.pargs.all) and (not self.app.pargs.wpcli) and - (not self.app.pargs.netdata) and (not self.app.pargs.composer) and - (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.redis)): - self.app.pargs.web = True + if ((not pargs.web) and (not pargs.nginx) and + (not pargs.php) and (not pargs.php73) and + (not pargs.mysql) and + (not pargs.all) and (not pargs.wpcli) and + (not pargs.netdata) and (not pargs.composer) and + (not pargs.phpmyadmin) and + (not pargs.redis)): + pargs.web = True - if self.app.pargs.all: - self.app.pargs.web = True + if pargs.all: + pargs.web = True - if self.app.pargs.web: + if pargs.web: if WOAptGet.is_installed(self, 'nginx-custom'): - self.app.pargs.nginx = True + pargs.nginx = True else: Log.info(self, "Nginx is not already installed") - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): apt_packages = apt_packages + WOVariables.wo_nginx else: Log.info(self, "Nginx Stable is not already installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): if not WOAptGet.is_installed(self, 'php7.3-fpm'): apt_packages = apt_packages + WOVariables.wo_php + \ @@ -95,7 +96,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): if not WOAptGet.is_installed(self, 'php7.2-fpm'): apt_packages = apt_packages + WOVariables.wo_php73 + \ @@ -105,19 +106,19 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.3 is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if WOAptGet.is_installed(self, 'mariadb-server'): apt_packages = apt_packages + WOVariables.wo_mysql else: Log.info(self, "MariaDB is not installed") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis else: Log.info(self, "Redis is not installed") - if self.app.pargs.wpcli: + if pargs.wpcli: if os.path.isfile('/usr/local/bin/wp'): packages = packages + [["https://github.com/wp-cli/wp-cli/" "releases/download/v{0}/" @@ -128,13 +129,13 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "WPCLI is not installed with WordOps") - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir('/opt/netdata'): packages = packages + [['https://my-netdata.io/' 'kickstart-static64.sh', '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: if os.path.isdir('/var/www/22222/htdocs/db/pma'): packages = packages + \ [["https://files.phpmyadmin.net" @@ -147,7 +148,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.error(self, "phpMyAdmin isn't installed") - if self.app.pargs.composer: + if pargs.composer: if os.path.isfile('/usr/local/bin/composer'): packages = packages + [["https://getcomposer.org/installer", "/var/lib/wo/tmp/composer-install", @@ -160,7 +161,7 @@ class WOStackUpgradeController(CementBaseController): Log.info(self, "During package update process non nginx-cached" " parts of your site may remain down") # Check prompt - if (not self.app.pargs.no_prompt): + if (not pargs.no_prompt): start_upgrade = input("Do you want to continue:[y/N]") if start_upgrade != "Y" and start_upgrade != "y": Log.error(self, "Not starting package update") @@ -191,25 +192,25 @@ class WOStackUpgradeController(CementBaseController): WOService.restart_service(self, 'redis-server') if len(packages): - if self.app.pargs.wpcli: + if pargs.wpcli: WOFileUtils.remove(self, ['/usr/local/bin/wp']) - if self.app.pargs.netdata: + if pargs.netdata: WOFileUtils.remove(self, ['/var/lib/wo/tmp/kickstart.sh']) Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) - if self.app.pargs.wpcli: + if pargs.wpcli: WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - if self.app.pargs.netdata: + if pargs.netdata: Log.info(self, "Upgrading Netdata, please wait...") WOShellExec.cmd_exec(self, "/bin/bash /var/lib/wo/tmp/" "kickstart.sh " "--dont-wait") - if self.app.pargs.composer: + if pargs.composer: Log.info(self, "Upgrading Composer, please wait...") WOShellExec.cmd_exec(self, "php -q /var/lib/wo" "/tmp/composer-install " @@ -218,7 +219,7 @@ class WOStackUpgradeController(CementBaseController): '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: Log.info(self, "Upgrading phpMyAdmin, please wait...") WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 04e5344..b507170 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -36,20 +36,21 @@ class WOUpdateController(CementBaseController): @expose(hide=True) def default(self): + pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") - if self.app.pargs.travis: + if pargs.travis: wo_branch = "updating-configuration" install_args = "--travis --force " - elif self.app.pargs.beta: + elif pargs.beta: wo_branch = "beta" install_args = "" else: wo_branch = "master" install_args = "" - if self.app.pargs.force: + if pargs.force: install_args = install_args + "--force " - if self.app.pargs.preserve: + if pargs.preserve: install_args = install_args + "--preserve " WODownload.download(self, [["https://raw.githubusercontent.com/" From f3a16e900f914be3a58f366883b92296b1846372 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 03:05:32 +0200 Subject: [PATCH 032/102] Fix redis stack install --- wo/cli/plugins/site.py | 2 +- wo/cli/plugins/stack.py | 387 +++++++++++++++-------------------- wo/cli/plugins/stack_pref.py | 59 ++++++ 3 files changed, 228 insertions(+), 220 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index ec63fd6..41538a0 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -867,7 +867,7 @@ class WOSiteUpdateController(CementBaseController): pargs = self.app.pargs if pargs.php72: - self.app.pargs.php = True + pargs.php = True if pargs.all: if pargs.site_name: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 7c75c30..9ce0a04 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -86,6 +86,9 @@ class WOStackController(CementBaseController): dict(help='Install phpRedisAdmin', action='store_true')), (['--proftpd'], dict(help='Install ProFTPd', action='store_true')), + (['--force'], + dict(help='Force install/remove/purge without prompt', + action='store_true')), ] usage = "wo stack (command) [options]" @@ -98,63 +101,64 @@ class WOStackController(CementBaseController): def install(self, packages=[], apt_packages=[], disp_msg=True): """Start installation of packages""" self.msg = [] + pargs = self.app.pargs try: # Default action for stack installation - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.mysql) and (not self.app.pargs.wpcli) and - (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.composer) and - (not self.app.pargs.netdata) and - (not self.app.pargs.dashboard) and - (not self.app.pargs.fail2ban) and - (not self.app.pargs.security) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.redis) and (not self.app.pargs.proftpd) and - (not self.app.pargs.phpredisadmin) and - (not self.app.pargs.php73)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.mysql) and (not pargs.wpcli) and + (not pargs.phpmyadmin) and + (not pargs.composer) and + (not pargs.netdata) and + (not pargs.dashboard) and + (not pargs.fail2ban) and + (not pargs.security) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.redis) and (not pargs.proftpd) and + (not pargs.phpredisadmin) and + (not pargs.php73)): + pargs.web = True + pargs.admin = True + pargs.security = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True - self.app.pargs.redis = True - self.app.pargs.proftpd = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True + pargs.redis = True + pargs.proftpd = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.admin: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.composer = True - self.app.pargs.utils = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True + if pargs.admin: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.adminer = True + pargs.phpmyadmin = True + pargs.composer = True + pargs.utils = True + pargs.netdata = True + pargs.dashboard = True + pargs.phpredisadmin = True - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # Redis - if self.app.pargs.redis: + if pargs.redis: if not WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis - self.app.pargs.php = True + pargs.php = True else: Log.info(self, "Redis already installed") # Nginx - if self.app.pargs.nginx: + if pargs.nginx: Log.debug(self, "Setting apt_packages variable for Nginx") if not (WOAptGet.is_installed(self, 'nginx-custom')): @@ -177,7 +181,7 @@ class WOStackController(CementBaseController): Log.debug(self, "Nginx Stable already installed") # PHP 7.2 - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Setting apt_packages variable for PHP 7.2") if not (WOAptGet.is_installed(self, 'php7.2-fpm')): if not (WOAptGet.is_installed(self, 'php7.3-fpm')): @@ -190,7 +194,7 @@ class WOStackController(CementBaseController): Log.info(self, "PHP 7.2 already installed") # PHP 7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Setting apt_packages variable for PHP 7.3") if not WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -203,7 +207,7 @@ class WOStackController(CementBaseController): Log.info(self, "PHP 7.3 already installed") # MariaDB 10.3 - if self.app.pargs.mysql: + if pargs.mysql: Log.debug(self, "Setting apt_packages variable for MySQL") if not WOShellExec.cmd_exec(self, "mysqladmin ping"): apt_packages = apt_packages + WOVariables.wo_mysql @@ -219,7 +223,7 @@ class WOStackController(CementBaseController): Log.info(self, "MySQL connection is already alive") # WP-CLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Setting packages variable for WP-CLI") if not WOShellExec.cmd_exec(self, "command -v wp"): packages = packages + [["https://github.com/wp-cli/wp-cli/" @@ -233,7 +237,7 @@ class WOStackController(CementBaseController): Log.info(self, "WP-CLI is already installed") # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: Log.debug(self, "Setting apt_packages variable for Fail2ban") if not WOAptGet.is_installed(self, 'fail2ban'): apt_packages = apt_packages + WOVariables.wo_fail2ban @@ -242,7 +246,7 @@ class WOStackController(CementBaseController): Log.info(self, "Fail2ban already installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: Log.debug(self, "Setting apt_packages variable for ProFTPd") if not WOAptGet.is_installed(self, 'proftpd-basic'): apt_packages = apt_packages + ["proftpd-basic"] @@ -251,11 +255,11 @@ class WOStackController(CementBaseController): Log.info(self, "ProFTPd already installed") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: if not os.path.isdir('/var/www/22222/htdocs/db/pma'): Log.debug(self, "Setting packages variable " "for phpMyAdmin ") - self.app.pargs.composer = True + pargs.composer = True packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", "/var/lib/wo/tmp/pma.tar.gz", @@ -265,7 +269,7 @@ class WOStackController(CementBaseController): Log.info(self, "phpMyAdmin already installed") # Composer - if self.app.pargs.composer: + if pargs.composer: if not os.path.isfile('/usr/local/bin/composer'): Log.debug(self, "Setting packages variable for Composer ") packages = packages + [["https://getcomposer.org/" @@ -277,12 +281,12 @@ class WOStackController(CementBaseController): Log.info(self, "Composer already installed") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: if not os.path.isdir('/var/www/22222/htdocs/' 'cache/redis/phpRedisAdmin'): Log.debug( self, "Setting packages variable for phpRedisAdmin") - self.app.pargs.composer = True + pargs.composer = True packages = packages + [["https://github.com/" "erikdubbelboer/" "phpRedisAdmin/archive" @@ -294,7 +298,7 @@ class WOStackController(CementBaseController): Log.info(self, "phpRedisAdmin already installed") # ADMINER - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Setting packages variable for Adminer ") packages = packages + [["https://github.com/vrana/adminer/" "releases/download/v{0}" @@ -313,7 +317,7 @@ class WOStackController(CementBaseController): "Adminer theme"]] # Netdata - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Setting packages variable for Netdata") if not os.path.exists('/opt/netdata'): packages = packages + [['https://my-netdata.io/' @@ -325,7 +329,7 @@ class WOStackController(CementBaseController): Log.info(self, "Netdata already installed") # WordOps Dashboard - if self.app.pargs.dashboard: + if pargs.dashboard: if not os.path.isfile('/var/www/22222/htdocs/index.php'): Log.debug( self, "Setting packages variable for WO-Dashboard") @@ -345,7 +349,7 @@ class WOStackController(CementBaseController): Log.info(self, "WordOps dashboard already installed") # UTILS - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Setting packages variable for utils") packages = packages + [["https://raw.githubusercontent.com" "/rtCamp/eeadmin/master/cache/nginx/" @@ -408,63 +412,6 @@ class WOStackController(CementBaseController): WODownload.download(self, packages) Log.debug(self, "Calling post_pref") post_pref(self, apt_packages, packages) - if 'redis-server' in apt_packages: - # set redis.conf parameter - # set maxmemory 10% for ram below 512MB and 20% for others - # set maxmemory-policy allkeys-lru - # enable systemd service - Log.debug(self, "Enabling redis systemd service") - WOShellExec.cmd_exec(self, "systemctl enable redis-server") - if os.path.isfile("/etc/redis/redis.conf"): - wo_ram = psutil.virtual_memory().total / (1024 * 1024) - if wo_ram < 1024: - Log.debug(self, "Setting maxmemory variable to " - "{0} in redis.conf" - .format(int(wo_ram*1024*1024*0.1))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable to " - "allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - Log.debug( - self, "Setting tcp-backlog variable to " - "in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "tcp-backlog 511", - "tcp-backlog 32768") - - WOService.restart_service(self, 'redis-server') - else: - Log.debug(self, "Setting maxmemory variable to {0} " - "in redis.conf" - .format(int(wo_ram*1024*1024*0.2))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable " - "to allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - WOService.restart_service(self, 'redis-server') if disp_msg: if (self.msg): @@ -479,46 +426,46 @@ class WOStackController(CementBaseController): """Start removal of packages""" apt_packages = [] packages = [] + pargs = self.app.pargs + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.php73) and (not pargs.mysql) and + (not pargs.wpcli) and (not pargs.phpmyadmin) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.composer) and (not pargs.netdata) and + (not pargs.fail2ban) and (not pargs.proftpd) and + (not pargs.security) and + (not pargs.all) and (not pargs.redis) and + (not pargs.phpredisadmin)): + pargs.web = True + pargs.admin = True + pargs.security = True - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.php73) and (not self.app.pargs.mysql) and - (not self.app.pargs.wpcli) and (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.composer) and (not self.app.pargs.netdata) and - (not self.app.pargs.fail2ban) and (not self.app.pargs.proftpd) and - (not self.app.pargs.security) and - (not self.app.pargs.all) and (not self.app.pargs.redis) and - (not self.app.pargs.phpredisadmin)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.admin: + pargs.composer = True + pargs.utils = True + pargs.netdata = True + if os.path.isdir('{0}22222/htdocs' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/*' + .format(WOVariables.wo_webroot)] - if self.app.pargs.admin: - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.composer = True - self.app.pargs.utils = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True - - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # NGINX - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): Log.debug(self, "Removing apt_packages variable of Nginx") apt_packages = apt_packages + WOVariables.wo_nginx @@ -526,7 +473,7 @@ class WOStackController(CementBaseController): Log.error(self, "Cannot Remove! Nginx Stable " "version not found.") # PHP 7.2 - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Removing apt_packages variable of PHP") if WOAptGet.is_installed(self, 'php7.2-fpm'): if not WOAptGet.is_installed(self, 'php7.3-fpm'): @@ -538,7 +485,7 @@ class WOStackController(CementBaseController): Log.error(self, "PHP 7.2 not found") # PHP7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Removing apt_packages variable of PHP 7.3") if WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -550,18 +497,18 @@ class WOStackController(CementBaseController): Log.error(self, "PHP 7.3 not found") # REDIS - if self.app.pargs.redis: + if pargs.redis: Log.debug(self, "Remove apt_packages variable of Redis") apt_packages = apt_packages + WOVariables.wo_redis # MariaDB - if self.app.pargs.mysql: + if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") apt_packages = apt_packages + WOVariables.wo_mysql packages = packages + ['/usr/bin/mysqltuner'] # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): Log.debug(self, "Remove apt_packages variable of Fail2ban") apt_packages = apt_packages + WOVariables.wo_fail2ban @@ -569,7 +516,7 @@ class WOStackController(CementBaseController): Log.error(self, "Fail2ban not found") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): Log.debug(self, "Remove apt_packages variable for ProFTPd") apt_packages = apt_packages + ["proftpd-basic"] @@ -577,19 +524,19 @@ class WOStackController(CementBaseController): Log.error(self, "ProFTPd not found") # WPCLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Removing package variable of WPCLI ") if os.path.isfile('/usr/local/bin/wp'): packages = packages + ['/usr/local/bin/wp'] else: Log.warn(self, "WP-CLI is not installed with WordOps") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: Log.debug(self, "Removing package variable of phpMyAdmin ") packages = packages + ['{0}22222/htdocs/db/pma' .format(WOVariables.wo_webroot)] # Composer - if self.app.pargs.composer: + if pargs.composer: Log.debug(self, "Removing package variable of Composer ") if os.path.isfile('/usr/local/bin/composer'): packages = packages + ['/usr/local/bin/composer'] @@ -597,7 +544,7 @@ class WOStackController(CementBaseController): Log.warn(self, "Composer is not installed with WordOps") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") if os.path.isdir('{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)): @@ -605,11 +552,11 @@ class WOStackController(CementBaseController): 'cache/redis/phpRedisAdmin' .format(WOVariables.wo_webroot)] # ADMINER - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Removing package variable of Adminer ") packages = packages + ['{0}22222/htdocs/db/adminer' .format(WOVariables.wo_webroot)] - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Removing package variable of utils ") packages = packages + ['{0}22222/htdocs/php/webgrind/' .format(WOVariables.wo_webroot), @@ -621,29 +568,31 @@ class WOStackController(CementBaseController): '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)] - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Removing Netdata") if os.path.isfile('/opt/netdata/usr/' 'libexec/netdata-uninstaller.sh'): packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] - if self.app.pargs.dashboard: + if pargs.dashboard: Log.debug(self, "Removing Wo-Dashboard") - packages = packages + ['{0}22222/htdocs/assets/' + packages = packages + ['{0}22222/htdocs/assets' .format(WOVariables.wo_webroot), '{0}22222/htdocs/index.php' .format(WOVariables.wo_webroot)] if (packages) or (apt_packages): - wo_prompt = input('Are you sure you to want to' - ' remove from server.' - '\nPackage configuration will remain' - ' on server after this operation.\n' - 'Any answer other than ' - '"yes" will be stop this' - ' operation : ') + if not pargs.force: + wo_prompt = input('Are you sure you to want to' + ' remove from server.' + '\nPackage configuration will remain' + ' on server after this operation.\n' + 'Any answer other than ' + '"yes" will be stop this' + ' operation : ') - if wo_prompt == 'YES' or wo_prompt == 'yes': + if (wo_prompt == 'YES' or wo_prompt == 'yes' + or pargs.force): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') @@ -672,46 +621,46 @@ class WOStackController(CementBaseController): """Start purging of packages""" apt_packages = [] packages = [] - + pargs = self.app.pargs # Default action for stack purge - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.php73) and (not self.app.pargs.mysql) and - (not self.app.pargs.wpcli) and (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.composer) and (not self.app.pargs.netdata) and - (not self.app.pargs.fail2ban) and (not self.app.pargs.proftpd) and - (not self.app.pargs.security) and - (not self.app.pargs.all) and (not self.app.pargs.redis) and - (not self.app.pargs.phpredisadmin)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.php73) and (not pargs.mysql) and + (not pargs.wpcli) and (not pargs.phpmyadmin) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.composer) and (not pargs.netdata) and + (not pargs.fail2ban) and (not pargs.proftpd) and + (not pargs.security) and + (not pargs.all) and (not pargs.redis) and + (not pargs.phpredisadmin)): + pargs.web = True + pargs.admin = True + pargs.security = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.admin: - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.utils = True - self.app.pargs.composer = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True + if pargs.admin: + pargs.utils = True + pargs.composer = True + pargs.netdata = True + if os.path.isdir('{0}22222/htdocs' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/*' + .format(WOVariables.wo_webroot)] - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # NGINX - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): Log.debug(self, "Purge apt_packages variable of Nginx") apt_packages = apt_packages + WOVariables.wo_nginx @@ -720,7 +669,7 @@ class WOStackController(CementBaseController): "Nginx Stable version not found.") # PHP - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Purge apt_packages variable PHP") if WOAptGet.is_installed(self, 'php7.2-fpm'): if not (WOAptGet.is_installed(self, 'php7.3-fpm')): @@ -732,7 +681,7 @@ class WOStackController(CementBaseController): Log.error(self, "Cannot Purge PHP 7.2. not found.") # PHP 7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Removing apt_packages variable of PHP 7.3") if WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -742,19 +691,19 @@ class WOStackController(CementBaseController): apt_packages = apt_packages + WOVariables.wo_php73 # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): Log.debug(self, "Purge apt_packages variable of Fail2ban") apt_packages = apt_packages + WOVariables.wo_fail2ban # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): Log.debug(self, "Purge apt_packages variable for ProFTPd") apt_packages = apt_packages + ["proftpd-basic"] # WP-CLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Purge package variable WPCLI") if os.path.isfile('/usr/local/bin/wp'): packages = packages + ['/usr/local/bin/wp'] @@ -762,13 +711,13 @@ class WOStackController(CementBaseController): Log.warn(self, "WP-CLI is not installed with WordOps") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: packages = packages + ['{0}22222/htdocs/db/pma'. format(WOVariables.wo_webroot)] Log.debug(self, "Purge package variable phpMyAdmin") # Composer - if self.app.pargs.composer: + if pargs.composer: Log.debug(self, "Removing package variable of Composer ") if os.path.isfile('/usr/local/bin/composer'): packages = packages + ['/usr/local/bin/composer'] @@ -776,7 +725,7 @@ class WOStackController(CementBaseController): Log.warn(self, "Composer is not installed with WordOps") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") if os.path.isdir('{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)): @@ -784,12 +733,12 @@ class WOStackController(CementBaseController): 'cache/redis/phpRedisAdmin' .format(WOVariables.wo_webroot)] # Adminer - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Purge package variable Adminer") packages = packages + ['{0}22222/htdocs/db/adminer' .format(WOVariables.wo_webroot)] # utils - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Purge package variable utils") packages = packages + ['{0}22222/htdocs/php/webgrind/' .format(WOVariables.wo_webroot), @@ -802,13 +751,13 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot) ] - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Removing Netdata") if os.path.isfile('/opt/netdata/usr/' 'libexec/netdata-uninstaller.sh'): packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] - if self.app.pargs.dashboard: + if pargs.dashboard: Log.debug(self, "Removing Wo-Dashboard") packages = packages + ['{0}22222/htdocs/assets/' .format(WOVariables.wo_webroot), @@ -823,7 +772,7 @@ class WOStackController(CementBaseController): '"yes" will be stop this ' 'operation :') - if wo_prompt == 'YES' or wo_prompt == 'yes': + if wo_prompt == 'YES' or wo_prompt == 'yes' or pargs.force: if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index c308c24..20726ba 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1160,6 +1160,65 @@ def post_pref(self, apt_packages, packages): msg="Adding ProFTPd into Git") WOService.reload_service(self, 'proftpd') + # Redis configuration + if set(["redis-server"]).issubset(set(apt_packages)): + # set redis.conf parameter + # set maxmemory 10% for ram below 512MB and 20% for others + # set maxmemory-policy allkeys-lru + # enable systemd service + Log.debug(self, "Enabling redis systemd service") + WOShellExec.cmd_exec(self, "systemctl enable redis-server") + if os.path.isfile("/etc/redis/redis.conf"): + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + if wo_ram < 1024: + Log.debug(self, "Setting maxmemory variable to " + "{0} in redis.conf" + .format(int(wo_ram*1024*1024*0.1))) + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram*1024*1024*0.1))) + Log.debug( + self, "Setting maxmemory-policy variable to " + "allkeys-lru in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory-policy " + "noeviction", + "maxmemory-policy " + "allkeys-lru") + Log.debug( + self, "Setting tcp-backlog variable to " + "in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "tcp-backlog 511", + "tcp-backlog 32768") + + WOService.restart_service(self, 'redis-server') + else: + Log.debug(self, "Setting maxmemory variable to {0} " + "in redis.conf" + .format(int(wo_ram*1024*1024*0.2))) + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram*1024*1024*0.1))) + Log.debug( + self, "Setting maxmemory-policy variable " + "to allkeys-lru in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory-policy " + "noeviction", + "maxmemory-policy " + "allkeys-lru") + WOService.restart_service(self, 'redis-server') + if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): Log.debug(self, "Setting Privileges" From c7410e1a5f2bd88a93a588d0e9ecc01408ce8c9e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 12:30:17 +0200 Subject: [PATCH 033/102] testing cement 2.6 --- requirements.txt | 2 +- setup.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 24fb221..9cacc2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.4.0 +cement>=2.6.0 diff --git a/setup.py b/setup.py index c05b21e..5c32f9e 100644 --- a/setup.py +++ b/setup.py @@ -72,12 +72,12 @@ setup(name='wo', test_suite='nose.collector', install_requires=[ # Required to build documentation - # "Sphinx >= 1.0", + "Sphinx >= 1.0", # Required for testing - # "nose", - # "coverage", + "nose", + "coverage", # Required to function - 'cement == 2.4', + 'cement == 2.6', 'pystache', 'python-apt', 'pynginxconfig', From 29a5633e8250f340dbfe59b168a832adb8bf26bf Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 12:36:29 +0200 Subject: [PATCH 034/102] update psutil & PyMySQL --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5c32f9e..6081aa4 100644 --- a/setup.py +++ b/setup.py @@ -81,8 +81,8 @@ setup(name='wo', 'pystache', 'python-apt', 'pynginxconfig', - 'PyMySQL == 0.8.0', - 'psutil == 3.1.1', + 'PyMySQL == 0.9.3', + 'psutil == 5.6.3', 'sh', 'SQLAlchemy', 'requests == 2.22.0', From d7e4a07254003e4745b50631b09e92a7226df2ba Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 13:13:30 +0200 Subject: [PATCH 035/102] Add Cloudflare restore real-ip --- CHANGELOG.md | 4 ++++ install | 6 +++--- wo/cli/plugins/stack_pref.py | 8 ++++++++ wo/cli/templates/cloudflare.mustache | 23 +++++++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 wo/cli/templates/cloudflare.mustache diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e12027..a9a1770 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Added - APT Packages configuration step with `wo stack upgrade` to apply new configurations +- Cloudflare restore real_ip configuration #### Changed - Moving package configuration in a new plugin stack_pref.py - Set WordOps backend password length from 16 to 24 +- Upgrade framework cement to 2.6.0 +- Upgrade PyMySQL to 0.9.3 +- Upgrade Psutil to 5.6.3 ### v3.9.7 - 2019-08-02 diff --git a/install b/install index 17c7243..c346832 100755 --- a/install +++ b/install @@ -406,9 +406,9 @@ wo_install_acme_sh() { wo_install() { { rm -f /etc/bash_completion.d/wo_auto.rc - rm -rf /tmp/WordOps - git clone -b "$wo_branch" --depth=50 https://github.com/WordOps/WordOps.git /tmp/WordOps - cd /tmp/WordOps || exit 1 + rm -rf /var/lib/wo/tmp/WordOps-* + curl -sL https://github.com/WordOps/WordOps/archive/${wo_branch}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp + cd /var/lib/wo/tmp/WordOps-${wo_branch} || exit 1 } \ >> "$wo_install_log" 2>&1 diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 20726ba..e432df6 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -227,6 +227,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/cloudflare.conf') + wo_nginx = open('/etc/nginx/conf.d/cloudflare.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'cloudflare.mustache', + out=wo_nginx) + wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/conf.d/' 'map-wp-fastcgi-cache.conf') diff --git a/wo/cli/templates/cloudflare.mustache b/wo/cli/templates/cloudflare.mustache new file mode 100644 index 0000000..38355fe --- /dev/null +++ b/wo/cli/templates/cloudflare.mustache @@ -0,0 +1,23 @@ +# WordOps (wo) set visitors real ip with Cloudflare +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/12; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; +real_ip_header CF-Connecting-IP; From c2641d02cf83dd190cf7d1d76bb99ef3ccafd200 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 15:39:43 +0200 Subject: [PATCH 036/102] Testing cement 2.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9cacc2a..6a4b1a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.6.0 +cement>=2.8.0 From d1f51f345d73d27276b42ccacba3ff4c6c7377eb Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 08:40:03 +0200 Subject: [PATCH 037/102] Update templates --- install | 2 -- wo/cli/templates/fastcgi.mustache | 8 ++--- wo/cli/templates/locations.mustache | 50 ++++++++++++++++++++++++++++- wo/cli/templates/redis.mustache | 2 +- wo/cli/templates/sysctl.mustache | 12 +++---- wo/cli/templates/wpcommon.mustache | 14 ++++++++ wo/cli/templates/wprocket.mustache | 16 ++++----- 7 files changed, 80 insertions(+), 24 deletions(-) diff --git a/install b/install index c346832..62cc83d 100755 --- a/install +++ b/install @@ -667,8 +667,6 @@ wo_tweak_kernel() { fi # apply sysctl tweaks sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf - else - sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf fi fi if [ ! -x /opt/wo-kernel.sh ]; then diff --git a/wo/cli/templates/fastcgi.mustache b/wo/cli/templates/fastcgi.mustache index 271e751..c5a68e2 100644 --- a/wo/cli/templates/fastcgi.mustache +++ b/wo/cli/templates/fastcgi.mustache @@ -1,5 +1,5 @@ # FastCGI cache settings -fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=60m max_size=256M; +fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=6h max_size=256M; fastcgi_cache_key "$scheme$request_method$host$request_uri"; fastcgi_cache_use_stale error timeout invalid_header updating http_500 http_503; fastcgi_cache_lock on; @@ -7,11 +7,11 @@ fastcgi_cache_lock_age 5s; fastcgi_cache_lock_timeout 5s; fastcgi_cache_methods GET HEAD; fastcgi_cache_background_update on; -fastcgi_cache_valid 200 1h; +fastcgi_cache_valid 200 24h; fastcgi_cache_valid 301 302 30m; fastcgi_cache_valid 499 502 503 1m; -fastcgi_cache_valid 404 15m; -fastcgi_cache_valid any 15m; +fastcgi_cache_valid 404 1h; +fastcgi_cache_valid any 1h; fastcgi_buffers 16 16k; fastcgi_buffer_size 32k; fastcgi_param SERVER_NAME $http_host; diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index 26e8230..f944a40 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -7,11 +7,35 @@ location = /favicon.ico { expires max; } # Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp)$ { +location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json)$ { add_header "Access-Control-Allow-Origin" "*"; access_log off; log_not_found off; expires max; + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; } # Cache css & js files location ~* \.(?:css(\.map)?|js(\.map)?)$ { @@ -19,6 +43,30 @@ location ~* \.(?:css(\.map)?|js(\.map)?)$ { access_log off; log_not_found off; expires 30d; + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; } # Security settings for better privacy # Deny hidden files diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index 88a49e0..7864dd9 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -17,7 +17,7 @@ location /redis-store { set_unescape_uri $key $arg_key ; redis2_query set $key $echo_request_body; redis2_query expire $key 14400; - redis2_pass redis; + redis2_pass redis; } diff --git a/wo/cli/templates/sysctl.mustache b/wo/cli/templates/sysctl.mustache index cc2c332..271298c 100644 --- a/wo/cli/templates/sysctl.mustache +++ b/wo/cli/templates/sysctl.mustache @@ -1,10 +1,10 @@ # Kernel sysctl configuration file for Linux # -# Version 1.16 - 2019-10-25 +# Version 1.18 - 2019-08-07 # Michiel Klaver - IT Professional -# Modified by VirtuBox +# Modified by VirtuBox (https://github.com/VirtuBox) +# For WordOps (https://github.com/WordOps/WordOps) # -# Instructions available on https://github.com/VirtuBox/ubuntu-nginx-web-server # # Sources : # https://klaver.it/linux/sysctl.conf @@ -27,10 +27,6 @@ # http://lartc.org/howto/lartc.kernel.obscure.html # http://en.wikipedia.org/wiki/Sysctl # -# Usage -# wget -O /etc/sysctl.d/60-ubuntu-nginx-web-server.conf https://virtubox.github.io/ubuntu-nginx-web-server/files/etc/sysctl.d/60-ubuntu-nginx-web-server.conf -# -# sysctl -e -p /etc/sysctl.d/60-ubuntu-nginx-web-server.conf # For binary values, 0 is disabled, 1 is enabled. See sysctl(8) and sysctl.conf(5) for more details. # @@ -200,6 +196,8 @@ net.core.somaxconn = 32768 # Increase number of incoming connections backlog net.core.netdev_max_backlog = 16384 net.core.dev_weight = 64 +net.core.netdev_budget = 600 +net.core.netdev_budget_usecs = 4000 # Increase the maximum amount of option memory buffers net.core.optmem_max = 65535 diff --git a/wo/cli/templates/wpcommon.mustache b/wo/cli/templates/wpcommon.mustache index f646cd6..33a9e4a 100644 --- a/wo/cli/templates/wpcommon.mustache +++ b/wo/cli/templates/wpcommon.mustache @@ -56,6 +56,20 @@ location /wp-content/plugins/ewww-image-optimizer/images { deny all; } } +# enable gzip on static assets - php files are forbidden +location /wp-content/cache { +# Cache css & js files + location ~* \.(?:css(\.map)?|js(\.map)?|.html)$ { + add_header "Access-Control-Allow-Origin" "*"; + access_log off; + log_not_found off; + expires 30d; + } + location ~ \.php$ { +#Prevent Direct Access Of PHP Files From Web Browsers + deny all; + } +} # Deny access to any files with a .php extension in the uploads directory # Works in sub-directory installs and also in multisite network # Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban) diff --git a/wo/cli/templates/wprocket.mustache b/wo/cli/templates/wprocket.mustache index e73f859..14e3600 100644 --- a/wo/cli/templates/wprocket.mustache +++ b/wo/cli/templates/wprocket.mustache @@ -1,21 +1,19 @@ # WPSC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf - # Use cached or actual file if they exists, Otherwise pass request to WordPress location / { - try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; + try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; } location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; + try_files $uri =404; + include fastcgi_params; fastcgi_pass {{upstream}}; } - location ~* /cache/wp-rocket/\.html$ { - etag on; - add_header Vary "Accept-Encoding, Cookie"; + etag on; + add_header Vary "Accept-Encoding, Cookie"; access_log off; log_not_found off; - expire 10h; -} \ No newline at end of file + expires 10h; +} From ad6e1bcb3a321419f744d6db6d435466efca053e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 20:30:15 +0200 Subject: [PATCH 038/102] testing cement 2.10.12 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6a4b1a5..29f8528 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.8.0 +cement>=2.10.12 diff --git a/setup.py b/setup.py index 6081aa4..4ba23dd 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ setup(name='wo', "nose", "coverage", # Required to function - 'cement == 2.6', + 'cement == 2.10.12', 'pystache', 'python-apt', 'pynginxconfig', From ca043e177d7b0320cf57ee1add5b56e5d8387578 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 20:36:03 +0200 Subject: [PATCH 039/102] testing again with cement 2.8.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 29f8528..6a4b1a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.10.12 +cement>=2.8.0 diff --git a/setup.py b/setup.py index 4ba23dd..57b98bc 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ setup(name='wo', "nose", "coverage", # Required to function - 'cement == 2.10.12', + 'cement == 2.8.0', 'pystache', 'python-apt', 'pynginxconfig', From c9a55760717e54053f4ca5c7018f9d7c93daaa01 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 21:16:19 +0200 Subject: [PATCH 040/102] Fix redis install --- wo/cli/plugins/stack_pref.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index e432df6..92bb787 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1225,6 +1225,8 @@ def post_pref(self, apt_packages, packages): "noeviction", "maxmemory-policy " "allkeys-lru") + WOFileUtils.chown(self, '/etc/redis/redis.conf', + redis, redis) WOService.restart_service(self, 'redis-server') if (packages): From 5682b96e64845c2fe4e82e160220c4cdcd2386a4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 21:42:26 +0200 Subject: [PATCH 041/102] Fix redis --- .travis.yml | 2 +- wo/cli/plugins/stack_pref.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f484bef..2200856 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ script: - unset LANG - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - - sudo apt-get install -qq --force-yes git python3-setuptools python3-dev python3-apt python3-pip ccze tree + - sudo apt-get install --assume-yes git python3-setuptools python3-dev python3-apt python3-pip ccze tree --quiet - sudo bash install --travis - sudo wo --help && sudo wo stack install && sudo wo stack install --proftpd - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 92bb787..1d08dfe 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1226,7 +1226,7 @@ def post_pref(self, apt_packages, packages): "maxmemory-policy " "allkeys-lru") WOFileUtils.chown(self, '/etc/redis/redis.conf', - redis, redis) + 'redis', 'redis', recursive=False) WOService.restart_service(self, 'redis-server') if (packages): From e68434ec6fca2f539fe852c8f975b4248482dd4b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 00:20:09 +0200 Subject: [PATCH 042/102] Simplify letsencrypt --- wo/cli/plugins/site_functions.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 4181afe..032ae6f 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -39,7 +39,7 @@ def pre_run_checks(self): Log.debug(self, "checking NGINX configuration ...") FNULL = open('/dev/null', 'w') subprocess.check_call(["/usr/sbin/nginx", "-t"], stdout=FNULL, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT) except CalledProcessError as e: Log.debug(self, "{0}".format(str(e))) raise SiteError("nginx configuration check failed.") @@ -90,7 +90,7 @@ def setupdomain(self, data): Log.debug(self, "Checking generated nginx conf, please wait...") FNULL = open('/dev/null', 'w') subprocess.check_call(["/usr/sbin/nginx", "-t"], stdout=FNULL, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT) Log.info(self, "[" + Log.ENDC + "Done" + Log.OKBLUE + "]") except CalledProcessError as e: Log.debug(self, "{0}".format(str(e))) @@ -1382,6 +1382,8 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, else: keylenght = "{0}".format(self.app.config.get('letsencrypt', 'keylength')) + wo_acme_exec = ("/etc/letsencrypt/acme.sh --config-home " + "'/etc/letsencrypt/config'") if wo_dns: acme_mode = "--dns {0}".format(wo_acme_dns) validation_mode = "DNS with {0}".format(wo_acme_dns) @@ -1394,9 +1396,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, if subdomain: Log.info(self, "Issuing subdomain SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} {1} " "-k {2} -f" @@ -1406,9 +1406,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, elif wildcard: Log.info(self, "Issuing Wildcard SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d *.{0} --dns {1} " "-k {2} -f" @@ -1418,9 +1416,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, else: Log.info(self, "Issuing domain SSL cert with acme.sh") Log.info(self, "Validation mode : {0}".format(validation_mode)) - ssl = WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " + ssl = WOShellExec.cmd_exec(self, "{0} ".format(wo_acme_exec) + "--issue " "-d {0} -d www.{0} {1} " "-k {2} -f" From cba64df1637209c325c0adb811ea341cbd907efe Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 00:23:04 +0200 Subject: [PATCH 043/102] simplify travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b9d72f0..86ad46a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: - rm -rf ~/.gnupg before_script: - - sudo rm -rf /etc/mysql/ + - sudo rm -rf /etc/mysql - sudo bash -c 'echo example.com > /etc/hostname' - sudo apt-get -qq purge mysql* graphviz* redis* - sudo apt-get -qq autoremove --purge From 2cbefcabcf05792114ad9400beda8552de383d23 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:27:27 +0200 Subject: [PATCH 044/102] Add package configuration with stack upgrade --- install | 4 ++-- wo/cli/plugins/stack_upgrade.py | 29 +++++++++++++++++------------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/install b/install index 9c9ec79..d7a2352 100755 --- a/install +++ b/install @@ -147,9 +147,9 @@ if [ -z "$wo_force_install" ]; then wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." exit 100 else - check_wo_linux_distro=$(lsb_release -sc | grep -E "trusty|xenial|bionic|disco|jessie|stretch|buster") + check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster") if [ -z "$check_wo_linux_distro" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 14.04/16.04/18.04/19.04 LTS, Debian 8.x/9.x/10.x and Raspbian 9.x" + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/19.04 LTS, Debian 9.x/10.x and Raspbian 9.x" exit 100 fi fi diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0344c09..5af52b8 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -1,17 +1,19 @@ -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.logging import Log -from wo.core.variables import WOVariables -from wo.core.aptget import WOAptGet -from wo.core.apt_repo import WORepo -from wo.core.services import WOService -from wo.core.fileutils import WOFileUtils -from wo.core.shellexec import WOShellExec -from wo.core.extract import WOExtract -from wo.core.download import WODownload import os import shutil +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.cli.plugins.stack import WOStackController +from wo.core.apt_repo import WORepo +from wo.core.aptget import WOAptGet +from wo.core.download import WODownload +from wo.core.extract import WOExtract +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.services import WOService +from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables + class WOStackUpgradeController(CementBaseController): class Meta: @@ -83,6 +85,9 @@ class WOStackUpgradeController(CementBaseController): # All package update apt_packages = [] packages = [] + empty_packages = [] + stack = WOStackController() + stack.app = self.app if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and (not self.app.pargs.php) and (not self.app.pargs.mysql) and @@ -187,7 +192,7 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) - + stack.post_pref(apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') From 629a0d2223ad8e1a47c8bd5d55bf5db0fbf6efa3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:28:37 +0200 Subject: [PATCH 045/102] update travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 86ad46a..c54a975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,8 @@ script: - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf + - sudo wo stack upgrade --nginx + - sudo wo stack upgrade --php - sudo wo stack upgrade --netdata --no-prompt - sudo wo stack upgrade --phpmyadmin --no-prompt - sudo wo stack upgrade --composer --no-prompt From 2cad38bce4d18d3006269e1e799942fd55e5b080 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 04:56:33 +0200 Subject: [PATCH 046/102] Add stack_pref --- wo/cli/plugins/stack.py | 1336 +----------------------------- wo/cli/plugins/stack_pref.py | 1357 +++++++++++++++++++++++++++++++ wo/cli/plugins/stack_upgrade.py | 8 +- 3 files changed, 1364 insertions(+), 1337 deletions(-) create mode 100644 wo/cli/plugins/stack_pref.py diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index a0a2bd0..bf245de 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1,7 +1,7 @@ """Stack Plugin for WordOps""" -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose +from cement.core import handler, hook import codecs import configparser @@ -20,6 +20,7 @@ from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_migrate import WOStackMigrateController from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController +from wo.cli.plugins.stack_pref import pre_pref, post_pref from wo.core.addswap import WOSwap from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet @@ -93,1335 +94,6 @@ class WOStackController(CementBaseController): """default action of wo stack command""" self.app.args.print_help() - @expose(hide=True) - def pre_pref(self, apt_packages): - """Pre settings to do before installation packages""" - - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - # add mariadb repository excepted on raspbian and ubuntu 19.04 - if (not WOVariables.wo_distro == 'raspbian'): - Log.info(self, "Adding repository for MySQL, please wait...") - mysql_pref = ("Package: *\nPin: origin " - "sfo1.mirrors.digitalocean.com" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'MariaDB.pref', 'w') as mysql_pref_file: - mysql_pref_file.write(mysql_pref) - WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVariables.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - WORepo.add_key(self, '0xF1656F24C74CD1D8', - keyserver="keyserver.ubuntu.com") - # generate random 24 characters root password - chars = ''.join(random.sample(string.ascii_letters, 24)) - # configure MySQL non-interactive install - if (not WOVariables.wo_distro == 'raspbian'): - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - else: - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") - # generate my.cnf root credentials - mysql_config = """ - [client] - user = root - password = {chars} - """.format(chars=chars) - config = configparser.ConfigParser() - config.read_string(mysql_config) - Log.debug(self, 'Writting configuration into MySQL file') - conf_path = "/etc/mysql/conf.d/my.cnf" - os.makedirs(os.path.dirname(conf_path), exist_ok=True) - with open(conf_path, encoding='utf-8', - mode='w') as configfile: - config.write(configfile) - Log.debug(self, 'Setting my.cnf permission') - WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) - - # add nginx repository - if set(WOVariables.wo_nginx).issubset(set(apt_packages)): - if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for NGINX, please wait...") - WORepo.add(self, ppa=WOVariables.wo_nginx_repo) - Log.debug(self, 'Adding ppa for Nginx') - else: - Log.info(self, "Adding repository for NGINX, please wait...") - WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) - Log.debug(self, 'Adding repository for Nginx') - WORepo.add_key(self, WOVariables.wo_nginx_key) - - # add php repository - if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or - set(WOVariables.wo_php).issubset(set(apt_packages))): - if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for PHP, please wait...") - Log.debug(self, 'Adding ppa for PHP') - WORepo.add(self, ppa=WOVariables.wo_php_repo) - else: - Log.info(self, "Adding repository for PHP, please wait...") - # Add repository for php - if (WOVariables.wo_platform_codename == 'buster'): - php_pref = ("Package: *\nPin: origin " - "packages.sury.org" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'PHP.pref', 'w') as php_pref_file: - php_pref_file.write(php_pref) - Log.debug(self, 'Adding repo_url of php for debian') - WORepo.add(self, repo_url=WOVariables.wo_php_repo) - Log.debug(self, 'Adding deb.sury GPG key') - WORepo.add_key(self, WOVariables.wo_php_key) - # add redis repository - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - Log.info(self, "Adding repository for Redis, please wait...") - if WOVariables.wo_distro == 'ubuntu': - Log.debug(self, 'Adding ppa for redis') - WORepo.add(self, ppa=WOVariables.wo_redis_repo) - else: - Log.debug(self, 'Adding repo_url of redis for debian') - - @expose(hide=True) - def post_pref(self, apt_packages, packages): - """Post activity after installation of packages""" - if (apt_packages): - - # Nginx configuration - if set(WOVariables.wo_nginx).issubset(set(apt_packages)): - - # Fix for white screen death with NGINX PLUS - if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', - 'SCRIPT_FILENAME'): - with open('/etc/nginx/fastcgi_params', - encoding='utf-8', mode='a') as wo_nginx: - wo_nginx.write('fastcgi_param \tSCRIPT_FILENAME ' - '\t$request_filename;\n') - - if os.path.isfile('/etc/nginx/nginx.conf'): - data = dict(php="9000", debug="9001", - php7="9070", debug7="9170") - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/upstream.conf') - wo_nginx = open('/etc/nginx/conf.d/upstream.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'upstream.mustache', out=wo_nginx) - wo_nginx.close() - - data = dict(phpconf=True if - WOAptGet.is_installed(self, 'php7.2-fpm') - else False) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/stub_status.conf') - wo_nginx = open('/etc/nginx/conf.d/stub_status.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'stub_status.mustache', out=wo_nginx) - wo_nginx.close() - - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/webp.conf') - wo_nginx = open('/etc/nginx/conf.d/webp.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'webp.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/' - 'map-wp-fastcgi-cache.conf') - wo_nginx = open('/etc/nginx/conf.d/' - 'map-wp-fastcgi-cache.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'map-wp.mustache', - out=wo_nginx) - wo_nginx.close() - - # Setup Nginx common directory - if not os.path.exists('/etc/nginx/common'): - Log.debug(self, 'Creating directory' - '/etc/nginx/common') - os.makedirs('/etc/nginx/common') - - if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) - - # Common Configuration - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsubdir.conf') - wo_nginx = open('/etc/nginx/common/wpsubdir.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsubdir.mustache', - out=wo_nginx) - wo_nginx.close() - - # PHP 7.2 conf - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php72.conf') - wo_nginx = open('/etc/nginx/common/php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php72.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc.mustache', - out=wo_nginx) - wo_nginx.close() - - # PHP 7.3 conf - if os.path.isdir("/etc/nginx/common"): - data = dict() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php73.conf') - wo_nginx = open('/etc/nginx/common/php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php73.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - # create redis conf - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - with open("/etc/nginx/common/release", - "a") as release_file: - release_file.write("v{0}" - .format(WOVariables.wo_version)) - release_file.close() - - # Following files should not be overwrited - - if not os.path.isfile('/etc/nginx/common/acl.conf'): - data = dict(webroot=WOVariables.wo_webroot) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/acl.conf') - wo_nginx = open('/etc/nginx/common/acl.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'acl.mustache', - out=wo_nginx) - wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/blockips.conf') - wo_nginx = open('/etc/nginx/conf.d/blockips.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'blockips.mustache', out=wo_nginx) - wo_nginx.close() - - if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/fastcgi.conf') - wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'fastcgi.mustache', out=wo_nginx) - wo_nginx.close() - - # add redis cache format if not already done - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d" - "/redis.conf")): - with open("/etc/nginx/conf.d/" - "redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - # Nginx-Plus does not have nginx - # package structure like this - # So creating directories - if not os.path.exists('/etc/nginx/sites-available'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-available') - - if not os.path.exists('/etc/nginx/sites-enabled'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-enabled') - - # 22222 port settings - if not os.path.isfile("/etc/nginx/sites-available/22222"): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/sites-available/' - '22222') - wo_nginx = open('/etc/nginx/sites-available/22222', - encoding='utf-8', mode='w') - self.app.render((data), '22222.mustache', - out=wo_nginx) - wo_nginx.close() - - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) - try: - WOShellExec.cmd_exec(self, "printf \"WordOps:" - "$(openssl passwd -crypt " - "{password} 2> /dev/null)\n\"" - "> /etc/nginx/htpasswd-wo " - "2>/dev/null" - .format(password=passwd)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to save HTTP Auth") - - # Create Symbolic link for 22222 - WOFileUtils.create_symlink(self, ['/etc/nginx/' - 'sites-available/' - '22222', - '/etc/nginx/' - 'sites-enabled/' - '22222']) - # Create log and cert folder and softlinks - if not os.path.exists('{0}22222/logs' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/logs " - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/logs' - .format(WOVariables.wo_webroot)) - - if not os.path.exists('{0}22222/cert' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/cert" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/cert' - .format(WOVariables.wo_webroot)) - - if not os.path.exists('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/conf/nginx" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)) - - WOFileUtils.create_symlink(self, ['/var/log/nginx/' - '22222.access.log', - '{0}22222/' - 'logs/access.log' - .format(WOVariables.wo_webroot)] - ) - - WOFileUtils.create_symlink(self, ['/var/log/nginx/' - '22222.error.log', - '{0}22222/' - 'logs/error.log' - .format(WOVariables.wo_webroot)] - ) - - try: - WOShellExec.cmd_exec(self, "openssl genrsa -out " - "{0}22222/cert/22222.key 2048" - .format(WOVariables.wo_webroot)) - WOShellExec.cmd_exec(self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key {0}22222/cert/22222.key " - "-out {0}22222/cert/" - "22222.csr" - .format(WOVariables.wo_webroot)) - - WOFileUtils.mvfile(self, "{0}22222/cert/22222.key" - .format(WOVariables.wo_webroot), - "{0}22222/cert/" - "22222.key.org" - .format(WOVariables.wo_webroot)) - - WOShellExec.cmd_exec(self, "openssl rsa -in " - "{0}22222/cert/" - "22222.key.org -out " - "{0}22222/cert/22222.key" - .format(WOVariables.wo_webroot)) - - WOShellExec.cmd_exec(self, "openssl x509 -req -days " - "3652 -in {0}22222/cert/" - "22222.csr -signkey {0}" - "22222/cert/22222.key -out " - "{0}22222/cert/22222.crt" - .format(WOVariables.wo_webroot)) - - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate HTTPS " - "certificate for 22222") - server_ip = requests.get('http://v4.wordops.eu') - - if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' - .format(WOVariables.wo_webroot)): - - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "a") as php_file: - php_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") - - # Nginx Configation into GIT - WOGit.add(self, - ["/etc/nginx"], msg="Adding Nginx into Git") - WOService.reload_service(self, 'nginx') - - if set(["nginx"]).issubset(set(apt_packages)): - - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) - else: - WOService.restart_service(self, 'nginx') - - # create nginx configuration for redis - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - if os.path.isdir('/etc/nginx/common'): - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}\n") - - if os.path.isfile("/etc/nginx/nginx.conf"): - if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): - with open("/etc/nginx/conf.d/redis.conf", - "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local]" - " '\n '$http_host" - " \"$request\" " - "$status $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - if set(WOVariables.wo_php).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.2/'): - Log.debug(self, 'Creating directory /var/log/php/7.2/') - os.makedirs('/var/log/php/7.2/') - - # Parse etc/php/7.2/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file " - "/etc/php/7.2/fpm/php.ini") - config.read('/etc/php/7.2/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.2/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.2/fpm/php.ini") - config.write(configfile) - - # Parse /etc/php/7.2/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.2-fpm.pid", - error_log="/var/log/php/7.2/fpm.log", - include="/etc/php/7.2/fpm/pool.d/*.conf") - Log.debug(self, "writting php7.2 configuration into " - "/etc/php/7.2/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() - - # Parse /etc/php/7.2/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.2/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php72-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writing PHP 7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www.conf") - config.write(configfile) - - with open("/etc/php/7.2/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") - - # Generate /etc/php/7.2/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php72-two-fpm.sock' - with open('/etc/php/7.2/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www-two.conf") - config.write(confifile) - - # Generate /etc/php/7.2/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9172' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.2/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/debug.conf") - config.write(confifile) - - with open("/etc/php/7.2/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") - - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.2/mods-available/" - "xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.2/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") - - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") - - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) - - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.2-fpm') - - # PHP7.3 configuration - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.3/'): - Log.debug(self, 'Creating directory /var/log/php/7.3/') - os.makedirs('/var/log/php/7.3/') - - # Parse etc/php/7.3/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file /etc/php/7.3/" - "fpm/php.ini") - config.read('/etc/php/7.3/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.3/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.3/fpm/php.ini") - config.write(configfile) - - # Parse /etc/php/7.3/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.3-fpm.pid", - error_log="/var/log/php7.3-fpm.log", - include="/etc/php/7.3/fpm/pool.d/*.conf") - Log.debug(self, "writting php 7.3 configuration into " - "/etc/php/7.3/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() - - # Parse /etc/php/7.3/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.3/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php73-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www.conf") - config.write(configfile) - - with open("/etc/php/7.3/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") - - # Generate /etc/php/7.3/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php73-two-fpm.sock' - with open('/etc/php/7.3/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www-two.conf") - config.write(confifile) - - # Generate /etc/php/7.3/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9173' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.3/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/debug.conf") - config.write(confifile) - - with open("/etc/php/7.3/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") - - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.3/mods-available" - "/xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.3/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") - - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") - - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) - - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.3-fpm') - - # create mysql config if it doesn't exist - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - if not os.path.isfile("/etc/mysql/my.cnf"): - config = ("[mysqld]\nwait_timeout = 30\n" - "interactive_timeout=60\nperformance_schema = 0" - "\nquery_cache_type = 1") - config_file = open("/etc/mysql/my.cnf", - encoding='utf-8', mode='w') - config_file.write(config) - config_file.close() - - WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) - WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' - '> /dev/null 2>&1', - comment='MySQL optimization cronjob ' - 'added by WordOps') - WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - WOService.reload_service(self, 'mysql') - - # create fail2ban configuration files - if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): - if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - data = dict() - Log.debug(self, "Setting up fail2ban jails configuration") - fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban.mustache', - out=fail2ban_config) - fail2ban_config.close() - - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'wo-wordpress.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-wp.mustache', - out=fail2ban_config) - fail2ban_config.close() - - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'nginx-forbidden.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-forbidden.mustache', - out=fail2ban_config) - fail2ban_config.close() - WOGit.add(self, ["/etc/fail2ban"], - msg="Adding Fail2ban into Git") - WOService.reload_service(self, 'fail2ban') - - # Proftpd configuration - if set(["proftpd-basic"]).issubset(set(apt_packages)): - if os.path.isfile("/etc/proftpd/proftpd.conf"): - Log.debug(self, "Setting up Proftpd configuration") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# DefaultRoot", - "DefaultRoot") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# RequireValidShell", - "RequireValidShell") - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "# PassivePorts " - " " - "49152 65534", - "PassivePorts " - " " - " 49000 50000") - # proftpd TLS configuration - if not os.path.isdir("/etc/proftpd/ssl"): - WOFileUtils.mkdir(self, "/etc/proftpd/ssl") - - try: - WOShellExec.cmd_exec(self, "openssl genrsa -out " - "/etc/proftpd/ssl/proftpd.key 2048") - WOShellExec.cmd_exec(self, "openssl req -new -batch " - "-subj /commonName=localhost/ " - "-key /etc/proftpd/ssl/proftpd.key " - "-out /etc/proftpd/ssl/proftpd.csr") - WOFileUtils.mvfile(self, "/etc/proftpd/ssl/proftpd.key", - "/etc/proftpd/ssl/proftpd.key.org") - WOShellExec.cmd_exec(self, "openssl rsa -in " - "/etc/proftpd/ssl/proftpd.key.org " - "-out /etc/proftpd/ssl/proftpd.key") - WOShellExec.cmd_exec(self, "openssl x509 -req -days " - "3652 -in /etc/proftpd/ssl/proftpd.csr " - "-signkey /etc/proftpd/ssl/proftpd.key " - " -out /etc/proftpd/ssl/proftpd.crt") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error( - self, "Failed to generate SSL " - "certificate for Proftpd") - WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.key", 0o700) - WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.crt", 0o700) - data = dict() - Log.debug(self, 'Writting the proftpd configuration to ' - 'file /etc/proftpd/tls.conf') - wo_proftpdconf = open('/etc/proftpd/tls.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'proftpd-tls.mustache', - out=wo_proftpdconf) - wo_proftpdconf.close() - WOFileUtils.searchreplace(self, "/etc/proftpd/" - "proftpd.conf", - "#Include /etc/proftpd/tls.conf", - "Include /etc/proftpd/tls.conf") - WOService.restart_service(self, 'proftpd') - - # add rule for proftpd with UFW - if WOAptGet.is_installed(self, 'ufw'): - try: - WOShellExec.cmd_exec(self, "ufw allow " - "49000:50000/tcp") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Unable to add UFW rule") - - if os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - with open("/etc/fail2ban/jail.d/custom.conf", - encoding='utf-8', mode='a') as f2bproftpd: - f2bproftpd.write("\n\n[proftpd]\nenabled = true\n") - WOService.reload_service(self, 'fail2ban') - - WOGit.add(self, ["/etc/proftpd"], - msg="Adding ProFTPd into Git") - WOService.reload_service(self, 'proftpd') - - if (packages): - if any('/usr/local/bin/wp' == x[1] for x in packages): - Log.debug(self, "Setting Privileges" - " to /usr/local/bin/wp file ") - WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - - if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] - for x in packages): - WOExtract.extract( - self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') - Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' - 'location /var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/db" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', - '{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)) - shutil.copyfile('{0}22222/htdocs/db/pma' - '/config.sample.inc.php' - .format(WOVariables.wo_webroot), - '{0}22222/htdocs/db/pma/config.inc.php' - .format(WOVariables.wo_webroot)) - Log.debug(self, 'Setting Blowfish Secret Key ' - 'FOR COOKIE AUTH to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - blowfish_key = ''.join([random.choice - (string.ascii_letters + - string.digits) - for n in range(25)]) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'blowfish_secret\']" - " = \'\';", - "$cfg[\'blowfish_secret\']" - " = \'{0}\';" - .format(blowfish_key)) - Log.debug(self, 'Setting HOST Server For Mysql to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'Servers\'][$i][\'host\']" - " = \'localhost\';", "$cfg" - "[\'Servers\'][$i][\'host\'] = \'{0}\';" - .format(WOVariables.wo_mysql_host)) - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # composer install and phpmyadmin update - if any('/var/lib/wo/tmp/composer-install' == x[1] - for x in packages): - Log.info(self, "Installing composer, please wait...") - WOShellExec.cmd_exec(self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") - shutil.copyfile('/var/lib/wo/tmp/composer.phar', - '/usr/local/bin/composer') - WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " - "update -n --no-dev -d " - "/var/www/22222/htdocs/db/pma/") - - # netdata install - if any('/var/lib/wo/tmp/kickstart.sh' == x[1] - for x in packages): - if ((not os.path.exists('/opt/netdata')) and - (not os.path.exists('/etc/netdata'))): - Log.info(self, "Installing Netdata, please wait...") - WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" - "kickstart.sh " - "--dont-wait") - # disable mail notifications - WOFileUtils.searchreplace(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - 'SEND_EMAIL="YES"', - 'SEND_EMAIL="NO"') - # make changes persistant - WOFileUtils.copyfile(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - "/opt/netdata/etc/netdata/" - "health_alarm_notify.conf") - # check if mysql credentials are available - if os.path.isfile('/etc/mysql/conf.d/my.cnf'): - try: - WOMysql.execute(self, - "create user " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "grant usage on *.* to " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "flush privileges;", - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.info( - self, "fail to setup mysql user for netdata") - WOService.restart_service(self, 'netdata') - - # WordOps Dashboard - if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] - for x in packages): - if not os.path.isfile('{0}22222/htdocs/index.php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting wo-dashboard.tar.gz " - "to location {0}22222/htdocs/" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/' - 'wo-dashboard.tar.gz', - '{0}22222/htdocs' - .format(WOVariables.wo_webroot)) - wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " - "grep -oP \"dev [^[:space:]]+ \" " - "| cut -d ' ' -f 2").read() - if (wo_wan != 'eth0' and wo_wan != ''): - WOFileUtils.searchreplace(self, - "{0}22222/htdocs/index.php" - .format(WOVariables.wo_webroot), - "eth0", - "{0}".format(wo_wan)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # Extplorer FileManager - if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting explorer.tar.gz " - "to location {0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', - '/var/lib/wo/tmp/') - shutil.move('/var/lib/wo/tmp/extplorer-{0}' - .format(WOVariables.wo_extplorer), - '{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # webgrind - if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file webgrind.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/webgrind.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directroy " - "{0}22222/htdocs/php" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/webgrind-master/', - '{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "/usr/local/bin/dot", "/usr/bin/dot") - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "Europe/Copenhagen", - WOVariables.wo_timezone) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "90", "100") - - Log.debug(self, "Setting Privileges of webroot permission to " - "{0}22222/htdocs/php/webgrind/ file " - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - # anemometer - if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file anemometer.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/anemometer.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory") - os.makedirs('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/Anemometer-master', - '{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)) - chars = ''.join(random.sample(string.ascii_letters, 8)) - try: - WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' - '/anemometer/install.sql' - .format(WOVariables.wo_webroot)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("Unable to import Anemometer database") - - WOMysql.execute(self, 'grant select on' - ' *.* to \'anemometer\'' - '@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get - ('mysql', - 'grant-host'), - chars)) - Log.debug(self, "grant all on slow-query-log.*" - " to anemometer@root_user" - " IDENTIFIED BY password ") - WOMysql.execute(self, 'grant all on slow_query_log.* to' - '\'anemometer\'@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get( - 'mysql', 'grant-host'), - chars), - errormsg="cannot grant priviledges", - log=False) - - # Custom Anemometer configuration - Log.debug(self, "configration Anemometer") - data = dict(host=WOVariables.wo_mysql_host, port='3306', - user='anemometer', password=chars) - wo_anemometer = open('{0}22222/htdocs/db/anemometer' - '/conf/config.inc.php' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') - self.app.render((data), 'anemometer.mustache', - out=wo_anemometer) - wo_anemometer.close() - - if any('/usr/bin/pt-query-advisor' == x[1] - for x in packages): - WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) - - # phpredisadmin - if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/cache/' - 'redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/cache/redis" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - if os.path.isfile("/usr/local/bin/composer"): - WOShellExec.cmd_exec(self, "sudo -u www-data -H " - "composer " - "create-project -n -s dev " - "erik-dubbelboer/php-redis-admin " - "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin ") - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/cache/file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - @expose(help="Install packages") def install(self, packages=[], apt_packages=[], disp_msg=True): """Start installation of packages""" @@ -1719,7 +391,7 @@ class WOStackController(CementBaseController): if (apt_packages) or (packages): Log.debug(self, "Calling pre_pref") - self.pre_pref(apt_packages) + pre_pref(self, apt_packages) if (apt_packages): meminfo = (os.popen('cat /proc/meminfo ' '| grep MemTotal').read()).split(":") @@ -1735,7 +407,7 @@ class WOStackController(CementBaseController): Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) Log.debug(self, "Calling post_pref") - self.post_pref(apt_packages, packages) + post_pref(self, apt_packages, packages) if 'redis-server' in apt_packages: # set redis.conf parameter # set maxmemory 10% for ram below 512MB and 20% for others diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py new file mode 100644 index 0000000..9b7a9ed --- /dev/null +++ b/wo/cli/plugins/stack_pref.py @@ -0,0 +1,1357 @@ +import codecs +import configparser +import os +import pwd +import random +import shutil +import string +import re +import requests + +import psutil +# from pynginxconfig import NginxConfig +from wo.cli.plugins.site_functions import * +from wo.cli.plugins.sitedb import * +from wo.cli.plugins.stack_services import WOStackStatusController +from wo.core.addswap import WOSwap +from wo.core.apt_repo import WORepo +from wo.core.aptget import WOAptGet +from wo.core.cron import WOCron +from wo.core.download import WODownload +from wo.core.extract import WOExtract +from wo.core.fileutils import WOFileUtils +from wo.core.git import WOGit +from wo.core.logging import Log +from wo.core.mysql import WOMysql +from wo.core.services import WOService +from wo.core.shellexec import CommandExecutionError, WOShellExec +from wo.core.variables import WOVariables + + +def pre_pref(self, apt_packages): + """Pre settings to do before installation packages""" + + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + # add mariadb repository excepted on raspbian and ubuntu 19.04 + if (not WOVariables.wo_distro == 'raspbian'): + Log.info(self, "Adding repository for MySQL, please wait...") + mysql_pref = ("Package: *\nPin: origin " + "sfo1.mirrors.digitalocean.com" + "\nPin-Priority: 1000\n") + with open('/etc/apt/preferences.d/' + 'MariaDB.pref', 'w') as mysql_pref_file: + mysql_pref_file.write(mysql_pref) + WORepo.add(self, repo_url=WOVariables.wo_mysql_repo) + Log.debug(self, 'Adding key for {0}' + .format(WOVariables.wo_mysql_repo)) + WORepo.add_key(self, '0xcbcb082a1bb943db', + keyserver="keyserver.ubuntu.com") + WORepo.add_key(self, '0xF1656F24C74CD1D8', + keyserver="keyserver.ubuntu.com") + # generate random 24 characters root password + chars = ''.join(random.sample(string.ascii_letters, 24)) + # configure MySQL non-interactive install + if (not WOVariables.wo_distro == 'raspbian'): + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + else: + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to initialize MySQL package") + # generate my.cnf root credentials + mysql_config = """ + [client] + user = root + password = {chars} + """.format(chars=chars) + config = configparser.ConfigParser() + config.read_string(mysql_config) + Log.debug(self, 'Writting configuration into MySQL file') + conf_path = "/etc/mysql/conf.d/my.cnf" + os.makedirs(os.path.dirname(conf_path), exist_ok=True) + with open(conf_path, encoding='utf-8', + mode='w') as configfile: + config.write(configfile) + Log.debug(self, 'Setting my.cnf permission') + WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) + + # add nginx repository + if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + if (WOVariables.wo_distro == 'ubuntu'): + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, ppa=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding ppa for Nginx') + else: + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding repository for Nginx') + WORepo.add_key(self, WOVariables.wo_nginx_key) + + # add php repository + if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or + set(WOVariables.wo_php).issubset(set(apt_packages))): + if (WOVariables.wo_distro == 'ubuntu'): + Log.info(self, "Adding repository for PHP, please wait...") + Log.debug(self, 'Adding ppa for PHP') + WORepo.add(self, ppa=WOVariables.wo_php_repo) + else: + Log.info(self, "Adding repository for PHP, please wait...") + # Add repository for php + if (WOVariables.wo_platform_codename == 'buster'): + php_pref = ("Package: *\nPin: origin " + "packages.sury.org" + "\nPin-Priority: 1000\n") + with open('/etc/apt/preferences.d/' + 'PHP.pref', 'w') as php_pref_file: + php_pref_file.write(php_pref) + Log.debug(self, 'Adding repo_url of php for debian') + WORepo.add(self, repo_url=WOVariables.wo_php_repo) + Log.debug(self, 'Adding deb.sury GPG key') + WORepo.add_key(self, WOVariables.wo_php_key) + # add redis repository + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + Log.info(self, "Adding repository for Redis, please wait...") + if WOVariables.wo_distro == 'ubuntu': + Log.debug(self, 'Adding ppa for redis') + WORepo.add(self, ppa=WOVariables.wo_redis_repo) + else: + Log.debug(self, 'Adding repo_url of redis for debian') + + +def post_pref(self, apt_packages, packages): + """Post activity after installation of packages""" + if (apt_packages): + + # Nginx configuration + if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + + # Fix for white screen death with NGINX PLUS + if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', + 'SCRIPT_FILENAME'): + with open('/etc/nginx/fastcgi_params', + encoding='utf-8', mode='a') as wo_nginx: + wo_nginx.write('fastcgi_param \tSCRIPT_FILENAME ' + '\t$request_filename;\n') + + if os.path.isfile('/etc/nginx/nginx.conf'): + data = dict(php="9000", debug="9001", + php7="9070", debug7="9170") + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/upstream.conf') + wo_nginx = open('/etc/nginx/conf.d/upstream.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'upstream.mustache', out=wo_nginx) + wo_nginx.close() + + data = dict(phpconf=True if + WOAptGet.is_installed(self, 'php7.2-fpm') + else False) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/stub_status.conf') + wo_nginx = open('/etc/nginx/conf.d/stub_status.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'stub_status.mustache', out=wo_nginx) + wo_nginx.close() + + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/webp.conf') + wo_nginx = open('/etc/nginx/conf.d/webp.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'webp.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/' + 'map-wp-fastcgi-cache.conf') + wo_nginx = open('/etc/nginx/conf.d/' + 'map-wp-fastcgi-cache.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'map-wp.mustache', + out=wo_nginx) + wo_nginx.close() + + # Setup Nginx common directory + if not os.path.exists('/etc/nginx/common'): + Log.debug(self, 'Creating directory' + '/etc/nginx/common') + os.makedirs('/etc/nginx/common') + + if os.path.exists('/etc/nginx/common'): + data = dict(webroot=WOVariables.wo_webroot) + + # Common Configuration + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/locations-wo.conf') + wo_nginx = open('/etc/nginx/common/locations-wo.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'locations.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsubdir.conf') + wo_nginx = open('/etc/nginx/common/wpsubdir.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsubdir.mustache', + out=wo_nginx) + wo_nginx.close() + + # PHP 7.2 conf + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php72.conf') + wo_nginx = open('/etc/nginx/common/php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php72.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() + + # PHP 7.3 conf + if os.path.isdir("/etc/nginx/common"): + data = dict() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php73.conf') + wo_nginx = open('/etc/nginx/common/php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php73.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + # create redis conf + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php73.conf') + wo_nginx = open('/etc/nginx/common/redis-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis-php7.mustache', + out=wo_nginx) + wo_nginx.close() + + with open("/etc/nginx/common/release", + "a") as release_file: + release_file.write("v{0}" + .format(WOVariables.wo_version)) + release_file.close() + + # Following files should not be overwrited + + if not os.path.isfile('/etc/nginx/common/acl.conf'): + data = dict(webroot=WOVariables.wo_webroot) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/acl.conf') + wo_nginx = open('/etc/nginx/common/acl.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'acl.mustache', + out=wo_nginx) + wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/blockips.conf') + wo_nginx = open('/etc/nginx/conf.d/blockips.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'blockips.mustache', out=wo_nginx) + wo_nginx.close() + + if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/fastcgi.conf') + wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'fastcgi.mustache', out=wo_nginx) + wo_nginx.close() + + # add redis cache format if not already done + if (os.path.isfile("/etc/nginx/nginx.conf") and + not os.path.isfile("/etc/nginx/conf.d" + "/redis.conf")): + with open("/etc/nginx/conf.d/" + "redis.conf", "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local] '\n" + "'$http_host \"$request\" $status" + " $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") + + # Nginx-Plus does not have nginx + # package structure like this + # So creating directories + if not os.path.exists('/etc/nginx/sites-available'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-available') + + if not os.path.exists('/etc/nginx/sites-enabled'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-enabled') + + # 22222 port settings + if not os.path.isfile("/etc/nginx/sites-available/22222"): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/sites-available/' + '22222') + wo_nginx = open('/etc/nginx/sites-available/22222', + encoding='utf-8', mode='w') + self.app.render((data), '22222.mustache', + out=wo_nginx) + wo_nginx.close() + + passwd = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(16)]) + try: + WOShellExec.cmd_exec(self, "printf \"WordOps:" + "$(openssl passwd -crypt " + "{password} 2> /dev/null)\n\"" + "> /etc/nginx/htpasswd-wo " + "2>/dev/null" + .format(password=passwd)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to save HTTP Auth") + + # Create Symbolic link for 22222 + WOFileUtils.create_symlink(self, ['/etc/nginx/' + 'sites-available/' + '22222', + '/etc/nginx/' + 'sites-enabled/' + '22222']) + # Create log and cert folder and softlinks + if not os.path.exists('{0}22222/logs' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/logs " + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/logs' + .format(WOVariables.wo_webroot)) + + if not os.path.exists('{0}22222/cert' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/cert" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/cert' + .format(WOVariables.wo_webroot)) + + if not os.path.exists('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/conf/nginx" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)) + + WOFileUtils.create_symlink(self, ['/var/log/nginx/' + '22222.access.log', + '{0}22222/' + 'logs/access.log' + .format(WOVariables.wo_webroot)] + ) + + WOFileUtils.create_symlink(self, ['/var/log/nginx/' + '22222.error.log', + '{0}22222/' + 'logs/error.log' + .format(WOVariables.wo_webroot)] + ) + + try: + WOShellExec.cmd_exec(self, "openssl genrsa -out " + "{0}22222/cert/22222.key 2048" + .format(WOVariables.wo_webroot)) + WOShellExec.cmd_exec(self, "openssl req -new -batch " + "-subj /commonName=localhost/ " + "-key {0}22222/cert/22222.key " + "-out {0}22222/cert/" + "22222.csr" + .format(WOVariables.wo_webroot)) + + WOFileUtils.mvfile(self, "{0}22222/cert/22222.key" + .format(WOVariables.wo_webroot), + "{0}22222/cert/" + "22222.key.org" + .format(WOVariables.wo_webroot)) + + WOShellExec.cmd_exec(self, "openssl rsa -in " + "{0}22222/cert/" + "22222.key.org -out " + "{0}22222/cert/22222.key" + .format(WOVariables.wo_webroot)) + + WOShellExec.cmd_exec(self, "openssl x509 -req -days " + "3652 -in {0}22222/cert/" + "22222.csr -signkey {0}" + "22222/cert/22222.key -out " + "{0}22222/cert/22222.crt" + .format(WOVariables.wo_webroot)) + + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error( + self, "Failed to generate HTTPS " + "certificate for 22222") + server_ip = requests.get('http://v4.wordops.eu') + + if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' + .format(WOVariables.wo_webroot)): + + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "a") as php_file: + php_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") + + # Nginx Configation into GIT + WOGit.add(self, + ["/etc/nginx"], msg="Adding Nginx into Git") + WOService.reload_service(self, 'nginx') + + if set(["nginx"]).issubset(set(apt_packages)): + + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) + else: + WOService.restart_service(self, 'nginx') + + # create nginx configuration for redis + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if os.path.isdir('/etc/nginx/common'): + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + + if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): + if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" + "upstream.conf", + "redis"): + with open("/etc/nginx/conf.d/upstream.conf", + "a") as redis_file: + redis_file.write("upstream redis {\n" + " server 127.0.0.1:6379;\n" + " keepalive 10;\n}\n") + + if os.path.isfile("/etc/nginx/nginx.conf"): + if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): + with open("/etc/nginx/conf.d/redis.conf", + "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local]" + " '\n '$http_host" + " \"$request\" " + "$status $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") + + if set(WOVariables.wo_php).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.2/'): + Log.debug(self, 'Creating directory /var/log/php/7.2/') + os.makedirs('/var/log/php/7.2/') + + # Parse etc/php/7.2/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file " + "/etc/php/7.2/fpm/php.ini") + config.read('/etc/php/7.2/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.2/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.2/fpm/php.ini") + config.write(configfile) + + # Parse /etc/php/7.2/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.2-fpm.pid", + error_log="/var/log/php/7.2/fpm.log", + include="/etc/php/7.2/fpm/pool.d/*.conf") + Log.debug(self, "writting php7.2 configuration into " + "/etc/php/7.2/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() + + # Parse /etc/php/7.2/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.2/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php72-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writing PHP 7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www.conf") + config.write(configfile) + + with open("/etc/php/7.2/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") + + # Generate /etc/php/7.2/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php72-two-fpm.sock' + with open('/etc/php/7.2/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www-two.conf") + config.write(confifile) + + # Generate /etc/php/7.2/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9172' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.2/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/debug.conf") + config.write(confifile) + + with open("/etc/php/7.2/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") + + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.2/mods-available/" + "xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.2/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") + + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") + + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) + + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.2-fpm') + + # PHP7.3 configuration + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.3/'): + Log.debug(self, 'Creating directory /var/log/php/7.3/') + os.makedirs('/var/log/php/7.3/') + + # Parse etc/php/7.3/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file /etc/php/7.3/" + "fpm/php.ini") + config.read('/etc/php/7.3/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.3/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.3/fpm/php.ini") + config.write(configfile) + + # Parse /etc/php/7.3/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.3-fpm.pid", + error_log="/var/log/php7.3-fpm.log", + include="/etc/php/7.3/fpm/pool.d/*.conf") + Log.debug(self, "writting php 7.3 configuration into " + "/etc/php/7.3/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() + + # Parse /etc/php/7.3/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.3/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php73-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www.conf") + config.write(configfile) + + with open("/etc/php/7.3/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") + + # Generate /etc/php/7.3/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php73-two-fpm.sock' + with open('/etc/php/7.3/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www-two.conf") + config.write(confifile) + + # Generate /etc/php/7.3/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9173' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.3/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/debug.conf") + config.write(confifile) + + with open("/etc/php/7.3/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") + + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.3/mods-available" + "/xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.3/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") + + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") + + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) + + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.3-fpm') + + # create mysql config if it doesn't exist + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if not os.path.isfile("/etc/mysql/my.cnf"): + config = ("[mysqld]\nwait_timeout = 30\n" + "interactive_timeout=60\nperformance_schema = 0" + "\nquery_cache_type = 1") + config_file = open("/etc/mysql/my.cnf", + encoding='utf-8', mode='w') + config_file.write(config) + config_file.close() + + WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' + '> /dev/null 2>&1', + comment='MySQL optimization cronjob ' + 'added by WordOps') + WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") + WOService.reload_service(self, 'mysql') + + # create fail2ban configuration files + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=fail2ban_config) + fail2ban_config.close() + + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=fail2ban_config) + fail2ban_config.close() + + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'nginx-forbidden.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-forbidden.mustache', + out=fail2ban_config) + fail2ban_config.close() + WOGit.add(self, ["/etc/fail2ban"], + msg="Adding Fail2ban into Git") + WOService.reload_service(self, 'fail2ban') + + # Proftpd configuration + if set(["proftpd-basic"]).issubset(set(apt_packages)): + if os.path.isfile("/etc/proftpd/proftpd.conf"): + Log.debug(self, "Setting up Proftpd configuration") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# DefaultRoot", + "DefaultRoot") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# RequireValidShell", + "RequireValidShell") + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "# PassivePorts " + " " + "49152 65534", + "PassivePorts " + " " + " 49000 50000") + # proftpd TLS configuration + if not os.path.isdir("/etc/proftpd/ssl"): + WOFileUtils.mkdir(self, "/etc/proftpd/ssl") + + try: + WOShellExec.cmd_exec(self, "openssl genrsa -out " + "/etc/proftpd/ssl/proftpd.key 2048") + WOShellExec.cmd_exec(self, "openssl req -new -batch " + "-subj /commonName=localhost/ " + "-key /etc/proftpd/ssl/proftpd.key " + "-out /etc/proftpd/ssl/proftpd.csr") + WOFileUtils.mvfile(self, "/etc/proftpd/ssl/proftpd.key", + "/etc/proftpd/ssl/proftpd.key.org") + WOShellExec.cmd_exec(self, "openssl rsa -in " + "/etc/proftpd/ssl/proftpd.key.org " + "-out /etc/proftpd/ssl/proftpd.key") + WOShellExec.cmd_exec(self, "openssl x509 -req -days " + "3652 -in /etc/proftpd/ssl/proftpd.csr " + "-signkey /etc/proftpd/ssl/proftpd.key " + " -out /etc/proftpd/ssl/proftpd.crt") + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error( + self, "Failed to generate SSL " + "certificate for Proftpd") + WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.key", 0o700) + WOFileUtils.chmod(self, "/etc/proftpd/ssl/proftpd.crt", 0o700) + data = dict() + Log.debug(self, 'Writting the proftpd configuration to ' + 'file /etc/proftpd/tls.conf') + wo_proftpdconf = open('/etc/proftpd/tls.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'proftpd-tls.mustache', + out=wo_proftpdconf) + wo_proftpdconf.close() + WOFileUtils.searchreplace(self, "/etc/proftpd/" + "proftpd.conf", + "#Include /etc/proftpd/tls.conf", + "Include /etc/proftpd/tls.conf") + WOService.restart_service(self, 'proftpd') + + # add rule for proftpd with UFW + if WOAptGet.is_installed(self, 'ufw'): + try: + WOShellExec.cmd_exec(self, "ufw allow " + "49000:50000/tcp") + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to add UFW rule") + + if os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + with open("/etc/fail2ban/jail.d/custom.conf", + encoding='utf-8', mode='a') as f2bproftpd: + f2bproftpd.write("\n\n[proftpd]\nenabled = true\n") + WOService.reload_service(self, 'fail2ban') + + WOGit.add(self, ["/etc/proftpd"], + msg="Adding ProFTPd into Git") + WOService.reload_service(self, 'proftpd') + + if (packages): + if any('/usr/local/bin/wp' == x[1] for x in packages): + Log.debug(self, "Setting Privileges" + " to /usr/local/bin/wp file ") + WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + + if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] + for x in packages): + WOExtract.extract( + self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' + 'location /var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/db" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', + '{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)) + shutil.copyfile('{0}22222/htdocs/db/pma' + '/config.sample.inc.php' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/db/pma/config.inc.php' + .format(WOVariables.wo_webroot)) + Log.debug(self, 'Setting Blowfish Secret Key ' + 'FOR COOKIE AUTH to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + blowfish_key = ''.join([random.choice + (string.ascii_letters + + string.digits) + for n in range(25)]) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'blowfish_secret\']" + " = \'\';", + "$cfg[\'blowfish_secret\']" + " = \'{0}\';" + .format(blowfish_key)) + Log.debug(self, 'Setting HOST Server For Mysql to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'Servers\'][$i][\'host\']" + " = \'localhost\';", "$cfg" + "[\'Servers\'][$i][\'host\'] = \'{0}\';" + .format(WOVariables.wo_mysql_host)) + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # composer install and phpmyadmin update + if any('/var/lib/wo/tmp/composer-install' == x[1] + for x in packages): + Log.info(self, "Installing composer, please wait...") + WOShellExec.cmd_exec(self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") + shutil.copyfile('/var/lib/wo/tmp/composer.phar', + '/usr/local/bin/composer') + WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) + Log.info(self, "Updating phpMyAdmin, please wait...") + WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " + "update -n --no-dev -d " + "/var/www/22222/htdocs/db/pma/") + + # netdata install + if any('/var/lib/wo/tmp/kickstart.sh' == x[1] + for x in packages): + if ((not os.path.exists('/opt/netdata')) and + (not os.path.exists('/etc/netdata'))): + Log.info(self, "Installing Netdata, please wait...") + WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" + "kickstart.sh " + "--dont-wait") + # disable mail notifications + WOFileUtils.searchreplace(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + 'SEND_EMAIL="YES"', + 'SEND_EMAIL="NO"') + # make changes persistant + WOFileUtils.copyfile(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + "/opt/netdata/etc/netdata/" + "health_alarm_notify.conf") + # check if mysql credentials are available + if os.path.isfile('/etc/mysql/conf.d/my.cnf'): + try: + WOMysql.execute(self, + "create user " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "grant usage on *.* to " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "flush privileges;", + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.info( + self, "fail to setup mysql user for netdata") + WOService.restart_service(self, 'netdata') + + # WordOps Dashboard + if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] + for x in packages): + if not os.path.isfile('{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting wo-dashboard.tar.gz " + "to location {0}22222/htdocs/" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/' + 'wo-dashboard.tar.gz', + '{0}22222/htdocs' + .format(WOVariables.wo_webroot)) + wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " + "grep -oP \"dev [^[:space:]]+ \" " + "| cut -d ' ' -f 2").read() + if (wo_wan != 'eth0' and wo_wan != ''): + WOFileUtils.searchreplace(self, + "{0}22222/htdocs/index.php" + .format(WOVariables.wo_webroot), + "eth0", + "{0}".format(wo_wan)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs" + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # Extplorer FileManager + if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting explorer.tar.gz " + "to location {0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', + '/var/lib/wo/tmp/') + shutil.move('/var/lib/wo/tmp/extplorer-{0}' + .format(WOVariables.wo_extplorer), + '{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + + # webgrind + if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file webgrind.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/webgrind.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directroy " + "{0}22222/htdocs/php" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/webgrind-master/', + '{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "/usr/local/bin/dot", "/usr/bin/dot") + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "Europe/Copenhagen", + WOVariables.wo_timezone) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "90", "100") + + Log.debug(self, "Setting Privileges of webroot permission to " + "{0}22222/htdocs/php/webgrind/ file " + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + # anemometer + if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file anemometer.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/anemometer.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory") + os.makedirs('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/Anemometer-master', + '{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)) + chars = ''.join(random.sample(string.ascii_letters, 8)) + try: + WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' + '/anemometer/install.sql' + .format(WOVariables.wo_webroot)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + raise SiteError("Unable to import Anemometer database") + + WOMysql.execute(self, 'grant select on' + ' *.* to \'anemometer\'' + '@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get + ('mysql', + 'grant-host'), + chars)) + Log.debug(self, "grant all on slow-query-log.*" + " to anemometer@root_user" + " IDENTIFIED BY password ") + WOMysql.execute(self, 'grant all on slow_query_log.* to' + '\'anemometer\'@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get( + 'mysql', 'grant-host'), + chars), + errormsg="cannot grant priviledges", + log=False) + + # Custom Anemometer configuration + Log.debug(self, "configration Anemometer") + data = dict(host=WOVariables.wo_mysql_host, port='3306', + user='anemometer', password=chars) + wo_anemometer = open('{0}22222/htdocs/db/anemometer' + '/conf/config.inc.php' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') + self.app.render((data), 'anemometer.mustache', + out=wo_anemometer) + wo_anemometer.close() + + if any('/usr/bin/pt-query-advisor' == x[1] + for x in packages): + WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + + # phpredisadmin + if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/cache/' + 'redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/cache/redis" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + if os.path.isfile("/usr/local/bin/composer"): + WOShellExec.cmd_exec(self, "sudo -u www-data -H " + "composer " + "create-project -n -s dev " + "erik-dubbelboer/php-redis-admin " + "/var/www/22222/htdocs/cache" + "/redis/phpRedisAdmin ") + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/cache/file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 5af52b8..9edad10 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -1,9 +1,8 @@ import os import shutil -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose -from wo.cli.plugins.stack import WOStackController +from cement.core import handler, hook from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.download import WODownload @@ -13,6 +12,7 @@ from wo.core.logging import Log from wo.core.services import WOService from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables +from wo.cli.plugins.stack_pref import post_pref class WOStackUpgradeController(CementBaseController): @@ -86,8 +86,6 @@ class WOStackUpgradeController(CementBaseController): apt_packages = [] packages = [] empty_packages = [] - stack = WOStackController() - stack.app = self.app if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and (not self.app.pargs.php) and (not self.app.pargs.mysql) and @@ -192,7 +190,7 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) - stack.post_pref(apt_packages, empty_packages) + post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') From dc81bfc27eaf2d914b600e6c7440d85bd96120c5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 05:11:13 +0200 Subject: [PATCH 047/102] Fix travis non-interactive --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c54a975..c4ba976 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ script: - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - - sudo wo stack upgrade --nginx - - sudo wo stack upgrade --php + - sudo wo stack upgrade --nginx --no-prompt + - sudo wo stack upgrade --php --no-prompt - sudo wo stack upgrade --netdata --no-prompt - sudo wo stack upgrade --phpmyadmin --no-prompt - sudo wo stack upgrade --composer --no-prompt From f3358f4314c048eece1aa41d09e07871085530f3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 06:57:39 +0200 Subject: [PATCH 048/102] Fix stack pref --- wo/cli/plugins/stack_pref.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 9b7a9ed..2667045 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -9,7 +9,6 @@ import re import requests import psutil -# from pynginxconfig import NginxConfig from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_services import WOStackStatusController From 881e2bc589151bd01d497c502d515127481ab313 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 07:49:08 +0200 Subject: [PATCH 049/102] Fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4ba976..f5164aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ script: - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis - sudo wp --allow-root --info - - sudo wo info + - sudo wo info || tail -a /var/log/wo/wordops.log - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - sudo wo stack upgrade --nginx --no-prompt From fdd60943ace4f43285d62817a4639afb78a9e948 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 07:49:35 +0200 Subject: [PATCH 050/102] Set back xenial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5164aa..c4ba976 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ script: - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis - sudo wp --allow-root --info - - sudo wo info || tail -a /var/log/wo/wordops.log + - sudo wo info - sudo cat /etc/nginx/nginx.conf - sudo cat /etc/mysql/my.cnf - sudo wo stack upgrade --nginx --no-prompt From 1a7be6215ca134ccedeeb2cea6bdc5eef23b57a9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 09:45:08 +0200 Subject: [PATCH 051/102] Reindent stack_pref --- wo/cli/plugins/stack_pref.py | 1156 +++++++++++++++++----------------- 1 file changed, 580 insertions(+), 576 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2667045..1e71bb2 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -228,241 +228,241 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - # Setup Nginx common directory - if not os.path.exists('/etc/nginx/common'): - Log.debug(self, 'Creating directory' - '/etc/nginx/common') - os.makedirs('/etc/nginx/common') + # Setup Nginx common directory + if not os.path.exists('/etc/nginx/common'): + Log.debug(self, 'Creating directory' + '/etc/nginx/common') + os.makedirs('/etc/nginx/common') - if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) + if os.path.exists('/etc/nginx/common'): + data = dict(webroot=WOVariables.wo_webroot) - # Common Configuration - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', - out=wo_nginx) - wo_nginx.close() + # Common Configuration + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/locations-wo.conf') + wo_nginx = open('/etc/nginx/common/locations-wo.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'locations.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsubdir.conf') - wo_nginx = open('/etc/nginx/common/wpsubdir.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsubdir.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsubdir.conf') + wo_nginx = open('/etc/nginx/common/wpsubdir.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsubdir.mustache', + out=wo_nginx) + wo_nginx.close() - # PHP 7.2 conf - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php72.conf') - wo_nginx = open('/etc/nginx/common/php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php.mustache', - out=wo_nginx) - wo_nginx.close() + # PHP 7.2 conf + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php72.conf') + wo_nginx = open('/etc/nginx/common/php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php72.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php72.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php72.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php72.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() - # PHP 7.3 conf - if os.path.isdir("/etc/nginx/common"): - data = dict() + # PHP 7.3 conf + if os.path.isdir("/etc/nginx/common"): + data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php73.conf') - wo_nginx = open('/etc/nginx/common/php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/php73.conf') + wo_nginx = open('/etc/nginx/common/php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php73.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpcommon-php73.conf') + wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpcommon-php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpfc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpfc-php7.mustache', + out=wo_nginx) + wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', - out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpsc-php73.conf') + wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpsc-php7.mustache', + out=wo_nginx) + wo_nginx.close() - # create redis conf - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() + # create redis conf + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', - out=wo_nginx) - wo_nginx.close() + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php73.conf') + wo_nginx = open('/etc/nginx/common/redis-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis-php7.mustache', + out=wo_nginx) + wo_nginx.close() - with open("/etc/nginx/common/release", - "a") as release_file: - release_file.write("v{0}" - .format(WOVariables.wo_version)) - release_file.close() + with open("/etc/nginx/common/release", + "a") as release_file: + release_file.write("v{0}" + .format(WOVariables.wo_version)) + release_file.close() - # Following files should not be overwrited + # Following files should not be overwrited - if not os.path.isfile('/etc/nginx/common/acl.conf'): - data = dict(webroot=WOVariables.wo_webroot) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/acl.conf') - wo_nginx = open('/etc/nginx/common/acl.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'acl.mustache', - out=wo_nginx) - wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/blockips.conf') - wo_nginx = open('/etc/nginx/conf.d/blockips.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'blockips.mustache', out=wo_nginx) - wo_nginx.close() + if not os.path.isfile('/etc/nginx/common/acl.conf'): + data = dict(webroot=WOVariables.wo_webroot) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/acl.conf') + wo_nginx = open('/etc/nginx/common/acl.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'acl.mustache', + out=wo_nginx) + wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/blockips.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/blockips.conf') + wo_nginx = open('/etc/nginx/conf.d/blockips.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'blockips.mustache', out=wo_nginx) + wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/fastcgi.conf') - wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'fastcgi.mustache', out=wo_nginx) - wo_nginx.close() + if not os.path.isfile('/etc/nginx/conf.d/fastcgi.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/fastcgi.conf') + wo_nginx = open('/etc/nginx/conf.d/fastcgi.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'fastcgi.mustache', out=wo_nginx) + wo_nginx.close() - # add redis cache format if not already done - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d" - "/redis.conf")): - with open("/etc/nginx/conf.d/" - "redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") + # add redis cache format if not already done + if (os.path.isfile("/etc/nginx/nginx.conf") and + not os.path.isfile("/etc/nginx/conf.d" + "/redis.conf")): + with open("/etc/nginx/conf.d/" + "redis.conf", "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local] '\n" + "'$http_host \"$request\" $status" + " $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") # Nginx-Plus does not have nginx # package structure like this # So creating directories - if not os.path.exists('/etc/nginx/sites-available'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-available') + if not os.path.exists('/etc/nginx/sites-available'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-available') - if not os.path.exists('/etc/nginx/sites-enabled'): - Log.debug(self, 'Creating directory' - '/etc/nginx/sites-available') - os.makedirs('/etc/nginx/sites-enabled') + if not os.path.exists('/etc/nginx/sites-enabled'): + Log.debug(self, 'Creating directory' + '/etc/nginx/sites-available') + os.makedirs('/etc/nginx/sites-enabled') - # 22222 port settings - if not os.path.isfile("/etc/nginx/sites-available/22222"): - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/sites-available/' - '22222') - wo_nginx = open('/etc/nginx/sites-available/22222', - encoding='utf-8', mode='w') - self.app.render((data), '22222.mustache', - out=wo_nginx) - wo_nginx.close() + # 22222 port settings + if not os.path.isfile("/etc/nginx/sites-available/22222"): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/sites-available/' + '22222') + wo_nginx = open('/etc/nginx/sites-available/22222', + encoding='utf-8', mode='w') + self.app.render((data), '22222.mustache', + out=wo_nginx) + wo_nginx.close() - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) - try: - WOShellExec.cmd_exec(self, "printf \"WordOps:" - "$(openssl passwd -crypt " - "{password} 2> /dev/null)\n\"" - "> /etc/nginx/htpasswd-wo " - "2>/dev/null" - .format(password=passwd)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to save HTTP Auth") + passwd = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(16)]) + try: + WOShellExec.cmd_exec(self, "printf \"WordOps:" + "$(openssl passwd -crypt " + "{password} 2> /dev/null)\n\"" + "> /etc/nginx/htpasswd-wo " + "2>/dev/null" + .format(password=passwd)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Failed to save HTTP Auth") # Create Symbolic link for 22222 - WOFileUtils.create_symlink(self, ['/etc/nginx/' - 'sites-available/' - '22222', - '/etc/nginx/' - 'sites-enabled/' - '22222']) - # Create log and cert folder and softlinks - if not os.path.exists('{0}22222/logs' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/logs " - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/logs' - .format(WOVariables.wo_webroot)) + WOFileUtils.create_symlink(self, ['/etc/nginx/' + 'sites-available/' + '22222', + '/etc/nginx/' + 'sites-enabled/' + '22222']) + # Create log and cert folder and softlinks + if not os.path.exists('{0}22222/logs' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/logs " + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/logs' + .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/cert' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/cert" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/cert' - .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/cert' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/cert" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/cert' + .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory " - "{0}22222/conf/nginx" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/conf/nginx' - .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory " + "{0}22222/conf/nginx" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/conf/nginx' + .format(WOVariables.wo_webroot)) WOFileUtils.create_symlink(self, ['/var/log/nginx/' '22222.access.log', @@ -525,75 +525,74 @@ def post_pref(self, apt_packages, packages): "ssl_certificate_key " "/var/www/22222/cert/22222.key;\n") - # Nginx Configation into GIT - WOGit.add(self, - ["/etc/nginx"], msg="Adding Nginx into Git") - WOService.reload_service(self, 'nginx') + # Nginx Configation into GIT + WOGit.add(self, + ["/etc/nginx"], msg="Adding Nginx into Git") + WOService.reload_service(self, 'nginx') - if set(["nginx"]).issubset(set(apt_packages)): + if set(["nginx"]).issubset(set(apt_packages)): + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) + else: + WOService.restart_service(self, 'nginx') - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) - else: - WOService.restart_service(self, 'nginx') + # create nginx configuration for redis + if set(WOVariables.wo_redis).issubset(set(apt_packages)): + if os.path.isdir('/etc/nginx/common'): + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() - # create nginx configuration for redis - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - if os.path.isdir('/etc/nginx/common'): - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() + if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): + if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" + "upstream.conf", + "redis"): + with open("/etc/nginx/conf.d/upstream.conf", + "a") as redis_file: + redis_file.write("upstream redis {\n" + " server 127.0.0.1:6379;\n" + " keepalive 10;\n}\n") - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}\n") + if os.path.isfile("/etc/nginx/nginx.conf"): + if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): + with open("/etc/nginx/conf.d/redis.conf", + "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local]" + " '\n '$http_host" + " \"$request\" " + "$status $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") - if os.path.isfile("/etc/nginx/nginx.conf"): - if not os.path.isfile("/etc/nginx/conf.d/redis.conf"): - with open("/etc/nginx/conf.d/redis.conf", - "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local]" - " '\n '$http_host" - " \"$request\" " - "$status $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") - - if set(WOVariables.wo_php).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.2/'): - Log.debug(self, 'Creating directory /var/log/php/7.2/') - os.makedirs('/var/log/php/7.2/') + if set(WOVariables.wo_php).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.2/'): + Log.debug(self, 'Creating directory /var/log/php/7.2/') + os.makedirs('/var/log/php/7.2/') # Parse etc/php/7.2/fpm/php.ini config = configparser.ConfigParser() @@ -752,12 +751,12 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.2-fpm') - # PHP7.3 configuration - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - # Create log directories - if not os.path.exists('/var/log/php/7.3/'): - Log.debug(self, 'Creating directory /var/log/php/7.3/') - os.makedirs('/var/log/php/7.3/') + # PHP7.3 configuration + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + # Create log directories + if not os.path.exists('/var/log/php/7.3/'): + Log.debug(self, 'Creating directory /var/log/php/7.3/') + os.makedirs('/var/log/php/7.3/') # Parse etc/php/7.3/fpm/php.ini config = configparser.ConfigParser() @@ -916,16 +915,16 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.3-fpm') - # create mysql config if it doesn't exist - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - if not os.path.isfile("/etc/mysql/my.cnf"): - config = ("[mysqld]\nwait_timeout = 30\n" - "interactive_timeout=60\nperformance_schema = 0" - "\nquery_cache_type = 1") - config_file = open("/etc/mysql/my.cnf", - encoding='utf-8', mode='w') - config_file.write(config) - config_file.close() + # create mysql config if it doesn't exist + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if not os.path.isfile("/etc/mysql/my.cnf"): + config = ("[mysqld]\nwait_timeout = 30\n" + "interactive_timeout=60\nperformance_schema = 0" + "\nquery_cache_type = 1") + config_file = open("/etc/mysql/my.cnf", + encoding='utf-8', mode='w') + config_file.write(config) + config_file.close() WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' @@ -935,32 +934,32 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") WOService.reload_service(self, 'mysql') - # create fail2ban configuration files - if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): - if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - data = dict() - Log.debug(self, "Setting up fail2ban jails configuration") - fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban.mustache', - out=fail2ban_config) - fail2ban_config.close() + # create fail2ban configuration files + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + fail2ban_config = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=fail2ban_config) + fail2ban_config.close() - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'wo-wordpress.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-wp.mustache', - out=fail2ban_config) - fail2ban_config.close() + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=fail2ban_config) + fail2ban_config.close() - Log.debug(self, "Setting up fail2ban wp filter") - fail2ban_config = open('/etc/fail2ban/filter.d/' - 'nginx-forbidden.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-forbidden.mustache', - out=fail2ban_config) - fail2ban_config.close() + Log.debug(self, "Setting up fail2ban wp filter") + fail2ban_config = open('/etc/fail2ban/filter.d/' + 'nginx-forbidden.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-forbidden.mustache', + out=fail2ban_config) + fail2ban_config.close() WOGit.add(self, ["/etc/fail2ban"], msg="Adding Fail2ban into Git") WOService.reload_service(self, 'fail2ban') @@ -1045,63 +1044,63 @@ def post_pref(self, apt_packages, packages): msg="Adding ProFTPd into Git") WOService.reload_service(self, 'proftpd') - if (packages): - if any('/usr/local/bin/wp' == x[1] for x in packages): - Log.debug(self, "Setting Privileges" - " to /usr/local/bin/wp file ") - WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + if (packages): + if any('/usr/local/bin/wp' == x[1] for x in packages): + Log.debug(self, "Setting Privileges" + " to /usr/local/bin/wp file ") + WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] - for x in packages): - WOExtract.extract( - self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') - Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' - 'location /var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/db" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/db' + if any('/var/lib/wo/tmp/pma.tar.gz' == x[1] + for x in packages): + WOExtract.extract( + self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') + Log.debug(self, 'Extracting file /var/lib/wo/tmp/pma.tar.gz to ' + 'location /var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/db" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/db' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', + '{0}22222/htdocs/db/pma/' + .format(WOVariables.wo_webroot)) + shutil.copyfile('{0}22222/htdocs/db/pma' + '/config.sample.inc.php' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/db/pma/config.inc.php' .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/phpmyadmin-STABLE/', - '{0}22222/htdocs/db/pma/' - .format(WOVariables.wo_webroot)) - shutil.copyfile('{0}22222/htdocs/db/pma' - '/config.sample.inc.php' - .format(WOVariables.wo_webroot), - '{0}22222/htdocs/db/pma/config.inc.php' - .format(WOVariables.wo_webroot)) - Log.debug(self, 'Setting Blowfish Secret Key ' - 'FOR COOKIE AUTH to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - blowfish_key = ''.join([random.choice - (string.ascii_letters + - string.digits) - for n in range(25)]) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'blowfish_secret\']" - " = \'\';", - "$cfg[\'blowfish_secret\']" - " = \'{0}\';" - .format(blowfish_key)) - Log.debug(self, 'Setting HOST Server For Mysql to ' - '{0}22222/htdocs/db/pma/config.inc.php file ' - .format(WOVariables.wo_webroot)) - WOFileUtils.searchreplace(self, - '{0}22222/htdocs/db/pma' - '/config.inc.php' - .format(WOVariables.wo_webroot), - "$cfg[\'Servers\'][$i][\'host\']" - " = \'localhost\';", "$cfg" - "[\'Servers\'][$i][\'host\'] = \'{0}\';" - .format(WOVariables.wo_mysql_host)) + Log.debug(self, 'Setting Blowfish Secret Key ' + 'FOR COOKIE AUTH to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + blowfish_key = ''.join([random.choice + (string.ascii_letters + + string.digits) + for n in range(25)]) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'blowfish_secret\']" + " = \'\';", + "$cfg[\'blowfish_secret\']" + " = \'{0}\';" + .format(blowfish_key)) + Log.debug(self, 'Setting HOST Server For Mysql to ' + '{0}22222/htdocs/db/pma/config.inc.php file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.searchreplace(self, + '{0}22222/htdocs/db/pma' + '/config.inc.php' + .format(WOVariables.wo_webroot), + "$cfg[\'Servers\'][$i][\'host\']" + " = \'localhost\';", "$cfg" + "[\'Servers\'][$i][\'host\'] = \'{0}\';" + .format(WOVariables.wo_mysql_host)) Log.debug(self, 'Setting Privileges of webroot permission to ' '{0}22222/htdocs/db/pma file '.format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222'.format(WOVariables.wo_webroot), @@ -1109,74 +1108,79 @@ def post_pref(self, apt_packages, packages): WOVariables.wo_php_user, recursive=True) - # composer install and phpmyadmin update - if any('/var/lib/wo/tmp/composer-install' == x[1] - for x in packages): - Log.info(self, "Installing composer, please wait...") - WOShellExec.cmd_exec(self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") - shutil.copyfile('/var/lib/wo/tmp/composer.phar', - '/usr/local/bin/composer') - WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " - "update -n --no-dev -d " - "/var/www/22222/htdocs/db/pma/") + # composer install and phpmyadmin update + if any('/var/lib/wo/tmp/composer-install' == x[1] + for x in packages): + Log.info(self, "Installing composer, please wait...") + WOShellExec.cmd_exec(self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") + shutil.copyfile('/var/lib/wo/tmp/composer.phar', + '/usr/local/bin/composer') + WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) + Log.info(self, "Updating phpMyAdmin, please wait...") + WOShellExec.cmd_exec(self, "/usr/local/bin/composer update --no-plugins --no-scripts " + "-n --no-dev -d " + "/var/www/22222/htdocs/db/pma/") + WOFileUtils.chown(self, '{0}22222/htdocs/db/pma' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) - # netdata install - if any('/var/lib/wo/tmp/kickstart.sh' == x[1] - for x in packages): - if ((not os.path.exists('/opt/netdata')) and - (not os.path.exists('/etc/netdata'))): - Log.info(self, "Installing Netdata, please wait...") - WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" - "kickstart.sh " - "--dont-wait") - # disable mail notifications - WOFileUtils.searchreplace(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - 'SEND_EMAIL="YES"', - 'SEND_EMAIL="NO"') - # make changes persistant - WOFileUtils.copyfile(self, "/opt/netdata/usr/" - "lib/netdata/conf.d/" - "health_alarm_notify.conf", - "/opt/netdata/etc/netdata/" - "health_alarm_notify.conf") - # check if mysql credentials are available - if os.path.isfile('/etc/mysql/conf.d/my.cnf'): - try: - WOMysql.execute(self, - "create user " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "grant usage on *.* to " - "'netdata'@'localhost';", - log=False) - WOMysql.execute(self, - "flush privileges;", - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.info( - self, "fail to setup mysql user for netdata") - WOService.restart_service(self, 'netdata') + # netdata install + if any('/var/lib/wo/tmp/kickstart.sh' == x[1] + for x in packages): + if ((not os.path.exists('/opt/netdata')) and + (not os.path.exists('/etc/netdata'))): + Log.info(self, "Installing Netdata, please wait...") + WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" + "kickstart.sh " + "--dont-wait") + # disable mail notifications + WOFileUtils.searchreplace(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + 'SEND_EMAIL="YES"', + 'SEND_EMAIL="NO"') + # make changes persistant + WOFileUtils.copyfile(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + "/opt/netdata/etc/netdata/" + "health_alarm_notify.conf") + # check if mysql credentials are available + if os.path.isfile('/etc/mysql/conf.d/my.cnf'): + try: + WOMysql.execute(self, + "create user " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "grant usage on *.* to " + "'netdata'@'localhost';", + log=False) + WOMysql.execute(self, + "flush privileges;", + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.info( + self, "fail to setup mysql user for netdata") + WOService.restart_service(self, 'netdata') - # WordOps Dashboard - if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] - for x in packages): - if not os.path.isfile('{0}22222/htdocs/index.php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting wo-dashboard.tar.gz " - "to location {0}22222/htdocs/" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/' - 'wo-dashboard.tar.gz', - '{0}22222/htdocs' - .format(WOVariables.wo_webroot)) + # WordOps Dashboard + if any('/var/lib/wo/tmp/wo-dashboard.tar.gz' == x[1] + for x in packages): + if not os.path.isfile('{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting wo-dashboard.tar.gz " + "to location {0}22222/htdocs/" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/' + 'wo-dashboard.tar.gz', + '{0}22222/htdocs' + .format(WOVariables.wo_webroot)) wo_wan = os.popen("/sbin/ip -4 route get 8.8.8.8 | " "grep -oP \"dev [^[:space:]]+ \" " "| cut -d ' ' -f 2").read() @@ -1195,162 +1199,162 @@ def post_pref(self, apt_packages, packages): WOVariables.wo_php_user, recursive=True) - # Extplorer FileManager - if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Extracting explorer.tar.gz " - "to location {0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', - '/var/lib/wo/tmp/') - shutil.move('/var/lib/wo/tmp/extplorer-{0}' - .format(WOVariables.wo_extplorer), - '{0}22222/htdocs/files' - .format(WOVariables.wo_webroot)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/files" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - - # webgrind - if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file webgrind.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/webgrind.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directroy " - "{0}22222/htdocs/php" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/webgrind-master/', - '{0}22222/htdocs/php/webgrind' - .format(WOVariables.wo_webroot)) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "/usr/local/bin/dot", "/usr/bin/dot") - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "Europe/Copenhagen", - WOVariables.wo_timezone) - - WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" - "config.php" - .format(WOVariables.wo_webroot), - "90", "100") - - Log.debug(self, "Setting Privileges of webroot permission to " - "{0}22222/htdocs/php/webgrind/ file " + # Extplorer FileManager + if any('/var/lib/wo/tmp/extplorer.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Extracting explorer.tar.gz " + "to location {0}22222/htdocs/files" + .format(WOVariables.wo_webroot)) + WOExtract.extract(self, '/var/lib/wo/tmp/extplorer.tar.gz', + '/var/lib/wo/tmp/') + shutil.move('/var/lib/wo/tmp/extplorer-{0}' + .format(WOVariables.wo_extplorer), + '{0}22222/htdocs/files' + .format(WOVariables.wo_webroot)) + Log.debug(self, "Setting Privileges to " + "{0}22222/htdocs/files" .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222' .format(WOVariables.wo_webroot), WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) - # anemometer - if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting file anemometer.tar.gz to " - "location /var/lib/wo/tmp/ ") - WOExtract.extract( - self, '/var/lib/wo/tmp/anemometer.tar.gz', - '/var/lib/wo/tmp/') - if not os.path.exists('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating directory") - os.makedirs('{0}22222/htdocs/db/' - .format(WOVariables.wo_webroot)) - if not os.path.exists('{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)): - shutil.move('/var/lib/wo/tmp/Anemometer-master', - '{0}22222/htdocs/db/anemometer' - .format(WOVariables.wo_webroot)) - chars = ''.join(random.sample(string.ascii_letters, 8)) - try: - WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' - '/anemometer/install.sql' - .format(WOVariables.wo_webroot)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - raise SiteError("Unable to import Anemometer database") - WOMysql.execute(self, 'grant select on' - ' *.* to \'anemometer\'' - '@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get - ('mysql', - 'grant-host'), - chars)) - Log.debug(self, "grant all on slow-query-log.*" - " to anemometer@root_user" - " IDENTIFIED BY password ") - WOMysql.execute(self, 'grant all on slow_query_log.* to' - '\'anemometer\'@\'{0}\' IDENTIFIED' - ' BY \'{1}\''.format(self.app.config.get( - 'mysql', 'grant-host'), - chars), - errormsg="cannot grant priviledges", - log=False) - - # Custom Anemometer configuration - Log.debug(self, "configration Anemometer") - data = dict(host=WOVariables.wo_mysql_host, port='3306', - user='anemometer', password=chars) - wo_anemometer = open('{0}22222/htdocs/db/anemometer' - '/conf/config.inc.php' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') - self.app.render((data), 'anemometer.mustache', - out=wo_anemometer) - wo_anemometer.close() - - if any('/usr/bin/pt-query-advisor' == x[1] - for x in packages): - WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) - - # phpredisadmin - if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] - for x in packages): - if not os.path.exists('{0}22222/htdocs/cache/' - 'redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)): - Log.debug(self, "Creating new directory " - "{0}22222/htdocs/cache/redis" - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) - if os.path.isfile("/usr/local/bin/composer"): - WOShellExec.cmd_exec(self, "sudo -u www-data -H " - "composer " - "create-project -n -s dev " - "erik-dubbelboer/php-redis-admin " - "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin ") - Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/cache/file ' + # webgrind + if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file webgrind.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/webgrind.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directroy " + "{0}22222/htdocs/php" .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/webgrind-master/', + '{0}22222/htdocs/php/webgrind' + .format(WOVariables.wo_webroot)) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "/usr/local/bin/dot", "/usr/bin/dot") + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "Europe/Copenhagen", + WOVariables.wo_timezone) + + WOFileUtils.searchreplace(self, "{0}22222/htdocs/php/webgrind/" + "config.php" + .format(WOVariables.wo_webroot), + "90", "100") + + Log.debug(self, "Setting Privileges of webroot permission to " + "{0}22222/htdocs/php/webgrind/ file " + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) + # anemometer + if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] + for x in packages): + Log.debug(self, "Extracting file anemometer.tar.gz to " + "location /var/lib/wo/tmp/ ") + WOExtract.extract( + self, '/var/lib/wo/tmp/anemometer.tar.gz', + '/var/lib/wo/tmp/') + if not os.path.exists('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating directory") + os.makedirs('{0}22222/htdocs/db/' + .format(WOVariables.wo_webroot)) + if not os.path.exists('{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)): + shutil.move('/var/lib/wo/tmp/Anemometer-master', + '{0}22222/htdocs/db/anemometer' + .format(WOVariables.wo_webroot)) + chars = ''.join(random.sample(string.ascii_letters, 8)) + try: + WOShellExec.cmd_exec(self, 'mysql < {0}22222/htdocs/db' + '/anemometer/install.sql' + .format(WOVariables.wo_webroot)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + raise SiteError("Unable to import Anemometer database") + + WOMysql.execute(self, 'grant select on' + ' *.* to \'anemometer\'' + '@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get + ('mysql', + 'grant-host'), + chars)) + Log.debug(self, "grant all on slow-query-log.*" + " to anemometer@root_user" + " IDENTIFIED BY password ") + WOMysql.execute(self, 'grant all on slow_query_log.* to' + '\'anemometer\'@\'{0}\' IDENTIFIED' + ' BY \'{1}\''.format(self.app.config.get( + 'mysql', 'grant-host'), + chars), + errormsg="cannot grant priviledges", + log=False) + + # Custom Anemometer configuration + Log.debug(self, "configration Anemometer") + data = dict(host=WOVariables.wo_mysql_host, port='3306', + user='anemometer', password=chars) + wo_anemometer = open('{0}22222/htdocs/db/anemometer' + '/conf/config.inc.php' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') + self.app.render((data), 'anemometer.mustache', + out=wo_anemometer) + wo_anemometer.close() + + if any('/usr/bin/pt-query-advisor' == x[1] + for x in packages): + WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + + # phpredisadmin + if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] + for x in packages): + if not os.path.exists('{0}22222/htdocs/cache/' + 'redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)): + Log.debug(self, "Creating new directory " + "{0}22222/htdocs/cache/redis" + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222' .format(WOVariables.wo_webroot), WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) + if os.path.isfile("/usr/local/bin/composer"): + WOShellExec.cmd_exec(self, "sudo -u www-data -H " + "composer " + "create-project -n -s dev " + "erik-dubbelboer/php-redis-admin " + "/var/www/22222/htdocs/cache" + "/redis/phpRedisAdmin ") + Log.debug(self, 'Setting Privileges of webroot permission to ' + '{0}22222/htdocs/cache/file ' + .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) From 7c6642cce5817ebc276da0613448760e5b5005c8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 11:46:45 +0200 Subject: [PATCH 052/102] Another fix for stack_pref --- wo/cli/plugins/stack_pref.py | 40 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 1e71bb2..01b3642 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1,22 +1,16 @@ import codecs import configparser import os -import pwd import random import shutil import string -import re import requests -import psutil from wo.cli.plugins.site_functions import * -from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_services import WOStackStatusController -from wo.core.addswap import WOSwap from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.cron import WOCron -from wo.core.download import WODownload from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit @@ -530,23 +524,23 @@ def post_pref(self, apt_packages, packages): ["/etc/nginx"], msg="Adding Nginx into Git") WOService.reload_service(self, 'nginx') - if set(["nginx"]).issubset(set(apt_packages)): - print("WordOps backend configuration was successful\n" - "You can access it on : https://{0}:22222" - .format(server_ip)) - print("HTTP Auth User Name: WordOps" + - "\nHTTP Auth Password : {0}".format(passwd)) - WOService.reload_service(self, 'nginx') - else: - self.msg = (self.msg + ["HTTP Auth User " - "Name: WordOps"] + - ["HTTP Auth Password : {0}" - .format(passwd)]) - self.msg = (self.msg + ["WordOps backend is available " - "on https://{0}:22222 " - "or https://{1}:22222" - .format(server_ip.text, - WOVariables.wo_fqdn)]) + if set(["nginx"]).issubset(set(apt_packages)): + print("WordOps backend configuration was successful\n" + "You can access it on : https://{0}:22222" + .format(server_ip)) + print("HTTP Auth User Name: WordOps" + + "\nHTTP Auth Password : {0}".format(passwd)) + WOService.reload_service(self, 'nginx') + else: + self.msg = (self.msg + ["HTTP Auth User " + "Name: WordOps"] + + ["HTTP Auth Password : {0}" + .format(passwd)]) + self.msg = (self.msg + ["WordOps backend is available " + "on https://{0}:22222 " + "or https://{1}:22222" + .format(server_ip.text, + WOVariables.wo_fqdn)]) else: WOService.restart_service(self, 'nginx') From 8c3cbc7917e9aa534fbc710727fb37cb09988fbb Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:25:00 +0200 Subject: [PATCH 053/102] Fix php upgrade --- wo/cli/plugins/stack_pref.py | 574 +++++++++++++++++------------------ 1 file changed, 287 insertions(+), 287 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 01b3642..23d652a 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -412,16 +412,16 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(16)]) + randompass = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(24)]) try: WOShellExec.cmd_exec(self, "printf \"WordOps:" "$(openssl passwd -crypt " "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo " "2>/dev/null" - .format(password=passwd)) + .format(password=randompass)) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Failed to save HTTP Auth") @@ -589,161 +589,161 @@ def post_pref(self, apt_packages, packages): os.makedirs('/var/log/php/7.2/') # Parse etc/php/7.2/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file " + config = configparser.ConfigParser() + Log.debug(self, "configuring php file " + "/etc/php/7.2/fpm/php.ini") + config.read('/etc/php/7.2/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.2/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " "/etc/php/7.2/fpm/php.ini") - config.read('/etc/php/7.2/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.2/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.2/fpm/php.ini") - config.write(configfile) + config.write(configfile) - # Parse /etc/php/7.2/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.2-fpm.pid", - error_log="/var/log/php/7.2/fpm.log", + # Parse /etc/php/7.2/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.2-fpm.pid", + error_log="/var/log/php/7.2/fpm.log", include="/etc/php/7.2/fpm/pool.d/*.conf") - Log.debug(self, "writting php7.2 configuration into " - "/etc/php/7.2/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() + Log.debug(self, "writting php7.2 configuration into " + "/etc/php/7.2/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() - # Parse /etc/php/7.2/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.2/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php72-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writing PHP 7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www.conf") - config.write(configfile) + # Parse /etc/php/7.2/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.2/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php72-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.2/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writing PHP 7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www.conf") + config.write(configfile) - with open("/etc/php/7.2/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") + with open("/etc/php/7.2/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") - # Generate /etc/php/7.2/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php72-two-fpm.sock' - with open('/etc/php/7.2/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/www-two.conf") - config.write(confifile) + # Generate /etc/php/7.2/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php72-two-fpm.sock' + with open('/etc/php/7.2/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/www-two.conf") + config.write(confifile) - # Generate /etc/php/7.2/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", - "/etc/php/7.2/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.2/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9172' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.2/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.2 configuration into " - "/etc/php/7.2/fpm/pool.d/debug.conf") - config.write(confifile) + # Generate /etc/php/7.2/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.2/fpm/pool.d/www.conf", + "/etc/php/7.2/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.2/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.2/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9172' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.2/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.2/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.2 configuration into " + "/etc/php/7.2/fpm/pool.d/debug.conf") + config.write(confifile) - with open("/etc/php/7.2/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") + with open("/etc/php/7.2/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.2/mods-available/" - "xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.2/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.2/mods-available/" + "xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.2/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php72' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php72' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.2-fpm') + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.2-fpm') # PHP7.3 configuration if set(WOVariables.wo_php73).issubset(set(apt_packages)): @@ -752,162 +752,162 @@ def post_pref(self, apt_packages, packages): Log.debug(self, 'Creating directory /var/log/php/7.3/') os.makedirs('/var/log/php/7.3/') - # Parse etc/php/7.3/fpm/php.ini - config = configparser.ConfigParser() - Log.debug(self, "configuring php file /etc/php/7.3/" - "fpm/php.ini") - config.read('/etc/php/7.3/fpm/php.ini') - config['PHP']['expose_php'] = 'Off' - config['PHP']['post_max_size'] = '100M' - config['PHP']['upload_max_filesize'] = '100M' - config['PHP']['max_execution_time'] = '300' - config['PHP']['max_input_time'] = '300' - config['PHP']['max_input_vars'] = '20000' - config['Date']['date.timezone'] = WOVariables.wo_timezone - config['opcache']['opcache.enable'] = '1' - config['opcache']['opcache.interned_strings_buffer'] = '8' - config['opcache']['opcache.max_accelerated_files'] = '10000' - config['opcache']['opcache.memory_consumption'] = '256' - config['opcache']['opcache.save_comments'] = '1' - config['opcache']['opcache.revalidate_freq'] = '5' - config['opcache']['opcache.consistency_checks'] = '0' - config['opcache']['opcache.validate_timestamps'] = '1' - with open('/etc/php/7.3/fpm/php.ini', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "Writting php configuration into " - "/etc/php/7.3/fpm/php.ini") - config.write(configfile) + # Parse etc/php/7.3/fpm/php.ini + config = configparser.ConfigParser() + Log.debug(self, "configuring php file /etc/php/7.3/" + "fpm/php.ini") + config.read('/etc/php/7.3/fpm/php.ini') + config['PHP']['expose_php'] = 'Off' + config['PHP']['post_max_size'] = '100M' + config['PHP']['upload_max_filesize'] = '100M' + config['PHP']['max_execution_time'] = '300' + config['PHP']['max_input_time'] = '300' + config['PHP']['max_input_vars'] = '20000' + config['Date']['date.timezone'] = WOVariables.wo_timezone + config['opcache']['opcache.enable'] = '1' + config['opcache']['opcache.interned_strings_buffer'] = '8' + config['opcache']['opcache.max_accelerated_files'] = '10000' + config['opcache']['opcache.memory_consumption'] = '256' + config['opcache']['opcache.save_comments'] = '1' + config['opcache']['opcache.revalidate_freq'] = '5' + config['opcache']['opcache.consistency_checks'] = '0' + config['opcache']['opcache.validate_timestamps'] = '1' + with open('/etc/php/7.3/fpm/php.ini', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "Writting php configuration into " + "/etc/php/7.3/fpm/php.ini") + config.write(configfile) - # Parse /etc/php/7.3/fpm/php-fpm.conf - data = dict(pid="/run/php/php7.3-fpm.pid", + # Parse /etc/php/7.3/fpm/php-fpm.conf + data = dict(pid="/run/php/php7.3-fpm.pid", error_log="/var/log/php7.3-fpm.log", include="/etc/php/7.3/fpm/pool.d/*.conf") - Log.debug(self, "writting php 7.3 configuration into " - "/etc/php/7.3/fpm/php-fpm.conf") - wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) - wo_php_fpm.close() + Log.debug(self, "writting php 7.3 configuration into " + "/etc/php/7.3/fpm/php-fpm.conf") + wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'php-fpm.mustache', out=wo_php_fpm) + wo_php_fpm.close() - # Parse /etc/php/7.3/fpm/pool.d/www.conf - config = configparser.ConfigParser() - config.read_file(codecs.open('/etc/php/7.3/fpm/' - 'pool.d/www.conf', - "r", "utf8")) - config['www']['ping.path'] = '/ping' - config['www']['pm.status_path'] = '/status' - config['www']['pm.max_requests'] = '1500' - config['www']['pm.max_children'] = '50' - config['www']['pm.start_servers'] = '10' - config['www']['pm.min_spare_servers'] = '5' - config['www']['pm.max_spare_servers'] = '15' - config['www']['request_terminate_timeout'] = '300' - config['www']['pm'] = 'ondemand' - config['www']['chdir'] = '/' - config['www']['prefix'] = '/var/run/php' - config['www']['listen'] = 'php73-fpm.sock' - config['www']['listen.mode'] = '0660' - config['www']['listen.backlog'] = '32768' - config['www']['catch_workers_output'] = 'yes' - with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', - encoding='utf-8', mode='w') as configfile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www.conf") - config.write(configfile) + # Parse /etc/php/7.3/fpm/pool.d/www.conf + config = configparser.ConfigParser() + config.read_file(codecs.open('/etc/php/7.3/fpm/' + 'pool.d/www.conf', + "r", "utf8")) + config['www']['ping.path'] = '/ping' + config['www']['pm.status_path'] = '/status' + config['www']['pm.max_requests'] = '1500' + config['www']['pm.max_children'] = '50' + config['www']['pm.start_servers'] = '10' + config['www']['pm.min_spare_servers'] = '5' + config['www']['pm.max_spare_servers'] = '15' + config['www']['request_terminate_timeout'] = '300' + config['www']['pm'] = 'ondemand' + config['www']['chdir'] = '/' + config['www']['prefix'] = '/var/run/php' + config['www']['listen'] = 'php73-fpm.sock' + config['www']['listen.mode'] = '0660' + config['www']['listen.backlog'] = '32768' + config['www']['catch_workers_output'] = 'yes' + with codecs.open('/etc/php/7.3/fpm/pool.d/www.conf', + encoding='utf-8', mode='w') as configfile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www.conf") + config.write(configfile) - with open("/etc/php/7.3/fpm/pool.d/www.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("\nphp_admin_value[open_basedir] " - "= \"/var/www/:/usr/share/php/:" - "/tmp/:/var/run/nginx-cache/:" - "/dev/shm:/dev/urandom\"\n") + with open("/etc/php/7.3/fpm/pool.d/www.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("\nphp_admin_value[open_basedir] " + "= \"/var/www/:/usr/share/php/:" + "/tmp/:/var/run/nginx-cache/:" + "/dev/shm:/dev/urandom\"\n") - # Generate /etc/php/7.3/fpm/pool.d/www-two.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/www-two.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "www-two.conf", "[www]", "[www-two]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') - config['www-two']['listen'] = 'php73-two-fpm.sock' - with open('/etc/php/7.3/fpm/pool.d/www-two.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/www-two.conf") - config.write(confifile) + # Generate /etc/php/7.3/fpm/pool.d/www-two.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/www-two.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "www-two.conf", "[www]", "[www-two]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/www-two.conf') + config['www-two']['listen'] = 'php73-two-fpm.sock' + with open('/etc/php/7.3/fpm/pool.d/www-two.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/www-two.conf") + config.write(confifile) - # Generate /etc/php/7.3/fpm/pool.d/debug.conf - WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", - "/etc/php/7.3/fpm/pool.d/debug.conf") - WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" - "debug.conf", "[www]", "[debug]") - config = configparser.ConfigParser() - config.read('/etc/php/7.3/fpm/pool.d/debug.conf') - config['debug']['listen'] = '127.0.0.1:9173' - config['debug']['rlimit_core'] = 'unlimited' - config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' - config['debug']['request_slowlog_timeout'] = '10s' - with open('/etc/php/7.3/fpm/pool.d/debug.conf', - encoding='utf-8', mode='w') as confifile: - Log.debug(self, "writting PHP 7.3 configuration into " - "/etc/php/7.3/fpm/pool.d/debug.conf") - config.write(confifile) + # Generate /etc/php/7.3/fpm/pool.d/debug.conf + WOFileUtils.copyfile(self, "/etc/php/7.3/fpm/pool.d/www.conf", + "/etc/php/7.3/fpm/pool.d/debug.conf") + WOFileUtils.searchreplace(self, "/etc/php/7.3/fpm/pool.d/" + "debug.conf", "[www]", "[debug]") + config = configparser.ConfigParser() + config.read('/etc/php/7.3/fpm/pool.d/debug.conf') + config['debug']['listen'] = '127.0.0.1:9173' + config['debug']['rlimit_core'] = 'unlimited' + config['debug']['slowlog'] = '/var/log/php/7.3/slow.log' + config['debug']['request_slowlog_timeout'] = '10s' + with open('/etc/php/7.3/fpm/pool.d/debug.conf', + encoding='utf-8', mode='w') as confifile: + Log.debug(self, "writting PHP 7.3 configuration into " + "/etc/php/7.3/fpm/pool.d/debug.conf") + config.write(confifile) - with open("/etc/php/7.3/fpm/pool.d/debug.conf", - encoding='utf-8', mode='a') as myfile: - myfile.write("php_admin_value[xdebug.profiler_output_dir] " - "= /tmp/ \nphp_admin_value[xdebug.profiler_" - "output_name] = cachegrind.out.%p-%H-%R " - "\nphp_admin_flag[xdebug.profiler_enable" - "_trigger] = on \nphp_admin_flag[xdebug." - "profiler_enable] = off\n") + with open("/etc/php/7.3/fpm/pool.d/debug.conf", + encoding='utf-8', mode='a') as myfile: + myfile.write("php_admin_value[xdebug.profiler_output_dir] " + "= /tmp/ \nphp_admin_value[xdebug.profiler_" + "output_name] = cachegrind.out.%p-%H-%R " + "\nphp_admin_flag[xdebug.profiler_enable" + "_trigger] = on \nphp_admin_flag[xdebug." + "profiler_enable] = off\n") - # Disable xdebug - if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" - " /etc/php/7.3/mods-available" - "/xdebug.ini"): - WOFileUtils.searchreplace(self, "/etc/php/7.3/" - "mods-available/" - "xdebug.ini", - "zend_extension", - ";zend_extension") + # Disable xdebug + if not WOShellExec.cmd_exec(self, "grep -q \';zend_extension\'" + " /etc/php/7.3/mods-available" + "/xdebug.ini"): + WOFileUtils.searchreplace(self, "/etc/php/7.3/" + "mods-available/" + "xdebug.ini", + "zend_extension", + ";zend_extension") - # PHP and Debug pull configuration - if not os.path.exists('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/fpm/status/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/fpm/status/' - .format(WOVariables.wo_webroot)) - open('{0}22222/htdocs/fpm/status/debug73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() - open('{0}22222/htdocs/fpm/status/php73' - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='a').close() + # PHP and Debug pull configuration + if not os.path.exists('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/fpm/status/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/fpm/status/' + .format(WOVariables.wo_webroot)) + open('{0}22222/htdocs/fpm/status/debug73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() + open('{0}22222/htdocs/fpm/status/php73' + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='a').close() - # Write info.php - if not os.path.exists('{0}22222/htdocs/php/' - .format(WOVariables.wo_webroot)): - Log.debug(self, 'Creating directory ' - '{0}22222/htdocs/php/ ' - .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/php' - .format(WOVariables.wo_webroot)) + # Write info.php + if not os.path.exists('{0}22222/htdocs/php/' + .format(WOVariables.wo_webroot)): + Log.debug(self, 'Creating directory ' + '{0}22222/htdocs/php/ ' + .format(WOVariables.wo_webroot)) + os.makedirs('{0}22222/htdocs/php' + .format(WOVariables.wo_webroot)) - with open("{0}22222/htdocs/php/info.php" - .format(WOVariables.wo_webroot), - encoding='utf-8', mode='w') as myfile: - myfile.write("") + with open("{0}22222/htdocs/php/info.php" + .format(WOVariables.wo_webroot), + encoding='utf-8', mode='w') as myfile: + myfile.write("") - WOFileUtils.chown(self, "{0}22222" - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + WOFileUtils.chown(self, "{0}22222" + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, recursive=True) - WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") - WOService.restart_service(self, 'php7.3-fpm') + WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") + WOService.restart_service(self, 'php7.3-fpm') # create mysql config if it doesn't exist if set(WOVariables.wo_mysql).issubset(set(apt_packages)): From 5b9af72e7c3510bfc64ba063081f0cca3cf5540c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:32:07 +0200 Subject: [PATCH 054/102] Fix php72 upgrade --- wo/cli/plugins/stack_upgrade.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 9edad10..0d2c3ea 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -190,6 +190,10 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.update(self) # Update packages WOAptGet.install(self, apt_packages) + if set(WOVariables.wo_php).issubset(set(apt_packages)): + WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") + WOFileUtils.rm(self, "/etc/php/7.2/fpm/" + "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): From 3a47162994ffc20ccc144c844e94a94d794db73e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:38:50 +0200 Subject: [PATCH 055/102] Remove outdated part of wo upgrade --- wo/cli/plugins/stack_upgrade.py | 55 +++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 0d2c3ea..bafeeff 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -31,7 +31,9 @@ class WOStackUpgradeController(CementBaseController): (['--nginx'], dict(help='Upgrade Nginx stack', action='store_true')), (['--php'], - dict(help='Upgrade PHP stack', action='store_true')), + dict(help='Upgrade PHP 7.2 stack', action='store_true')), + (['--php73'], + dict(help='Upgrade PHP 7.3 stack', action='store_true')), (['--mysql'], dict(help='Upgrade MySQL stack', action='store_true')), (['--wpcli'], @@ -49,37 +51,6 @@ class WOStackUpgradeController(CementBaseController): action='store_true')), ] - @expose(hide=True) - def upgrade_php56(self): - if WOVariables.wo_distro == "ubuntu": - if os.path.isfile("/etc/apt/sources.list.d/ondrej-php5-5_6-{0}." - "list".format(WOVariables.wo_platform_codename)): - Log.error(self, "Unable to find PHP 5.5") - else: - if not(os.path.isfile(WOVariables.wo_repo_file_path) and - WOFileUtils.grep(self, WOVariables.wo_repo_file_path, - "php55")): - Log.error(self, "Unable to find PHP 5.5") - - Log.info(self, "During PHP update process non nginx-cached" - " parts of your site may remain down.") - - # Check prompt - if (not self.app.pargs.no_prompt): - start_upgrade = input("Do you want to continue:[y/N]") - if start_upgrade != "Y" and start_upgrade != "y": - Log.error(self, "Not starting PHP package update") - - if WOVariables.wo_distro == "ubuntu": - WORepo.remove(self, ppa="ppa:ondrej/php5") - WORepo.add(self, ppa=WOVariables.wo_php_repo) - - Log.info(self, "Updating apt-cache, please wait...") - WOAptGet.update(self) - Log.info(self, "Installing packages, please wait ...") - WOAptGet.install(self, WOVariables.wo_php + - WOVariables.wo_php_extra) - @expose(hide=True) def default(self): # All package update @@ -88,7 +59,8 @@ class WOStackUpgradeController(CementBaseController): empty_packages = [] if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and - (not self.app.pargs.php) and (not self.app.pargs.mysql) and + (not self.app.pargs.php) and (not self.app.pargs.php73) and + (not self.app.pargs.mysql) and (not self.app.pargs.all) and (not self.app.pargs.wpcli) and (not self.app.pargs.netdata) and (not self.app.pargs.composer) and (not self.app.pargs.phpmyadmin) and @@ -106,7 +78,6 @@ class WOStackUpgradeController(CementBaseController): self.app.pargs.php = True self.app.pargs.mysql = True self.app.pargs.wpcli = True - self.app.pargs.netdata = True if self.app.pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): @@ -124,6 +95,16 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") + if self.app.pargs.php73: + if WOAptGet.is_installed(self, 'php7.3-fpm'): + if not WOAptGet.is_installed(self, 'php7.2-fpm'): + apt_packages = apt_packages + WOVariables.wo_php73 + \ + WOVariables.wo_php_extra + else: + apt_packages = apt_packages + WOVariables.wo_php73 + else: + Log.info(self, "PHP 7.3 is not installed") + if self.app.pargs.mysql: if WOAptGet.is_installed(self, 'mariadb-server'): apt_packages = apt_packages + WOVariables.wo_mysql @@ -194,12 +175,18 @@ class WOStackUpgradeController(CementBaseController): WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") WOFileUtils.rm(self, "/etc/php/7.2/fpm/" "pool.d/www-two.conf") + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOFileUtils.rm(self, "/etc/php/7.3/fpm/pool.d/www.conf") + WOFileUtils.rm(self, "/etc/php/7.3/fpm/" + "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): WOService.restart_service(self, 'nginx') if set(WOVariables.wo_php).issubset(set(apt_packages)): WOService.restart_service(self, 'php7.2-fpm') + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOService.restart_service(self, 'php7.3-fpm') if set(WOVariables.wo_mysql).issubset(set(apt_packages)): WOService.restart_service(self, 'mysql') if set(WOVariables.wo_redis).issubset(set(apt_packages)): From f2abaeb5da26b726972815d7af55e57e481255f8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 12:50:12 +0200 Subject: [PATCH 056/102] Another fix for php7.x-fpm upgrade --- wo/cli/plugins/stack_upgrade.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index bafeeff..dbcb074 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -169,16 +169,12 @@ class WOStackUpgradeController(CementBaseController): if len(apt_packages): # apt-get update WOAptGet.update(self) + if set(WOVariables.wo_php).issubset(set(apt_packages)): + WOAptGet.purge(self, ['php7.2-fpm']) + if set(WOVariables.wo_php73).issubset(set(apt_packages)): + WOAptGet.purge(self, ['php7.3-fpm']) # Update packages WOAptGet.install(self, apt_packages) - if set(WOVariables.wo_php).issubset(set(apt_packages)): - WOFileUtils.rm(self, "/etc/php/7.2/fpm/pool.d/www.conf") - WOFileUtils.rm(self, "/etc/php/7.2/fpm/" - "pool.d/www-two.conf") - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - WOFileUtils.rm(self, "/etc/php/7.3/fpm/pool.d/www.conf") - WOFileUtils.rm(self, "/etc/php/7.3/fpm/" - "pool.d/www-two.conf") post_pref(self, apt_packages, empty_packages) # Post Actions after package updates if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): From 436130ee8d09d520c2d2f19b19f5f3010ba86304 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 13:05:56 +0200 Subject: [PATCH 057/102] Fix passwd variable --- setup.py | 2 +- wo/cli/plugins/stack_migrate.py | 4 ---- wo/cli/plugins/stack_pref.py | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index b4026f9..4c93342 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup(name='wo', classifiers=[], keywords='', author='WordOps', - author_email='core@wordops.net', + author_email='contact@wordops.io', url='https://wordops.net', license='MIT', packages=find_packages(exclude=['ez_setup', 'examples', 'tests', diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index a81d2b4..c1a84c6 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -79,10 +79,6 @@ class WOStackMigrateController(CementBaseController): # Install MariaDB apt_packages = WOVariables.wo_mysql - # If PHP is installed then install php7.2-mysql - if WOAptGet.is_installed(self, "php7.2-fpm"): - apt_packages = apt_packages + ["php7.2-mysql"] - Log.info(self, "Updating apt-cache, hang on...") WOAptGet.update(self) Log.info(self, "Installing MariaDB, hang on...") diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 23d652a..c4f3863 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -412,7 +412,7 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() - randompass = ''.join([random.choice + passwd = ''.join([random.choice (string.ascii_letters + string.digits) for n in range(24)]) try: @@ -421,7 +421,7 @@ def post_pref(self, apt_packages, packages): "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo " "2>/dev/null" - .format(password=randompass)) + .format(password=passwd)) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Failed to save HTTP Auth") From 7dfef0566c844d76a15a0bdad29e8d507700cc68 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 13:11:24 +0200 Subject: [PATCH 058/102] Fix package purge --- wo/cli/plugins/stack_upgrade.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index dbcb074..a834857 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -170,9 +170,11 @@ class WOStackUpgradeController(CementBaseController): # apt-get update WOAptGet.update(self) if set(WOVariables.wo_php).issubset(set(apt_packages)): - WOAptGet.purge(self, ['php7.2-fpm']) + WOAptGet.remove(self, ['php7.2-fpm'], + auto=False, purge=True) if set(WOVariables.wo_php73).issubset(set(apt_packages)): - WOAptGet.purge(self, ['php7.3-fpm']) + WOAptGet.remove(self, ['php7.3-fpm'], + auto=False, purge=True) # Update packages WOAptGet.install(self, apt_packages) post_pref(self, apt_packages, empty_packages) From 2877b535a65511f20f13da09c1abc526dd426288 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 14:22:20 +0200 Subject: [PATCH 059/102] add more mysql settings --- wo/cli/plugins/stack.py | 68 ---------------------- wo/cli/plugins/stack_pref.py | 107 ++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 77 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index bf245de..5d753b7 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -469,74 +469,6 @@ class WOStackController(CementBaseController): 'redis', recursive=False) WOService.restart_service(self, 'redis-server') - if 'mariadb-server' in apt_packages: - # setting innodb memory usage - wo_ram = psutil.virtual_memory().total / (1024 * 1024) - wo_ram_innodb = int(wo_ram*0.3) - wo_ram_log_buffer = int(wo_ram_innodb*0.25) - wo_ram_log_size = int(wo_ram_log_buffer*0.5) - # replacing default values - if os.path.isfile("/etc/mysql/my.cnf"): - Log.debug(self, "Tuning MySQL configuration") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_buffer_pool_size = 256M", - "innodb_buffer_pool_size = {0}M" - .format(wo_ram_innodb)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_log_buffer_size = 8M", - "innodb_log_buffer_size = {0}M" - .format(wo_ram_log_buffer)) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "#innodb_log_file_size = 50M", - "innodb_log_file_size = {0}M" - .format(wo_ram_log_size)) - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "wait_timeout " - "= 600", - "wait_timeout " - "= 120") - # disabling mariadb binlog - WOFileUtils.searchreplace(self, - "/etc/mysql/my.cnf", - "log_bin " - "= /var/log/mysql/" - "mariadb-bin", - "#log_bin " - " = /var/log/" - "mysql/mariadb-bin") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - 'log_bin_index ' - "= /var/log/mysql/" - "mariadb-bin.index", - "#log_bin_index " - "= /var/log/mysql/" - "mariadb-bin.index") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "expire_logs_days = 10", - "#expire_logs_days " - "= 10") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "max_binlog_size " - "= 100M", - "#max_binlog_size " - "= 100M") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_open_files =" - " 400", - "innodb_open_files =" - " 16000") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "innodb_io_capacity =" - " 400", - "innodb_io_capacity =" - " 16000") - WOService.stop_service(self, 'mysql') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', - '/var/lib/mysql/ib_logfile0.bak') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', - '/var/lib/mysql/ib_logfile1.bak') - WOService.start_service(self, 'mysql') if disp_msg: if (self.msg): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index c4f3863..5f47d2f 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -5,6 +5,7 @@ import random import shutil import string import requests +import psutil from wo.cli.plugins.site_functions import * from wo.cli.plugins.stack_services import WOStackStatusController @@ -413,8 +414,8 @@ def post_pref(self, apt_packages, packages): wo_nginx.close() passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(24)]) + (string.ascii_letters + string.digits) + for n in range(24)]) try: WOShellExec.cmd_exec(self, "printf \"WordOps:" "$(openssl passwd -crypt " @@ -919,14 +920,102 @@ def post_pref(self, apt_packages, packages): encoding='utf-8', mode='w') config_file.write(config) config_file.close() + else: + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + wo_ram_innodb = int(wo_ram*0.3) + wo_ram_log_buffer = int(wo_ram_innodb*0.25) + wo_ram_log_size = int(wo_ram_log_buffer*0.5) + # replacing default values + Log.debug(self, "Tuning MySQL configuration") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_buffer_pool_size = 256M", + "innodb_buffer_pool_size = {0}M" + .format(wo_ram_innodb)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_log_buffer_size = 8M", + "innodb_log_buffer_size = {0}M" + .format(wo_ram_log_buffer)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#innodb_log_file_size = 50M", + "innodb_log_file_size = {0}M" + .format(wo_ram_log_size)) + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "wait_timeout " + "= 600", + "wait_timeout " + "= 120") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "skip-external-locking", + "skip-external-locking\n" + "skip-name-resolve = 1\n") + # disabling mariadb binlog + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "log_bin " + "= /var/log/mysql/" + "mariadb-bin", + "#log_bin " + " = /var/log/" + "mysql/mariadb-bin") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + 'log_bin_index ' + "= /var/log/mysql/" + "mariadb-bin.index", + "#log_bin_index " + "= /var/log/mysql/" + "mariadb-bin.index") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "expire_logs_days = 10", + "#expire_logs_days " + "= 10") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "max_binlog_size " + "= 100M", + "#max_binlog_size " + "= 100M") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_open_files =" + " 400", + "innodb_open_files =" + " 16000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_io_capacity =" + " 400", + "innodb_io_capacity =" + " 16000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "query_cache_size = 64M", + "query_cache_size = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#query_cache_type = DEMAND", + "query_cache_type = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "max_allowed_packet = 16M", + "max_allowed_packet = 64M") + if (wo_ram_innodb > 1000) and (wo_ram_innodb < 64000): + wo_innodb_instance = int(wo_ram_innodb/1000) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "# * Security Features", + "innodb_buffer_pool_instances " + "= {0}\n" + .format(wo_innodb_instance) + + "# * Security Features") - WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) - WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' - '> /dev/null 2>&1', - comment='MySQL optimization cronjob ' - 'added by WordOps') - WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - WOService.reload_service(self, 'mysql') + WOService.stop_service(self, 'mysql') + WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', + '/var/lib/mysql/ib_logfile0.bak') + WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', + '/var/lib/mysql/ib_logfile1.bak') + WOService.start_service(self, 'mysql') + + WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' + '> /dev/null 2>&1', + comment='MySQL optimization cronjob ' + 'added by WordOps') + WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") + WOService.restart_service(self, 'mysql') # create fail2ban configuration files if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): From fc52635c952a55da5dc6b8e5bc5121f7726cf992 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 14:40:48 +0200 Subject: [PATCH 060/102] more MariaDB optimizations --- wo/cli/plugins/stack_pref.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 5f47d2f..2098a62 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -949,6 +949,7 @@ def post_pref(self, apt_packages, packages): "skip-external-locking", "skip-external-locking\n" "skip-name-resolve = 1\n") + # disabling mariadb binlog WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", @@ -990,6 +991,12 @@ def post_pref(self, apt_packages, packages): WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "#query_cache_type = DEMAND", "query_cache_type = 0") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#open-files-limit = 2000", + "open-files-limit = 10000") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "table_open_cache = 400", + "table_open_cache = 16000") WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "max_allowed_packet = 16M", "max_allowed_packet = 64M") From 304a31b8e8df61daf4530f92bc03def7fb6c5724 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 19:17:59 +0200 Subject: [PATCH 061/102] Implement `--wprocket` * cleanup templates by using variables * add configuration for wp-rocket * update mapping for wp-rocket --- wo/cli/plugins/site.py | 51 ++++++++++++++-- wo/cli/plugins/site_functions.py | 24 +++++--- wo/cli/plugins/stack_pref.py | 41 ++++++++----- wo/cli/templates/22222.mustache | 2 +- wo/cli/templates/locations-php7.mustache | 54 ----------------- wo/cli/templates/locations.mustache | 2 +- wo/cli/templates/map-wp.mustache | 7 ++- wo/cli/templates/php.mustache | 4 +- wo/cli/templates/php7.mustache | 10 ---- wo/cli/templates/redis-php7.mustache | 42 ------------- wo/cli/templates/redis.mustache | 4 +- wo/cli/templates/upstream.mustache | 2 +- wo/cli/templates/virtualconf-php7.mustache | 2 +- wo/cli/templates/virtualconf.mustache | 2 +- wo/cli/templates/webp.mustache | 2 +- wo/cli/templates/wpcommon-php7.mustache | 70 ---------------------- wo/cli/templates/wpcommon.mustache | 4 +- wo/cli/templates/wpfc-php7.mustache | 21 ------- wo/cli/templates/wpfc.mustache | 4 +- wo/cli/templates/wprocket.mustache | 21 +++++++ wo/cli/templates/wpsc-php7.mustache | 17 ------ wo/cli/templates/wpsc.mustache | 4 +- 22 files changed, 131 insertions(+), 259 deletions(-) delete mode 100644 wo/cli/templates/locations-php7.mustache delete mode 100644 wo/cli/templates/php7.mustache delete mode 100644 wo/cli/templates/redis-php7.mustache delete mode 100644 wo/cli/templates/wpcommon-php7.mustache delete mode 100644 wo/cli/templates/wpfc-php7.mustache create mode 100644 wo/cli/templates/wprocket.mustache delete mode 100644 wo/cli/templates/wpsc-php7.mustache diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index d5e5e59..20948e4 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -333,6 +333,9 @@ class WOSiteCreateController(CementBaseController): (['--wpsc'], dict(help="create WordPress single/multi site with wpsc cache", action='store_true')), + (['--wprocket'], + dict(help="create WordPress single/multi site with WP-Rocket", + action='store_true')), (['--wpredis'], dict(help="create WordPress single/multi site " "with redis cache", @@ -423,7 +426,8 @@ class WOSiteCreateController(CementBaseController): if stype == 'proxy': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['proxy'] = True data['host'] = host @@ -433,14 +437,16 @@ class WOSiteCreateController(CementBaseController): if self.app.pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=False, php73=True, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['basic'] = True if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, multisite=False, + wpfc=False, wpsc=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot) if stype == 'php': @@ -451,7 +457,8 @@ class WOSiteCreateController(CementBaseController): data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', wo_db_host='') @@ -479,6 +486,7 @@ class WOSiteCreateController(CementBaseController): if ((not self.app.pargs.wpfc) and (not self.app.pargs.wpsc) and + (not self.app.pargs.wprocket) and (not self.app.pargs.wpredis)): data['basic'] = True @@ -488,6 +496,7 @@ class WOSiteCreateController(CementBaseController): data['basic'] = False self.app.pargs.wpredis = True + # Check rerequired packages are installed or not wo_auth = site_package_check(self, stype) @@ -825,6 +834,8 @@ class WOSiteUpdateController(CementBaseController): dict(help="update to wpfc cache", action='store_true')), (['--wpsc'], dict(help="update to wpsc cache", action='store_true')), + (['--wprocket'], + dict(help="update to WP-Rocket cache", action='store_true')), (['--wpredis'], dict(help="update to redis cache", action='store_true')), (['-le', '--letsencrypt'], @@ -868,6 +879,7 @@ class WOSiteUpdateController(CementBaseController): if not (pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpsubdir or pargs.wpsubdomain or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpredis or pargs.letsencrypt or pargs.hsts or pargs.dns or pargs.force): Log.error(self, "Please provide options to update sites.") @@ -946,6 +958,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.password and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpsubdir or pargs.wpsubdomain or pargs.hsts)): try: @@ -958,6 +971,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.hsts and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpsubdir or pargs.wpsubdomain or pargs.password)): try: @@ -999,7 +1013,7 @@ class WOSiteUpdateController(CementBaseController): if stype == 'php': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, currsitetype=oldsitetype, currcachetype=oldcachetype) @@ -1007,7 +1021,7 @@ class WOSiteUpdateController(CementBaseController): data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, multisite=False, + wpsc=False, wpredis=False, wprocket=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', wo_db_host='', @@ -1061,21 +1075,31 @@ class WOSiteUpdateController(CementBaseController): data['wpfc'] = False data['wpsc'] = False data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpfc': data['basic'] = False data['wpfc'] = True data['wpsc'] = False data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpsc': data['basic'] = False data['wpfc'] = False data['wpsc'] = True data['wpredis'] = False + data['wprocket'] = False elif oldcachetype == 'wpredis': data['basic'] = False data['wpfc'] = False data['wpsc'] = False data['wpredis'] = True + data['wprocket'] = False + elif oldcachetype == 'wprocket': + data['basic'] = False + data['wpfc'] = False + data['wpsc'] = False + data['wpredis'] = False + data['wprocket'] = True if pargs.php73 == 'on': data['php73'] = True @@ -1233,6 +1257,11 @@ class WOSiteUpdateController(CementBaseController): data['basic'] = False cache = 'wpredis' + if pargs.wprocket and data['currcachetype'] != 'wprocket': + data['wprocket'] = True + data['basic'] = False + cache = 'wprocket' + if (php73 is old_php73) and (stype == oldsitetype and cache == oldcachetype): return 1 @@ -1620,6 +1649,16 @@ class WOSiteUpdateController(CementBaseController): "`tail /var/log/wo/wordops.log` and please try again") return 1 + if oldcachetype == 'wprocket' and not data['wprocket']: + try: + uninstallwp_plugin(self, 'wp-rocket', data) + except SiteError as e: + Log.debug(self, str(e)) + Log.info(self, Log.FAIL + "Update site failed." + "Check the log for details: " + "`tail /var/log/wo/wordops.log` and please try again") + return 1 + # Service Nginx Reload if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 032ae6f..dd66040 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -897,12 +897,12 @@ def site_package_check(self, stype): if (os.path.isdir("/etc/nginx/common") and not os.path.isfile("/etc/nginx/common/locations-wo.conf")): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/locations-wo.conf') wo_nginx = open('/etc/nginx/common/locations-wo.conf', encoding='utf-8', mode='w') - self.app.render((data), 'locations-php7.mustache', + self.app.render((data), 'locations.mustache', out=wo_nginx) wo_nginx.close() @@ -910,7 +910,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/php73.conf') wo_nginx = open('/etc/nginx/common/php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', + self.app.render((data), 'php.mustache', out=wo_nginx) wo_nginx.close() @@ -918,7 +918,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpcommon-php73.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', + self.app.render((data), 'wpcommon.mustache', out=wo_nginx) wo_nginx.close() @@ -926,7 +926,7 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpfc-php73.conf') wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', + self.app.render((data), 'wpfc.mustache', out=wo_nginx) wo_nginx.close() @@ -934,18 +934,26 @@ def site_package_check(self, stype): 'file /etc/nginx/common/wpsc-php73.conf') wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', + self.app.render((data), 'wpsc.mustache', + out=wo_nginx) + wo_nginx.close() + + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wprocket-php73.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wprocket.mustache', out=wo_nginx) wo_nginx.close() if (os.path.isfile("/etc/nginx/nginx.conf") and not os.path.isfile("/etc/nginx/common/redis-php73.conf")): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') wo_nginx = open('/etc/nginx/common/redis-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', + self.app.render((data), 'redis.mustache', out=wo_nginx) wo_nginx.close() diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2098a62..df9bf9c 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -230,7 +230,7 @@ def post_pref(self, apt_packages, packages): os.makedirs('/etc/nginx/common') if os.path.exists('/etc/nginx/common'): - data = dict(webroot=WOVariables.wo_webroot) + data = dict() # Common Configuration Log.debug(self, 'Writting the nginx configuration to ' @@ -248,7 +248,7 @@ def post_pref(self, apt_packages, packages): self.app.render((data), 'wpsubdir.mustache', out=wo_nginx) wo_nginx.close() - + data = dict(upstream="php72") # PHP 7.2 conf Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/php72.conf') @@ -258,6 +258,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/wpcommon-php72.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php72.conf', @@ -282,15 +290,23 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wprocket-php72.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wprocket.mustache', + out=wo_nginx) + wo_nginx.close() + # PHP 7.3 conf if os.path.isdir("/etc/nginx/common"): - data = dict() + data = dict(upstream="php73") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/php73.conf') wo_nginx = open('/etc/nginx/common/php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'php7.mustache', + self.app.render((data), 'php.mustache', out=wo_nginx) wo_nginx.close() @@ -298,7 +314,7 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpcommon-php73.conf') wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon-php7.mustache', + self.app.render((data), 'wpcommon.mustache', out=wo_nginx) wo_nginx.close() @@ -306,7 +322,7 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpfc-php73.conf') wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpfc-php7.mustache', + self.app.render((data), 'wpfc.mustache', out=wo_nginx) wo_nginx.close() @@ -314,26 +330,23 @@ def post_pref(self, apt_packages, packages): 'file /etc/nginx/common/wpsc-php73.conf') wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'wpsc-php7.mustache', + self.app.render((data), 'wpsc.mustache', out=wo_nginx) wo_nginx.close() - # create redis conf - data = dict() Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', + 'file /etc/nginx/common/wprocket-php73.conf') + wo_nginx = open('/etc/nginx/common/wprocket-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', + self.app.render((data), 'wprocket.mustache', out=wo_nginx) wo_nginx.close() - data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') wo_nginx = open('/etc/nginx/common/redis-php73.conf', encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', + self.app.render((data), 'redis.mustache', out=wo_nginx) wo_nginx.close() diff --git a/wo/cli/templates/22222.mustache b/wo/cli/templates/22222.mustache index 3106ee7..e34fa36 100644 --- a/wo/cli/templates/22222.mustache +++ b/wo/cli/templates/22222.mustache @@ -1,4 +1,4 @@ -# WordOps admin NGINX CONFIGURATION - WO v3.9.5 +# WordOps admin NGINX CONFIGURATION - WO v3.9.7 server { diff --git a/wo/cli/templates/locations-php7.mustache b/wo/cli/templates/locations-php7.mustache deleted file mode 100644 index 267891c..0000000 --- a/wo/cli/templates/locations-php7.mustache +++ /dev/null @@ -1,54 +0,0 @@ -# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# Basic locations files -location = /favicon.ico { - access_log off; - log_not_found off; - expires max; -} -location = /robots.txt { -# Some WordPress plugin gererate robots.txt file -# Refer #340 issue - try_files $uri $uri/ /index.php$is_args$args @robots; - access_log off; - log_not_found off; -} -# fallback for robots.txt with default wordpress rules -location @robots { - return 200 "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n"; -} -# Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp)$ { - add_header "Access-Control-Allow-Origin" "*"; - access_log off; - log_not_found off; - expires max; -} -# Cache css & js files -location ~* \.(?:css(\.map)?|js(\.map)?)$ { - add_header "Access-Control-Allow-Origin" "*"; - access_log off; - log_not_found off; - expires 30d; -} -# Security settings for better privacy -# Deny hidden files -location ~ /\.(?!well-known\/) { - deny all; -} -# letsencrypt validation -location /.well-known/acme-challenge/ { - alias /var/www/html/.well-known/acme-challenge/; - allow all; -} -# Return 403 forbidden for readme.(txt|html) or license.(txt|html) or example.(txt|html) or other common git repository files -location ~* "/(^$|readme|license|example|README|LEGALNOTICE|INSTALLATION|CHANGELOG)\.(txt|html|md)" { - deny all; -} -# Deny backup extensions & log files and return 403 forbidden -location ~* "\.(old|orig|original|php#|php~|php_bak|save|swo|aspx?|tpl|sh|bash|bak?|cfg|cgi|dll|exe|git|hg|ini|jsp|log|mdb|out|sql|svn|swp|tar|rdf)$" { - deny all; -} -location ~* "/(=|\$&|_mm|(wp-)?config\.|cgi-|etc/passwd|muieblack)" { - deny all; -} diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index ba1d66f..26e8230 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -1,4 +1,4 @@ -# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.5 +# NGINX CONFIGURATION FOR COMMON LOCATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # Basic locations files location = /favicon.ico { diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 5cdb3d9..f764020 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -1,4 +1,4 @@ -# NGINX CONFIGURATION FOR FASTCGI_CACHE EXCEPTION - WO v3.9.5 +# NGINX CONFIGURATION FOR FASTCGI_CACHE EXCEPTION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # do not cache xhtml request @@ -52,6 +52,11 @@ map $is_args $query_no_cache { "" 0; } +map $https $https_prefix { + default ""; + on "-https" +} + # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; diff --git a/wo/cli/templates/php.mustache b/wo/cli/templates/php.mustache index 59903a8..27c861d 100644 --- a/wo/cli/templates/php.mustache +++ b/wo/cli/templates/php.mustache @@ -1,4 +1,4 @@ -# PHP NGINX CONFIGURATION - WO v3.9.5 +# PHP NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE location / { try_files $uri $uri/ /index.php$is_args$args; @@ -6,5 +6,5 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } diff --git a/wo/cli/templates/php7.mustache b/wo/cli/templates/php7.mustache deleted file mode 100644 index 4005572..0000000 --- a/wo/cli/templates/php7.mustache +++ /dev/null @@ -1,10 +0,0 @@ -# PHP NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -location / { - try_files $uri $uri/ /index.php$is_args$args; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; -} diff --git a/wo/cli/templates/redis-php7.mustache b/wo/cli/templates/redis-php7.mustache deleted file mode 100644 index 5e21f7a..0000000 --- a/wo/cli/templates/redis-php7.mustache +++ /dev/null @@ -1,42 +0,0 @@ -# Redis NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf - -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - try_files $uri $uri/ /index.php$is_args$args; -} - -location /redis-fetch { - internal ; - set $redis_key $args; - redis_pass redis; -} -location /redis-store { - internal ; - set_unescape_uri $key $arg_key ; - redis2_query set $key $echo_request_body; - redis2_query expire $key 14400; - redis2_pass redis; -} - -location ~ \.php$ { - set $key "nginx-cache:$scheme$request_method$host$request_uri"; - try_files $uri =404; - - srcache_fetch_skip $skip_cache; - srcache_store_skip $skip_cache; - - srcache_response_cache_control off; - - set_escape_uri $escaped_key $key; - - srcache_fetch GET /redis-fetch $key; - srcache_store PUT /redis-store key=$escaped_key; - - more_set_headers 'X-SRCache-Fetch-Status $srcache_fetch_status'; - more_set_headers 'X-SRCache-Store-Status $srcache_store_status'; - - include fastcgi_params; - fastcgi_pass php73; -} diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index cdde0b1..dcbb0c1 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -1,4 +1,4 @@ -# Redis NGINX CONFIGURATION - WO v3.9.5 +# Redis NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf @@ -39,5 +39,5 @@ location ~ \.php$ { more_set_headers 'X-SRCache-Store-Status $srcache_store_status'; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } diff --git a/wo/cli/templates/upstream.mustache b/wo/cli/templates/upstream.mustache index 686a6a5..232dade 100644 --- a/wo/cli/templates/upstream.mustache +++ b/wo/cli/templates/upstream.mustache @@ -1,4 +1,4 @@ -# NGINX UPSTREAM CONFIGURATION - WO v3.9.5 +# NGINX UPSTREAM CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE #------------------------------- # PHP 5.6 diff --git a/wo/cli/templates/virtualconf-php7.mustache b/wo/cli/templates/virtualconf-php7.mustache index 625fd2a..56c1e96 100644 --- a/wo/cli/templates/virtualconf-php7.mustache +++ b/wo/cli/templates/virtualconf-php7.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}} + {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php73.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/virtualconf.mustache b/wo/cli/templates/virtualconf.mustache index d8610de..87d1100 100644 --- a/wo/cli/templates/virtualconf.mustache +++ b/wo/cli/templates/virtualconf.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}} + {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php72.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/webp.mustache b/wo/cli/templates/webp.mustache index 3ed187b..d53dc46 100644 --- a/wo/cli/templates/webp.mustache +++ b/wo/cli/templates/webp.mustache @@ -1,4 +1,4 @@ -# WEBP NGINX CONFIGURATION - WO v3.9.5 +# WEBP NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE map $http_accept $webp_suffix { diff --git a/wo/cli/templates/wpcommon-php7.mustache b/wo/cli/templates/wpcommon-php7.mustache deleted file mode 100644 index 85d2739..0000000 --- a/wo/cli/templates/wpcommon-php7.mustache +++ /dev/null @@ -1,70 +0,0 @@ -# WordPress COMMON SETTINGS - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# Limit access to avoid brute force attack -location = /wp-login.php { - limit_req zone=one burst=1 nodelay; - include fastcgi_params; - fastcgi_pass php73; -} -# Disable wp-config.txt -location = /wp-config.txt { - deny all; - access_log off; - log_not_found off; -} -location = /robots.txt { -# Some WordPress plugin gererate robots.txt file -# Refer #340 issue - try_files $uri $uri/ /index.php?$args @robots; - access_log off; - log_not_found off; -} -# fallback for robots.txt with default wordpress rules -location @robots { - return 200 "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n"; -} -# webp rewrite rules for jpg and png images -# try to load alternative image.png.webp before image.png -location /wp-content/uploads { - location ~ \.(png|jpe?g)$ { - add_header Vary "Accept-Encoding"; - add_header "Access-Control-Allow-Origin" "*"; - add_header Cache-Control "public, no-transform"; - access_log off; - log_not_found off; - expires max; - try_files $uri$webp_suffix $uri =404; - } - location ~ \.php$ { -#Prevent Direct Access Of PHP Files From Web Browsers - deny all; - } -} -# webp rewrite rules for EWWW testing image -location /wp-content/plugins/ewww-image-optimizer/images { - location ~ \.(png|jpe?g)$ { - add_header Vary "Accept-Encoding"; - add_header "Access-Control-Allow-Origin" "*"; - add_header Cache-Control "public, no-transform"; - access_log off; - log_not_found off; - expires max; - try_files $uri$webp_suffix $uri =404; - } - location ~ \.php$ { -#Prevent Direct Access Of PHP Files From Web Browsers - deny all; - } -} -# Deny access to any files with a .php extension in the uploads directory -# Works in sub-directory installs and also in multisite network -# Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban) -location ~* /(?:uploads|files)/.*\.php$ { - deny all; -} -# mitigate DoS attack CVE with WordPress script concatenation -# add the following line to wp-config.php -# define( 'CONCATENATE_SCRIPTS', false ); -location ~ \/wp-admin\/load-(scripts|styles).php { - deny all; -} diff --git a/wo/cli/templates/wpcommon.mustache b/wo/cli/templates/wpcommon.mustache index 7e6edb3..f646cd6 100644 --- a/wo/cli/templates/wpcommon.mustache +++ b/wo/cli/templates/wpcommon.mustache @@ -1,10 +1,10 @@ -# WordPress COMMON SETTINGS - WO v3.9.5 +# WordPress COMMON SETTINGS - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # Limit access to avoid brute force attack location = /wp-login.php { limit_req zone=one burst=1 nodelay; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; } # Disable wp-config.txt location = /wp-config.txt { diff --git a/wo/cli/templates/wpfc-php7.mustache b/wo/cli/templates/wpfc-php7.mustache deleted file mode 100644 index 3667c40..0000000 --- a/wo/cli/templates/wpfc-php7.mustache +++ /dev/null @@ -1,21 +0,0 @@ -# WPFC NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf - -add_header X-fastcgi-cache $upstream_cache_status; -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - try_files $uri $uri/ /index.php$is_args$args; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; - fastcgi_cache_bypass $skip_cache; - fastcgi_no_cache $skip_cache; - fastcgi_cache WORDPRESS; -} -location ~ /purge(/.*) { - fastcgi_cache_purge WORDPRESS "$scheme$request_method$host$1"; - access_log off; -} diff --git a/wo/cli/templates/wpfc.mustache b/wo/cli/templates/wpfc.mustache index 447870f..5dbae7c 100644 --- a/wo/cli/templates/wpfc.mustache +++ b/wo/cli/templates/wpfc.mustache @@ -1,4 +1,4 @@ -# WPFC NGINX CONFIGURATION - WO v3.9.5 +# WPFC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $skip_cache variable set in /etc/nginx/conf.d/map-wp.conf @@ -10,7 +10,7 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; fastcgi_cache_bypass $skip_cache; fastcgi_no_cache $skip_cache; fastcgi_cache WORDPRESS; diff --git a/wo/cli/templates/wprocket.mustache b/wo/cli/templates/wprocket.mustache new file mode 100644 index 0000000..e73f859 --- /dev/null +++ b/wo/cli/templates/wprocket.mustache @@ -0,0 +1,21 @@ +# WPSC NGINX CONFIGURATION - WO v3.9.7 +# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE +# $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf + +# Use cached or actual file if they exists, Otherwise pass request to WordPress +location / { + try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; +} +location ~ \.php$ { + try_files $uri =404; + include fastcgi_params; + fastcgi_pass {{upstream}}; +} + +location ~* /cache/wp-rocket/\.html$ { + etag on; + add_header Vary "Accept-Encoding, Cookie"; + access_log off; + log_not_found off; + expire 10h; +} \ No newline at end of file diff --git a/wo/cli/templates/wpsc-php7.mustache b/wo/cli/templates/wpsc-php7.mustache deleted file mode 100644 index 1fe48a6..0000000 --- a/wo/cli/templates/wpsc-php7.mustache +++ /dev/null @@ -1,17 +0,0 @@ -# WPSC NGINX CONFIGURATION - WO v3.9.5 -# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE -# $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf - -# Use cached or actual file if they exists, Otherwise pass request to WordPress -location / { - # If we add index.php?$args its break WooCommerce like plugins - # Ref: #330 - try_files /wp-content/cache/supercache/$http_host/$cache_uri/index.html $uri $uri/ /index.php; -} -location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; - fastcgi_pass php73; - # Following line is needed by WP Super Cache plugin - fastcgi_param SERVER_NAME $http_host; -} diff --git a/wo/cli/templates/wpsc.mustache b/wo/cli/templates/wpsc.mustache index cfe5070..ce5baa7 100644 --- a/wo/cli/templates/wpsc.mustache +++ b/wo/cli/templates/wpsc.mustache @@ -1,4 +1,4 @@ -# WPSC NGINX CONFIGURATION - WO v3.9.5 +# WPSC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf @@ -11,7 +11,7 @@ location / { location ~ \.php$ { try_files $uri =404; include fastcgi_params; - fastcgi_pass php72; + fastcgi_pass {{upstream}}; # Following line is needed by WP Super Cache plugin fastcgi_param SERVER_NAME $http_host; } From b566c76a7b8f7e45ce9565a543803bd64bba7dc8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 19:29:04 +0200 Subject: [PATCH 062/102] add coverage with codacy --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c4ba976..0bdc7df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,10 @@ script: - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - sudo bash install --travis + - python3 -m pip install coverage + - python3 -m pip install codacy-coverage + - coverage xml + - python-codacy-coverage -c $TRAVIS_COMMIT -d $TRAVIS_BUILD_DIR -r coverage.xml - sudo wo --help && sudo wo stack install && sudo wo stack install --proftpd - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc From fa826224aff7ed549672bc8bb52d765b9fbb86f7 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 5 Aug 2019 21:48:14 +0200 Subject: [PATCH 063/102] Fix wprocket --- .travis.yml | 4 ---- wo/cli/plugins/site.py | 4 ++-- wo/cli/plugins/stack_pref.py | 14 ++++++++++++++ wo/cli/templates/map-wp.mustache | 10 +++++----- wo/cli/templates/nginx-core.mustache | 8 ++++---- wo/cli/templates/virtualconf-php7.mustache | 2 +- wo/cli/templates/virtualconf.mustache | 2 +- 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0bdc7df..c4ba976 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,10 +32,6 @@ script: - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - sudo bash install --travis - - python3 -m pip install coverage - - python3 -m pip install codacy-coverage - - coverage xml - - python-codacy-coverage -c $TRAVIS_COMMIT -d $TRAVIS_BUILD_DIR -r coverage.xml - sudo wo --help && sudo wo stack install && sudo wo stack install --proftpd - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 20948e4..ec63fd6 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1502,7 +1502,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - if ((oldcachetype in ['wpsc', 'basic', 'wpredis'] and + if ((oldcachetype in ['wpsc', 'basic', 'wpredis', 'wprocket'] and (data['wpfc'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpfc'])): try: @@ -1541,7 +1541,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - elif ((oldcachetype in ['wpsc', 'basic', 'wpfc'] and + elif ((oldcachetype in ['wpsc', 'basic', 'wpfc', 'wprocket'] and (data['wpredis'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpredis'])): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index df9bf9c..b7c84fb 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -173,6 +173,20 @@ def post_pref(self, apt_packages, packages): # Nginx configuration if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + # Nginx main configuration + if os.path.isfile('/etc/nginx/nginx.conf'): + if (WOVariables.wo_distro == 'ubuntu' or + WOVariables.wo_platform_codename == 'buster'): + data = dict(tls13=True) + else: + data = dict(tls13=False) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/nginx.conf') + wo_nginx = open('/etc/nginx/nginx.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'nginx-core.mustache', out=wo_nginx) + wo_nginx.close() # Fix for white screen death with NGINX PLUS if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index f764020..87be917 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -52,11 +52,6 @@ map $is_args $query_no_cache { "" 0; } -map $https $https_prefix { - default ""; - on "-https" -} - # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; @@ -68,3 +63,8 @@ map $skip_cache $cache_uri { 0 $request_uri; default 'null cache'; } + +map $https $https_prefix { + default ""; + on "-https" +} diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index dd4631e..4201f52 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -66,12 +66,12 @@ http { ssl_session_cache shared:SSL:50m; ssl_session_tickets off; ssl_prefer_server_ciphers on; - ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; - ssl_protocols TLSv1.2 TLSv1.3; + {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; + ssl_protocols TLSv1.2 TLSv1.3;{{/tls13}} ssl_ecdh_curve X25519:P-521:P-384:P-256; # Previous TLS v1.2 configuration - # ssl_protocols TLSv1.2; - # ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES; + {{^tls13}}ssl_protocols TLSv1.2; + ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES;{{/tls13}} # Common security headers more_set_headers "X-Frame-Options : SAMEORIGIN"; diff --git a/wo/cli/templates/virtualconf-php7.mustache b/wo/cli/templates/virtualconf-php7.mustache index 56c1e96..fa137cb 100644 --- a/wo/cli/templates/virtualconf-php7.mustache +++ b/wo/cli/templates/virtualconf-php7.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf{{/wprocket}} + {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf;{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php73.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/virtualconf.mustache b/wo/cli/templates/virtualconf.mustache index 87d1100..19a1afb 100644 --- a/wo/cli/templates/virtualconf.mustache +++ b/wo/cli/templates/virtualconf.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf{{/wprocket}} + {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf;{{/wprocket}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php72.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} From 09b39b2f9b5848c92430d38a6434ed8429cc98ba Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 00:31:47 +0200 Subject: [PATCH 064/102] Fix typo --- wo/cli/plugins/stack_pref.py | 2 +- wo/cli/templates/map-wp.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index b7c84fb..06bb82f 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -535,7 +535,6 @@ def post_pref(self, apt_packages, packages): Log.error( self, "Failed to generate HTTPS " "certificate for 22222") - server_ip = requests.get('http://v4.wordops.eu') if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' .format(WOVariables.wo_webroot)): @@ -553,6 +552,7 @@ def post_pref(self, apt_packages, packages): WOService.reload_service(self, 'nginx') if set(["nginx"]).issubset(set(apt_packages)): + server_ip = requests.get('http://v4.wordops.eu') print("WordOps backend configuration was successful\n" "You can access it on : https://{0}:22222" .format(server_ip)) diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 87be917..00d21da 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -66,5 +66,5 @@ map $skip_cache $cache_uri { map $https $https_prefix { default ""; - on "-https" + on "-https"; } From 6c12a950a4b792a1db98b1547c09d40e2b3a6383 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 01:31:51 +0200 Subject: [PATCH 065/102] fix server_ip --- wo/cli/plugins/stack_pref.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 06bb82f..3726783 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -550,9 +550,8 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") WOService.reload_service(self, 'nginx') - + server_ip = requests.get('http://v4.wordops.eu') if set(["nginx"]).issubset(set(apt_packages)): - server_ip = requests.get('http://v4.wordops.eu') print("WordOps backend configuration was successful\n" "You can access it on : https://{0}:22222" .format(server_ip)) From b1e3ddb3d08bba13f9f702c5acf978ff4f1c1f97 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 01:49:13 +0200 Subject: [PATCH 066/102] export logs with travis --- .travis.yml | 2 ++ wo/cli/plugins/site_functions.py | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index c4ba976..980747d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,5 +55,7 @@ script: - sudo wo stack upgrade --composer --no-prompt - sudo wo update --travis - sudo wo stack status + - sudo tar -I pigz -cf /var/log/wo + - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index dd66040..a37f7bb 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -957,15 +957,6 @@ def site_package_check(self, stype): out=wo_nginx) wo_nginx.close() - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/upstream.conf", - "php73"): - with open("/etc/nginx/conf.d/upstream.conf", "a") as php_file: - php_file.write("upstream php73 {\nserver" - "unix:/var/run/php/php73-fpm.sock;\n}\n" - "upstream debug73" - " {\nserver 127.0.0.1:9173;\n}\n") - return(stack.install(apt_packages=apt_packages, packages=packages, disp_msg=False)) From 354644d7549f4c45f0b15a2e43b55b895cb4e3f3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 11:37:03 +0200 Subject: [PATCH 067/102] Fix for redis and variable glob --- wo/cli/plugins/stack_pref.py | 18 +++++++++--------- wo/cli/templates/redis.mustache | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 3726783..f6700e6 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -536,15 +536,15 @@ def post_pref(self, apt_packages, packages): self, "Failed to generate HTTPS " "certificate for 22222") - if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' - .format(WOVariables.wo_webroot)): + if not os.path.isfile('{0}22222/conf/nginx/ssl.conf' + .format(WOVariables.wo_webroot)): - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "a") as php_file: - php_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "a") as php_file: + php_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") # Nginx Configation into GIT WOGit.add(self, @@ -574,7 +574,7 @@ def post_pref(self, apt_packages, packages): # create nginx configuration for redis if set(WOVariables.wo_redis).issubset(set(apt_packages)): if os.path.isdir('/etc/nginx/common'): - data = dict() + data = dict(upstream="php72") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php72.conf') wo_nginx = open('/etc/nginx/common/redis-php72.conf', diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index dcbb0c1..88a49e0 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -10,7 +10,7 @@ location / { location /redis-fetch { internal ; set $redis_key $args; - redis_pass redis; + redis_pass redis; } location /redis-store { internal ; From 4f3a75dc2c2809d0247086db7761548d01ce5539 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:10:50 +0200 Subject: [PATCH 068/102] Additional fix for redis --- wo/cli/plugins/site_functions.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index a37f7bb..d3b5d1a 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -290,7 +290,7 @@ def setupwordpress(self, data): Log.debug(self, "Setting up wp-config file") if not data['multisite']: Log.debug(self, "Generating wp-config for WordPress Single site") - Log.debug(self, "bash -c \"php {0} --allow-root " + Log.debug(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' --dbuser=\'{2}\' " @@ -302,7 +302,7 @@ def setupwordpress(self, data): .format(data['wo_db_pass'], "\n\ndefine(\'WP_DEBUG\', false);")) try: - if WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root" + if WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root" .format(WOVariables.wo_wpcli_path) + " config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' " @@ -325,7 +325,7 @@ def setupwordpress(self, data): raise SiteError("generate wp-config failed for wp single site") else: Log.debug(self, "Generating wp-config for WordPress multisite") - Log.debug(self, "bash -c \"php {0} --allow-root " + Log.debug(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' --dbhost=\'{2}\' " @@ -340,7 +340,7 @@ def setupwordpress(self, data): " false);", "\n\ndefine(\'WP_DEBUG\', false);")) try: - if WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root" + if WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root" .format(WOVariables.wo_wpcli_path) + " config create " + "--dbname=\'{0}\' --dbprefix=\'{1}\' " @@ -365,36 +365,36 @@ def setupwordpress(self, data): try: - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_CACHE_KEY_SALT " "\'{0}:\'\"".format(wo_domain_name)) - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_MEMORY_LIMIT " "\'128M\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_MAX_MEMORY_LIMIT " "\'256M\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set CONCATENATE_SCRIPTS " "false --raw\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_POST_REVISIONS " "\'10\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set MEDIA_TRASH " "true --raw\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set EMPTY_TRASH_DAYS " "\'15\'\"") - WOShellExec.cmd_exec(self, "bash -c \"php {0} --allow-root " + WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "config set WP_AUTO_UPDATE_CORE " "minor\"") @@ -456,14 +456,14 @@ def setupwordpress(self, data): if not data['multisite']: Log.debug(self, "Creating tables for WordPress Single site") - Log.debug(self, "php {0} --allow-root core install " + Log.debug(self, "{0} --allow-root core install " .format(WOVariables.wo_wpcli_path) + "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " .format(data['www_domain'], wo_wp_user) + "--admin_password= --admin_email=\'{1}\'" .format(wo_wp_pass, wo_wp_email)) try: - if WOShellExec.cmd_exec(self, "php {0} --allow-root core " + if WOShellExec.cmd_exec(self, "{0} --allow-root core " .format(WOVariables.wo_wpcli_path) + "install --url=\'{0}\' --title=\'{0}\' " "--admin_name=\'{1}\' " @@ -480,7 +480,7 @@ def setupwordpress(self, data): raise SiteError("setup WordPress tables failed for single site") else: Log.debug(self, "Creating tables for WordPress multisite") - Log.debug(self, "php {0} --allow-root " + Log.debug(self, "{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "core multisite-install " "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " @@ -491,7 +491,7 @@ def setupwordpress(self, data): subdomains='--subdomains' if not data['wpsubdir'] else '')) try: - if WOShellExec.cmd_exec(self, "php {0} --allow-root " + if WOShellExec.cmd_exec(self, "{0} --allow-root " .format(WOVariables.wo_wpcli_path) + "core multisite-install " "--url=\'{0}\' --title=\'{0}\' " @@ -513,7 +513,7 @@ def setupwordpress(self, data): Log.debug(self, "Updating WordPress permalink") try: - WOShellExec.cmd_exec(self, " php {0} --allow-root " + WOShellExec.cmd_exec(self, " {0} --allow-root " .format(WOVariables.wo_wpcli_path) + "rewrite structure " "/%year%/%monthnum%/%day%/%postname%/") @@ -619,7 +619,7 @@ def installwp_plugin(self, plugin_name, data): .format(plugin_name)) WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) try: - WOShellExec.cmd_exec(self, "php {0} plugin " + WOShellExec.cmd_exec(self, "{0} plugin " .format(WOVariables.wo_wpcli_path) + "--allow-root install " "{0}".format(plugin_name)) @@ -628,7 +628,7 @@ def installwp_plugin(self, plugin_name, data): raise SiteError("plugin installation failed") try: - WOShellExec.cmd_exec(self, "php {0} plugin " + WOShellExec.cmd_exec(self, "{0} plugin " .format(WOVariables.wo_wpcli_path) + "--allow-root activate " "{0} {na}" @@ -856,7 +856,7 @@ def site_package_check(self, stype): if (os.path.isfile("/etc/nginx/nginx.conf") and not os.path.isfile("/etc/nginx/common/redis-php72.conf")): - data = dict() + data = dict(upstream="php72") Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php72.conf') wo_nginx = open('/etc/nginx/common/redis-php72.conf', From ed4e2da768c50b20cb5bc31ff2cf8f0b2856a4f5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:42:00 +0200 Subject: [PATCH 069/102] Fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 980747d..744d6f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ script: - sudo wo stack upgrade --composer --no-prompt - sudo wo update --travis - sudo wo stack status - - sudo tar -I pigz -cf /var/log/wo + - sudo tar -I pigz -cf wordops.tar.gz /var/log/wo - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge From 57241a4f060fb96174b7aa83ca30976d34a2d123 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 6 Aug 2019 12:56:24 +0200 Subject: [PATCH 070/102] Final fix for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 744d6f5..3cd9119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,6 +56,6 @@ script: - sudo wo update --travis - sudo wo stack status - sudo tar -I pigz -cf wordops.tar.gz /var/log/wo - - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "$1") && echo ""; } | tee -a $HOME/.transfer.log && echo "" + - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/$(basename "wordops.tar.gz") && echo "" | sudo tee -a $HOME/.transfer.log && echo "" - echo "yes" | sudo wo stack purge --all - sudo bash install --purge From 6657e3f76cee3b126c6d0f0411a1b25a135d9d4c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 02:34:12 +0200 Subject: [PATCH 071/102] Fix WP download --- wo/cli/plugins/site_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d3b5d1a..d9233a1 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -240,7 +240,7 @@ def setupwordpress(self, data): # Random characters wo_random = (''.join(random.sample(string.ascii_uppercase + string.ascii_lowercase + - string.digits, 15))) + string.digits, 24))) wo_wp_prefix = '' # wo_wp_user = '' # wo_wp_pass = '' From 8d9d52e09ee3d22199d77bf89c1df46198935d15 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 02:45:26 +0200 Subject: [PATCH 072/102] Reduce WO size --- wo/cli/plugins/clean.py | 17 +-- wo/cli/plugins/secure.py | 49 ++++---- wo/cli/plugins/stack_pref.py | 3 +- wo/cli/plugins/stack_services.py | 185 ++++++++++++++++--------------- wo/cli/plugins/stack_upgrade.py | 63 +++++------ wo/cli/plugins/update.py | 9 +- 6 files changed, 169 insertions(+), 157 deletions(-) diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 1a8643b..589bf80 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -37,22 +37,23 @@ class WOCleanController(CementBaseController): @expose(hide=True) def default(self): - if (not (self.app.pargs.all or self.app.pargs.fastcgi or - self.app.pargs.memcached or self.app.pargs.opcache or - self.app.pargs.redis)): + pargs = self.app.pargs + if (not (pargs.all or pargs.fastcgi or + pargs.memcached or pargs.opcache or + pargs.redis)): self.clean_fastcgi() - if self.app.pargs.all: + if pargs.all: self.clean_memcached() self.clean_fastcgi() self.clean_opcache() self.clean_redis() - if self.app.pargs.fastcgi: + if pargs.fastcgi: self.clean_fastcgi() - if self.app.pargs.memcached: + if pargs.memcached: self.clean_memcached() - if self.app.pargs.opcache: + if pargs.opcache: self.clean_opcache() - if self.app.pargs.redis: + if pargs.redis: self.clean_redis() @expose(hide=True) diff --git a/wo/cli/plugins/secure.py b/wo/cli/plugins/secure.py index c9a7de6..119a31b 100644 --- a/wo/cli/plugins/secure.py +++ b/wo/cli/plugins/secure.py @@ -35,31 +35,33 @@ class WOSecureController(CementBaseController): @expose(hide=True) def default(self): - if self.app.pargs.auth: + pargs = self.app.pargs + if pargs.auth: self.secure_auth() - if self.app.pargs.port: + if pargs.port: self.secure_port() - if self.app.pargs.ip: + if pargs.ip: self.secure_ip() @expose(hide=True) def secure_auth(self): """This function secures authentication""" + pargs = self.app.pargs passwd = ''.join([random.choice (string.ascii_letters + string.digits) for n in range(24)]) - if not self.app.pargs.user_input: + if not pargs.user_input: username = input("Provide HTTP authentication user " "name [{0}] :".format(WOVariables.wo_user)) - self.app.pargs.user_input = username + pargs.user_input = username if username == "": - self.app.pargs.user_input = WOVariables.wo_user - if not self.app.pargs.user_pass: + pargs.user_input = WOVariables.wo_user + if not pargs.user_pass: password = getpass.getpass("Provide HTTP authentication " "password [{0}] :".format(passwd)) - self.app.pargs.user_pass = password + pargs.user_pass = password if password == "": - self.app.pargs.user_pass = passwd + pargs.user_pass = passwd Log.debug(self, "printf username:" "$(openssl passwd -crypt " "password 2> /dev/null)\n\"" @@ -68,8 +70,8 @@ class WOSecureController(CementBaseController): "$(openssl passwd -crypt " "{password} 2> /dev/null)\n\"" "> /etc/nginx/htpasswd-wo 2>/dev/null" - .format(username=self.app.pargs.user_input, - password=self.app.pargs.user_pass), + .format(username=pargs.user_input, + password=pargs.user_pass), log=False) WOGit.add(self, ["/etc/nginx"], msg="Adding changed secure auth into Git") @@ -77,41 +79,42 @@ class WOSecureController(CementBaseController): @expose(hide=True) def secure_port(self): """This function Secures port""" - if self.app.pargs.user_input: - while not self.app.pargs.user_input.isdigit(): + pargs = self.app.pargs + if pargs.user_input: + while not pargs.user_input.isdigit(): Log.info(self, "Please enter a valid port number ") - self.app.pargs.user_input = input("WordOps " + pargs.user_input = input("WordOps " "admin port [22222]:") - if not self.app.pargs.user_input: + if not pargs.user_input: port = input("WordOps admin port [22222]:") if port == "": - self.app.pargs.user_input = 22222 + pargs.user_input = 22222 while not port.isdigit() and port != "": Log.info(self, "Please Enter valid port number :") port = input("WordOps admin port [22222]:") - self.app.pargs.user_input = port + pargs.user_input = port WOShellExec.cmd_exec(self, "sed -i \"s/listen.*/listen " "{port} default_server ssl http2;/\" " "/etc/nginx/sites-available/22222" - .format(port=self.app.pargs.user_input)) + .format(port=pargs.user_input)) WOGit.add(self, ["/etc/nginx"], msg="Adding changed secure port into Git") if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") Log.info(self, "Successfully port changed {port}" - .format(port=self.app.pargs.user_input)) + .format(port=pargs.user_input)) @expose(hide=True) def secure_ip(self): """IP whitelisting""" - newlist = [] - if not self.app.pargs.user_input: + pargs = self.app.pargs + if not pargs.user_input: ip = input("Enter the comma separated IP addresses " "to white list [127.0.0.1]:") - self.app.pargs.user_input = ip + pargs.user_input = ip try: - user_ip = self.app.pargs.user_input.split(',') + user_ip = pargs.user_input.split(',') except Exception as e: Log.debug(self, "{0}".format(e)) user_ip = ['127.0.0.1'] diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index f6700e6..c308c24 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1235,7 +1235,8 @@ def post_pref(self, apt_packages, packages): '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) Log.info(self, "Updating phpMyAdmin, please wait...") - WOShellExec.cmd_exec(self, "/usr/local/bin/composer update --no-plugins --no-scripts " + WOShellExec.cmd_exec(self, "/usr/local/bin/composer update " + "--no-plugins --no-scripts " "-n --no-dev -d " "/var/www/22222/htdocs/db/pma/") WOFileUtils.chown(self, '{0}22222/htdocs/db/pma' diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 56985b8..86796bc 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -18,24 +18,25 @@ class WOStackStatusController(CementBaseController): def start(self): """Start services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.redis or - self.app.pargs.fail2ban or - self.app.pargs.proftpd or - self.app.pargs.netdata): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.redis or + pargs.fail2ban or + pargs.proftpd or + pargs.netdata): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -45,13 +46,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -64,27 +65,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -98,26 +99,27 @@ class WOStackStatusController(CementBaseController): def stop(self): """Stop services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.fail2ban or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.fail2ban or + pargs.netdata or + pargs.proftpd or + pargs.redis): + pargs.nginx = True + pargs.php = True + pargs.mysql = True # nginx - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") # php7.2 - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -129,14 +131,14 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") # php7.3 - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") # mysql - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -150,28 +152,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") # redis - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -185,24 +187,25 @@ class WOStackStatusController(CementBaseController): def restart(self): """Restart services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -213,13 +216,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if ((WOAptGet.is_installed(self, 'mysql-server') or @@ -233,27 +236,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -267,24 +270,25 @@ class WOStackStatusController(CementBaseController): def status(self): """Status of services""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -295,13 +299,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -314,27 +318,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: @@ -348,25 +352,26 @@ class WOStackStatusController(CementBaseController): def reload(self): """Reload service""" services = [] - if not (self.app.pargs.nginx or self.app.pargs.php or - self.app.pargs.php73 or - self.app.pargs.mysql or - self.app.pargs.netdata or - self.app.pargs.proftpd or - self.app.pargs.redis or - self.app.pargs.fail2ban): - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True + pargs = self.app.pargs + if not (pargs.nginx or pargs.php or + pargs.php73 or + pargs.mysql or + pargs.netdata or + pargs.proftpd or + pargs.redis or + pargs.fail2ban): + pargs.nginx = True + pargs.php = True + pargs.mysql = True - if self.app.pargs.nginx: + if pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom') or WOAptGet.is_installed(self, 'nginx-mainline')): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): services = services + ['php7.2-fpm'] else: @@ -377,13 +382,13 @@ class WOStackStatusController(CementBaseController): else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if ((WOVariables.wo_mysql_host is "localhost") or (WOVariables.wo_mysql_host is "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or @@ -396,27 +401,27 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir("/opt/netdata"): services = services + ['netdata'] else: diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index a834857..5b84e22 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -57,35 +57,36 @@ class WOStackUpgradeController(CementBaseController): apt_packages = [] packages = [] empty_packages = [] + pargs = pargs = self.app.pargs - if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and - (not self.app.pargs.php) and (not self.app.pargs.php73) and - (not self.app.pargs.mysql) and - (not self.app.pargs.all) and (not self.app.pargs.wpcli) and - (not self.app.pargs.netdata) and (not self.app.pargs.composer) and - (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.redis)): - self.app.pargs.web = True + if ((not pargs.web) and (not pargs.nginx) and + (not pargs.php) and (not pargs.php73) and + (not pargs.mysql) and + (not pargs.all) and (not pargs.wpcli) and + (not pargs.netdata) and (not pargs.composer) and + (not pargs.phpmyadmin) and + (not pargs.redis)): + pargs.web = True - if self.app.pargs.all: - self.app.pargs.web = True + if pargs.all: + pargs.web = True - if self.app.pargs.web: + if pargs.web: if WOAptGet.is_installed(self, 'nginx-custom'): - self.app.pargs.nginx = True + pargs.nginx = True else: Log.info(self, "Nginx is not already installed") - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): apt_packages = apt_packages + WOVariables.wo_nginx else: Log.info(self, "Nginx Stable is not already installed") - if self.app.pargs.php: + if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): if not WOAptGet.is_installed(self, 'php7.3-fpm'): apt_packages = apt_packages + WOVariables.wo_php + \ @@ -95,7 +96,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") - if self.app.pargs.php73: + if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): if not WOAptGet.is_installed(self, 'php7.2-fpm'): apt_packages = apt_packages + WOVariables.wo_php73 + \ @@ -105,19 +106,19 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.3 is not installed") - if self.app.pargs.mysql: + if pargs.mysql: if WOAptGet.is_installed(self, 'mariadb-server'): apt_packages = apt_packages + WOVariables.wo_mysql else: Log.info(self, "MariaDB is not installed") - if self.app.pargs.redis: + if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis else: Log.info(self, "Redis is not installed") - if self.app.pargs.wpcli: + if pargs.wpcli: if os.path.isfile('/usr/local/bin/wp'): packages = packages + [["https://github.com/wp-cli/wp-cli/" "releases/download/v{0}/" @@ -128,13 +129,13 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "WPCLI is not installed with WordOps") - if self.app.pargs.netdata: + if pargs.netdata: if os.path.isdir('/opt/netdata'): packages = packages + [['https://my-netdata.io/' 'kickstart-static64.sh', '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: if os.path.isdir('/var/www/22222/htdocs/db/pma'): packages = packages + \ [["https://files.phpmyadmin.net" @@ -147,7 +148,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.error(self, "phpMyAdmin isn't installed") - if self.app.pargs.composer: + if pargs.composer: if os.path.isfile('/usr/local/bin/composer'): packages = packages + [["https://getcomposer.org/installer", "/var/lib/wo/tmp/composer-install", @@ -160,7 +161,7 @@ class WOStackUpgradeController(CementBaseController): Log.info(self, "During package update process non nginx-cached" " parts of your site may remain down") # Check prompt - if (not self.app.pargs.no_prompt): + if (not pargs.no_prompt): start_upgrade = input("Do you want to continue:[y/N]") if start_upgrade != "Y" and start_upgrade != "y": Log.error(self, "Not starting package update") @@ -191,25 +192,25 @@ class WOStackUpgradeController(CementBaseController): WOService.restart_service(self, 'redis-server') if len(packages): - if self.app.pargs.wpcli: + if pargs.wpcli: WOFileUtils.remove(self, ['/usr/local/bin/wp']) - if self.app.pargs.netdata: + if pargs.netdata: WOFileUtils.remove(self, ['/var/lib/wo/tmp/kickstart.sh']) Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) - if self.app.pargs.wpcli: + if pargs.wpcli: WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) - if self.app.pargs.netdata: + if pargs.netdata: Log.info(self, "Upgrading Netdata, please wait...") WOShellExec.cmd_exec(self, "/bin/bash /var/lib/wo/tmp/" "kickstart.sh " "--dont-wait") - if self.app.pargs.composer: + if pargs.composer: Log.info(self, "Upgrading Composer, please wait...") WOShellExec.cmd_exec(self, "php -q /var/lib/wo" "/tmp/composer-install " @@ -218,7 +219,7 @@ class WOStackUpgradeController(CementBaseController): '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: Log.info(self, "Upgrading phpMyAdmin, please wait...") WOExtract.extract(self, '/var/lib/wo/tmp/pma.tar.gz', '/var/lib/wo/tmp/') diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 04e5344..b507170 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -36,20 +36,21 @@ class WOUpdateController(CementBaseController): @expose(hide=True) def default(self): + pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") - if self.app.pargs.travis: + if pargs.travis: wo_branch = "updating-configuration" install_args = "--travis --force " - elif self.app.pargs.beta: + elif pargs.beta: wo_branch = "beta" install_args = "" else: wo_branch = "master" install_args = "" - if self.app.pargs.force: + if pargs.force: install_args = install_args + "--force " - if self.app.pargs.preserve: + if pargs.preserve: install_args = install_args + "--preserve " WODownload.download(self, [["https://raw.githubusercontent.com/" From cd1fe2620145739bd262fb2277cbcf62ef795649 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 03:05:32 +0200 Subject: [PATCH 073/102] Fix redis stack install --- wo/cli/plugins/site.py | 2 +- wo/cli/plugins/stack.py | 391 +++++++++++++++-------------------- wo/cli/plugins/stack_pref.py | 59 ++++++ 3 files changed, 228 insertions(+), 224 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index ec63fd6..41538a0 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -867,7 +867,7 @@ class WOSiteUpdateController(CementBaseController): pargs = self.app.pargs if pargs.php72: - self.app.pargs.php = True + pargs.php = True if pargs.all: if pargs.site_name: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 5d753b7..9ce0a04 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -86,6 +86,9 @@ class WOStackController(CementBaseController): dict(help='Install phpRedisAdmin', action='store_true')), (['--proftpd'], dict(help='Install ProFTPd', action='store_true')), + (['--force'], + dict(help='Force install/remove/purge without prompt', + action='store_true')), ] usage = "wo stack (command) [options]" @@ -98,63 +101,64 @@ class WOStackController(CementBaseController): def install(self, packages=[], apt_packages=[], disp_msg=True): """Start installation of packages""" self.msg = [] + pargs = self.app.pargs try: # Default action for stack installation - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.mysql) and (not self.app.pargs.wpcli) and - (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.composer) and - (not self.app.pargs.netdata) and - (not self.app.pargs.dashboard) and - (not self.app.pargs.fail2ban) and - (not self.app.pargs.security) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.redis) and (not self.app.pargs.proftpd) and - (not self.app.pargs.phpredisadmin) and - (not self.app.pargs.php73)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.mysql) and (not pargs.wpcli) and + (not pargs.phpmyadmin) and + (not pargs.composer) and + (not pargs.netdata) and + (not pargs.dashboard) and + (not pargs.fail2ban) and + (not pargs.security) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.redis) and (not pargs.proftpd) and + (not pargs.phpredisadmin) and + (not pargs.php73)): + pargs.web = True + pargs.admin = True + pargs.security = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True - self.app.pargs.redis = True - self.app.pargs.proftpd = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True + pargs.redis = True + pargs.proftpd = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.admin: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.composer = True - self.app.pargs.utils = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True + if pargs.admin: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.adminer = True + pargs.phpmyadmin = True + pargs.composer = True + pargs.utils = True + pargs.netdata = True + pargs.dashboard = True + pargs.phpredisadmin = True - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # Redis - if self.app.pargs.redis: + if pargs.redis: if not WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis - self.app.pargs.php = True + pargs.php = True else: Log.info(self, "Redis already installed") # Nginx - if self.app.pargs.nginx: + if pargs.nginx: Log.debug(self, "Setting apt_packages variable for Nginx") if not (WOAptGet.is_installed(self, 'nginx-custom')): @@ -177,7 +181,7 @@ class WOStackController(CementBaseController): Log.debug(self, "Nginx Stable already installed") # PHP 7.2 - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Setting apt_packages variable for PHP 7.2") if not (WOAptGet.is_installed(self, 'php7.2-fpm')): if not (WOAptGet.is_installed(self, 'php7.3-fpm')): @@ -190,7 +194,7 @@ class WOStackController(CementBaseController): Log.info(self, "PHP 7.2 already installed") # PHP 7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Setting apt_packages variable for PHP 7.3") if not WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -203,7 +207,7 @@ class WOStackController(CementBaseController): Log.info(self, "PHP 7.3 already installed") # MariaDB 10.3 - if self.app.pargs.mysql: + if pargs.mysql: Log.debug(self, "Setting apt_packages variable for MySQL") if not WOShellExec.cmd_exec(self, "mysqladmin ping"): apt_packages = apt_packages + WOVariables.wo_mysql @@ -219,7 +223,7 @@ class WOStackController(CementBaseController): Log.info(self, "MySQL connection is already alive") # WP-CLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Setting packages variable for WP-CLI") if not WOShellExec.cmd_exec(self, "command -v wp"): packages = packages + [["https://github.com/wp-cli/wp-cli/" @@ -233,7 +237,7 @@ class WOStackController(CementBaseController): Log.info(self, "WP-CLI is already installed") # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: Log.debug(self, "Setting apt_packages variable for Fail2ban") if not WOAptGet.is_installed(self, 'fail2ban'): apt_packages = apt_packages + WOVariables.wo_fail2ban @@ -242,7 +246,7 @@ class WOStackController(CementBaseController): Log.info(self, "Fail2ban already installed") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: Log.debug(self, "Setting apt_packages variable for ProFTPd") if not WOAptGet.is_installed(self, 'proftpd-basic'): apt_packages = apt_packages + ["proftpd-basic"] @@ -251,11 +255,11 @@ class WOStackController(CementBaseController): Log.info(self, "ProFTPd already installed") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: if not os.path.isdir('/var/www/22222/htdocs/db/pma'): Log.debug(self, "Setting packages variable " "for phpMyAdmin ") - self.app.pargs.composer = True + pargs.composer = True packages = packages + [["https://github.com/phpmyadmin/" "phpmyadmin/archive/STABLE.tar.gz", "/var/lib/wo/tmp/pma.tar.gz", @@ -265,7 +269,7 @@ class WOStackController(CementBaseController): Log.info(self, "phpMyAdmin already installed") # Composer - if self.app.pargs.composer: + if pargs.composer: if not os.path.isfile('/usr/local/bin/composer'): Log.debug(self, "Setting packages variable for Composer ") packages = packages + [["https://getcomposer.org/" @@ -277,12 +281,12 @@ class WOStackController(CementBaseController): Log.info(self, "Composer already installed") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: if not os.path.isdir('/var/www/22222/htdocs/' 'cache/redis/phpRedisAdmin'): Log.debug( self, "Setting packages variable for phpRedisAdmin") - self.app.pargs.composer = True + pargs.composer = True packages = packages + [["https://github.com/" "erikdubbelboer/" "phpRedisAdmin/archive" @@ -294,7 +298,7 @@ class WOStackController(CementBaseController): Log.info(self, "phpRedisAdmin already installed") # ADMINER - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Setting packages variable for Adminer ") packages = packages + [["https://github.com/vrana/adminer/" "releases/download/v{0}" @@ -313,7 +317,7 @@ class WOStackController(CementBaseController): "Adminer theme"]] # Netdata - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Setting packages variable for Netdata") if not os.path.exists('/opt/netdata'): packages = packages + [['https://my-netdata.io/' @@ -325,7 +329,7 @@ class WOStackController(CementBaseController): Log.info(self, "Netdata already installed") # WordOps Dashboard - if self.app.pargs.dashboard: + if pargs.dashboard: if not os.path.isfile('/var/www/22222/htdocs/index.php'): Log.debug( self, "Setting packages variable for WO-Dashboard") @@ -345,7 +349,7 @@ class WOStackController(CementBaseController): Log.info(self, "WordOps dashboard already installed") # UTILS - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Setting packages variable for utils") packages = packages + [["https://raw.githubusercontent.com" "/rtCamp/eeadmin/master/cache/nginx/" @@ -408,67 +412,6 @@ class WOStackController(CementBaseController): WODownload.download(self, packages) Log.debug(self, "Calling post_pref") post_pref(self, apt_packages, packages) - if 'redis-server' in apt_packages: - # set redis.conf parameter - # set maxmemory 10% for ram below 512MB and 20% for others - # set maxmemory-policy allkeys-lru - # enable systemd service - Log.debug(self, "Enabling redis systemd service") - WOShellExec.cmd_exec(self, "systemctl enable redis-server") - if os.path.isfile("/etc/redis/redis.conf"): - wo_ram = psutil.virtual_memory().total / (1024 * 1024) - if wo_ram < 1024: - Log.debug(self, "Setting maxmemory variable to " - "{0} in redis.conf" - .format(int(wo_ram*1024*1024*0.1))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable to " - "allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - Log.debug( - self, "Setting tcp-backlog variable to " - "in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "tcp-backlog 511", - "tcp-backlog 32768") - - WOService.restart_service(self, 'redis-server') - else: - Log.debug(self, "Setting maxmemory variable to {0} " - "in redis.conf" - .format(int(wo_ram*1024*1024*0.2))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable " - "to allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - WOFileUtils.chown(self, '/etc/redis/redis.conf', - 'redis', - 'redis', - recursive=False) - WOService.restart_service(self, 'redis-server') if disp_msg: if (self.msg): @@ -483,46 +426,46 @@ class WOStackController(CementBaseController): """Start removal of packages""" apt_packages = [] packages = [] + pargs = self.app.pargs + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.php73) and (not pargs.mysql) and + (not pargs.wpcli) and (not pargs.phpmyadmin) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.composer) and (not pargs.netdata) and + (not pargs.fail2ban) and (not pargs.proftpd) and + (not pargs.security) and + (not pargs.all) and (not pargs.redis) and + (not pargs.phpredisadmin)): + pargs.web = True + pargs.admin = True + pargs.security = True - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.php73) and (not self.app.pargs.mysql) and - (not self.app.pargs.wpcli) and (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.composer) and (not self.app.pargs.netdata) and - (not self.app.pargs.fail2ban) and (not self.app.pargs.proftpd) and - (not self.app.pargs.security) and - (not self.app.pargs.all) and (not self.app.pargs.redis) and - (not self.app.pargs.phpredisadmin)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.admin: + pargs.composer = True + pargs.utils = True + pargs.netdata = True + if os.path.isdir('{0}22222/htdocs' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/*' + .format(WOVariables.wo_webroot)] - if self.app.pargs.admin: - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.composer = True - self.app.pargs.utils = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True - - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # NGINX - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): Log.debug(self, "Removing apt_packages variable of Nginx") apt_packages = apt_packages + WOVariables.wo_nginx @@ -530,7 +473,7 @@ class WOStackController(CementBaseController): Log.error(self, "Cannot Remove! Nginx Stable " "version not found.") # PHP 7.2 - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Removing apt_packages variable of PHP") if WOAptGet.is_installed(self, 'php7.2-fpm'): if not WOAptGet.is_installed(self, 'php7.3-fpm'): @@ -542,7 +485,7 @@ class WOStackController(CementBaseController): Log.error(self, "PHP 7.2 not found") # PHP7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Removing apt_packages variable of PHP 7.3") if WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -554,18 +497,18 @@ class WOStackController(CementBaseController): Log.error(self, "PHP 7.3 not found") # REDIS - if self.app.pargs.redis: + if pargs.redis: Log.debug(self, "Remove apt_packages variable of Redis") apt_packages = apt_packages + WOVariables.wo_redis # MariaDB - if self.app.pargs.mysql: + if pargs.mysql: Log.debug(self, "Removing apt_packages variable of MySQL") apt_packages = apt_packages + WOVariables.wo_mysql packages = packages + ['/usr/bin/mysqltuner'] # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): Log.debug(self, "Remove apt_packages variable of Fail2ban") apt_packages = apt_packages + WOVariables.wo_fail2ban @@ -573,7 +516,7 @@ class WOStackController(CementBaseController): Log.error(self, "Fail2ban not found") # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): Log.debug(self, "Remove apt_packages variable for ProFTPd") apt_packages = apt_packages + ["proftpd-basic"] @@ -581,19 +524,19 @@ class WOStackController(CementBaseController): Log.error(self, "ProFTPd not found") # WPCLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Removing package variable of WPCLI ") if os.path.isfile('/usr/local/bin/wp'): packages = packages + ['/usr/local/bin/wp'] else: Log.warn(self, "WP-CLI is not installed with WordOps") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: Log.debug(self, "Removing package variable of phpMyAdmin ") packages = packages + ['{0}22222/htdocs/db/pma' .format(WOVariables.wo_webroot)] # Composer - if self.app.pargs.composer: + if pargs.composer: Log.debug(self, "Removing package variable of Composer ") if os.path.isfile('/usr/local/bin/composer'): packages = packages + ['/usr/local/bin/composer'] @@ -601,7 +544,7 @@ class WOStackController(CementBaseController): Log.warn(self, "Composer is not installed with WordOps") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") if os.path.isdir('{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)): @@ -609,11 +552,11 @@ class WOStackController(CementBaseController): 'cache/redis/phpRedisAdmin' .format(WOVariables.wo_webroot)] # ADMINER - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Removing package variable of Adminer ") packages = packages + ['{0}22222/htdocs/db/adminer' .format(WOVariables.wo_webroot)] - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Removing package variable of utils ") packages = packages + ['{0}22222/htdocs/php/webgrind/' .format(WOVariables.wo_webroot), @@ -625,29 +568,31 @@ class WOStackController(CementBaseController): '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)] - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Removing Netdata") if os.path.isfile('/opt/netdata/usr/' 'libexec/netdata-uninstaller.sh'): packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] - if self.app.pargs.dashboard: + if pargs.dashboard: Log.debug(self, "Removing Wo-Dashboard") - packages = packages + ['{0}22222/htdocs/assets/' + packages = packages + ['{0}22222/htdocs/assets' .format(WOVariables.wo_webroot), '{0}22222/htdocs/index.php' .format(WOVariables.wo_webroot)] if (packages) or (apt_packages): - wo_prompt = input('Are you sure you to want to' - ' remove from server.' - '\nPackage configuration will remain' - ' on server after this operation.\n' - 'Any answer other than ' - '"yes" will be stop this' - ' operation : ') + if not pargs.force: + wo_prompt = input('Are you sure you to want to' + ' remove from server.' + '\nPackage configuration will remain' + ' on server after this operation.\n' + 'Any answer other than ' + '"yes" will be stop this' + ' operation : ') - if wo_prompt == 'YES' or wo_prompt == 'yes': + if (wo_prompt == 'YES' or wo_prompt == 'yes' + or pargs.force): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') @@ -676,46 +621,46 @@ class WOStackController(CementBaseController): """Start purging of packages""" apt_packages = [] packages = [] - + pargs = self.app.pargs # Default action for stack purge - if ((not self.app.pargs.web) and (not self.app.pargs.admin) and - (not self.app.pargs.nginx) and (not self.app.pargs.php) and - (not self.app.pargs.php73) and (not self.app.pargs.mysql) and - (not self.app.pargs.wpcli) and (not self.app.pargs.phpmyadmin) and - (not self.app.pargs.adminer) and (not self.app.pargs.utils) and - (not self.app.pargs.composer) and (not self.app.pargs.netdata) and - (not self.app.pargs.fail2ban) and (not self.app.pargs.proftpd) and - (not self.app.pargs.security) and - (not self.app.pargs.all) and (not self.app.pargs.redis) and - (not self.app.pargs.phpredisadmin)): - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.security = True + if ((not pargs.web) and (not pargs.admin) and + (not pargs.nginx) and (not pargs.php) and + (not pargs.php73) and (not pargs.mysql) and + (not pargs.wpcli) and (not pargs.phpmyadmin) and + (not pargs.adminer) and (not pargs.utils) and + (not pargs.composer) and (not pargs.netdata) and + (not pargs.fail2ban) and (not pargs.proftpd) and + (not pargs.security) and + (not pargs.all) and (not pargs.redis) and + (not pargs.phpredisadmin)): + pargs.web = True + pargs.admin = True + pargs.security = True - if self.app.pargs.all: - self.app.pargs.web = True - self.app.pargs.admin = True - self.app.pargs.php73 = True + if pargs.all: + pargs.web = True + pargs.admin = True + pargs.php73 = True - if self.app.pargs.web: - self.app.pargs.nginx = True - self.app.pargs.php = True - self.app.pargs.mysql = True - self.app.pargs.wpcli = True + if pargs.web: + pargs.nginx = True + pargs.php = True + pargs.mysql = True + pargs.wpcli = True - if self.app.pargs.admin: - self.app.pargs.adminer = True - self.app.pargs.phpmyadmin = True - self.app.pargs.utils = True - self.app.pargs.composer = True - self.app.pargs.netdata = True - self.app.pargs.dashboard = True - self.app.pargs.phpredisadmin = True + if pargs.admin: + pargs.utils = True + pargs.composer = True + pargs.netdata = True + if os.path.isdir('{0}22222/htdocs' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/*' + .format(WOVariables.wo_webroot)] - if self.app.pargs.security: - self.app.pargs.fail2ban = True + if pargs.security: + pargs.fail2ban = True # NGINX - if self.app.pargs.nginx: + if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): Log.debug(self, "Purge apt_packages variable of Nginx") apt_packages = apt_packages + WOVariables.wo_nginx @@ -724,7 +669,7 @@ class WOStackController(CementBaseController): "Nginx Stable version not found.") # PHP - if self.app.pargs.php: + if pargs.php: Log.debug(self, "Purge apt_packages variable PHP") if WOAptGet.is_installed(self, 'php7.2-fpm'): if not (WOAptGet.is_installed(self, 'php7.3-fpm')): @@ -736,7 +681,7 @@ class WOStackController(CementBaseController): Log.error(self, "Cannot Purge PHP 7.2. not found.") # PHP 7.3 - if self.app.pargs.php73: + if pargs.php73: Log.debug(self, "Removing apt_packages variable of PHP 7.3") if WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): @@ -746,19 +691,19 @@ class WOStackController(CementBaseController): apt_packages = apt_packages + WOVariables.wo_php73 # fail2ban - if self.app.pargs.fail2ban: + if pargs.fail2ban: if WOAptGet.is_installed(self, 'fail2ban'): Log.debug(self, "Purge apt_packages variable of Fail2ban") apt_packages = apt_packages + WOVariables.wo_fail2ban # proftpd - if self.app.pargs.proftpd: + if pargs.proftpd: if WOAptGet.is_installed(self, 'proftpd-basic'): Log.debug(self, "Purge apt_packages variable for ProFTPd") apt_packages = apt_packages + ["proftpd-basic"] # WP-CLI - if self.app.pargs.wpcli: + if pargs.wpcli: Log.debug(self, "Purge package variable WPCLI") if os.path.isfile('/usr/local/bin/wp'): packages = packages + ['/usr/local/bin/wp'] @@ -766,13 +711,13 @@ class WOStackController(CementBaseController): Log.warn(self, "WP-CLI is not installed with WordOps") # PHPMYADMIN - if self.app.pargs.phpmyadmin: + if pargs.phpmyadmin: packages = packages + ['{0}22222/htdocs/db/pma'. format(WOVariables.wo_webroot)] Log.debug(self, "Purge package variable phpMyAdmin") # Composer - if self.app.pargs.composer: + if pargs.composer: Log.debug(self, "Removing package variable of Composer ") if os.path.isfile('/usr/local/bin/composer'): packages = packages + ['/usr/local/bin/composer'] @@ -780,7 +725,7 @@ class WOStackController(CementBaseController): Log.warn(self, "Composer is not installed with WordOps") # PHPREDISADMIN - if self.app.pargs.phpredisadmin: + if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") if os.path.isdir('{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)): @@ -788,12 +733,12 @@ class WOStackController(CementBaseController): 'cache/redis/phpRedisAdmin' .format(WOVariables.wo_webroot)] # Adminer - if self.app.pargs.adminer: + if pargs.adminer: Log.debug(self, "Purge package variable Adminer") packages = packages + ['{0}22222/htdocs/db/adminer' .format(WOVariables.wo_webroot)] # utils - if self.app.pargs.utils: + if pargs.utils: Log.debug(self, "Purge package variable utils") packages = packages + ['{0}22222/htdocs/php/webgrind/' .format(WOVariables.wo_webroot), @@ -806,13 +751,13 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot) ] - if self.app.pargs.netdata: + if pargs.netdata: Log.debug(self, "Removing Netdata") if os.path.isfile('/opt/netdata/usr/' 'libexec/netdata-uninstaller.sh'): packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] - if self.app.pargs.dashboard: + if pargs.dashboard: Log.debug(self, "Removing Wo-Dashboard") packages = packages + ['{0}22222/htdocs/assets/' .format(WOVariables.wo_webroot), @@ -827,7 +772,7 @@ class WOStackController(CementBaseController): '"yes" will be stop this ' 'operation :') - if wo_prompt == 'YES' or wo_prompt == 'yes': + if wo_prompt == 'YES' or wo_prompt == 'yes' or pargs.force: if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index c308c24..20726ba 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1160,6 +1160,65 @@ def post_pref(self, apt_packages, packages): msg="Adding ProFTPd into Git") WOService.reload_service(self, 'proftpd') + # Redis configuration + if set(["redis-server"]).issubset(set(apt_packages)): + # set redis.conf parameter + # set maxmemory 10% for ram below 512MB and 20% for others + # set maxmemory-policy allkeys-lru + # enable systemd service + Log.debug(self, "Enabling redis systemd service") + WOShellExec.cmd_exec(self, "systemctl enable redis-server") + if os.path.isfile("/etc/redis/redis.conf"): + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + if wo_ram < 1024: + Log.debug(self, "Setting maxmemory variable to " + "{0} in redis.conf" + .format(int(wo_ram*1024*1024*0.1))) + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram*1024*1024*0.1))) + Log.debug( + self, "Setting maxmemory-policy variable to " + "allkeys-lru in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory-policy " + "noeviction", + "maxmemory-policy " + "allkeys-lru") + Log.debug( + self, "Setting tcp-backlog variable to " + "in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "tcp-backlog 511", + "tcp-backlog 32768") + + WOService.restart_service(self, 'redis-server') + else: + Log.debug(self, "Setting maxmemory variable to {0} " + "in redis.conf" + .format(int(wo_ram*1024*1024*0.2))) + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram*1024*1024*0.1))) + Log.debug( + self, "Setting maxmemory-policy variable " + "to allkeys-lru in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory-policy " + "noeviction", + "maxmemory-policy " + "allkeys-lru") + WOService.restart_service(self, 'redis-server') + if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): Log.debug(self, "Setting Privileges" From a9df4118864c7dc26a9bce2d33eaf348aa67f31c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 12:30:17 +0200 Subject: [PATCH 074/102] testing cement 2.6 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4c93342..54e1881 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,10 @@ setup(name='wo', test_suite='nose.collector', install_requires=[ # Required to build documentation - # "Sphinx >= 1.0", + "Sphinx >= 1.0", # Required for testing - # "nose", - # "coverage", + "nose", + "coverage", # Required to function 'cement == 2.8.0', 'pystache', From 3597a80449ca7fd091d379c3f176c3bdf38c290a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 7 Aug 2019 13:13:30 +0200 Subject: [PATCH 075/102] Add Cloudflare restore real-ip --- CHANGELOG.md | 11 ++++++++--- install | 6 +++--- wo/cli/plugins/stack_pref.py | 8 ++++++++ wo/cli/templates/cloudflare.mustache | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 wo/cli/templates/cloudflare.mustache diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9fd55..86019a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.7.1 - 2019-08-09 +- APT Packages configuration step with `wo stack upgrade` to apply new configurations +- Cloudflare restore real_ip configuration + #### Changed -- Cement framework updated to v2.8.0 -- Psutil updated to v5.6.3 -- PyMySQL updated to v0.9.3 +- Moving package configuration in a new plugin stack_pref.py +- Set WordOps backend password length from 16 to 24 +- Upgrade framework cement to 2.6.0 +- Upgrade PyMySQL to 0.9.3 +- Upgrade Psutil to 5.6.3 #### Fixed diff --git a/install b/install index d7a2352..955dd8c 100755 --- a/install +++ b/install @@ -406,9 +406,9 @@ wo_install_acme_sh() { wo_install() { { rm -f /etc/bash_completion.d/wo_auto.rc - rm -rf /tmp/WordOps - git clone -b "$wo_branch" --depth=50 https://github.com/WordOps/WordOps.git /tmp/WordOps - cd /tmp/WordOps || exit 1 + rm -rf /var/lib/wo/tmp/WordOps-* + curl -sL https://github.com/WordOps/WordOps/archive/${wo_branch}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp + cd /var/lib/wo/tmp/WordOps-${wo_branch} || exit 1 } \ >> "$wo_install_log" 2>&1 diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 20726ba..e432df6 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -227,6 +227,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/cloudflare.conf') + wo_nginx = open('/etc/nginx/conf.d/cloudflare.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'cloudflare.mustache', + out=wo_nginx) + wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/conf.d/' 'map-wp-fastcgi-cache.conf') diff --git a/wo/cli/templates/cloudflare.mustache b/wo/cli/templates/cloudflare.mustache new file mode 100644 index 0000000..38355fe --- /dev/null +++ b/wo/cli/templates/cloudflare.mustache @@ -0,0 +1,23 @@ +# WordOps (wo) set visitors real ip with Cloudflare +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/12; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; +real_ip_header CF-Connecting-IP; From cd423482509b12b11c4f357c83df80610945371e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 08:40:03 +0200 Subject: [PATCH 076/102] Update templates --- install | 2 -- wo/cli/templates/fastcgi.mustache | 8 ++--- wo/cli/templates/locations.mustache | 50 ++++++++++++++++++++++++++++- wo/cli/templates/redis.mustache | 2 +- wo/cli/templates/sysctl.mustache | 12 +++---- wo/cli/templates/wpcommon.mustache | 14 ++++++++ wo/cli/templates/wprocket.mustache | 16 ++++----- 7 files changed, 80 insertions(+), 24 deletions(-) diff --git a/install b/install index 955dd8c..00f0f1e 100755 --- a/install +++ b/install @@ -667,8 +667,6 @@ wo_tweak_kernel() { fi # apply sysctl tweaks sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf - else - sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf fi fi if [ ! -x /opt/wo-kernel.sh ]; then diff --git a/wo/cli/templates/fastcgi.mustache b/wo/cli/templates/fastcgi.mustache index 271e751..c5a68e2 100644 --- a/wo/cli/templates/fastcgi.mustache +++ b/wo/cli/templates/fastcgi.mustache @@ -1,5 +1,5 @@ # FastCGI cache settings -fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=60m max_size=256M; +fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=WORDPRESS:50m inactive=6h max_size=256M; fastcgi_cache_key "$scheme$request_method$host$request_uri"; fastcgi_cache_use_stale error timeout invalid_header updating http_500 http_503; fastcgi_cache_lock on; @@ -7,11 +7,11 @@ fastcgi_cache_lock_age 5s; fastcgi_cache_lock_timeout 5s; fastcgi_cache_methods GET HEAD; fastcgi_cache_background_update on; -fastcgi_cache_valid 200 1h; +fastcgi_cache_valid 200 24h; fastcgi_cache_valid 301 302 30m; fastcgi_cache_valid 499 502 503 1m; -fastcgi_cache_valid 404 15m; -fastcgi_cache_valid any 15m; +fastcgi_cache_valid 404 1h; +fastcgi_cache_valid any 1h; fastcgi_buffers 16 16k; fastcgi_buffer_size 32k; fastcgi_param SERVER_NAME $http_host; diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index 26e8230..f944a40 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -7,11 +7,35 @@ location = /favicon.ico { expires max; } # Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp)$ { +location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json)$ { add_header "Access-Control-Allow-Origin" "*"; access_log off; log_not_found off; expires max; + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; } # Cache css & js files location ~* \.(?:css(\.map)?|js(\.map)?)$ { @@ -19,6 +43,30 @@ location ~* \.(?:css(\.map)?|js(\.map)?)$ { access_log off; log_not_found off; expires 30d; + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; } # Security settings for better privacy # Deny hidden files diff --git a/wo/cli/templates/redis.mustache b/wo/cli/templates/redis.mustache index 88a49e0..7864dd9 100644 --- a/wo/cli/templates/redis.mustache +++ b/wo/cli/templates/redis.mustache @@ -17,7 +17,7 @@ location /redis-store { set_unescape_uri $key $arg_key ; redis2_query set $key $echo_request_body; redis2_query expire $key 14400; - redis2_pass redis; + redis2_pass redis; } diff --git a/wo/cli/templates/sysctl.mustache b/wo/cli/templates/sysctl.mustache index cc2c332..271298c 100644 --- a/wo/cli/templates/sysctl.mustache +++ b/wo/cli/templates/sysctl.mustache @@ -1,10 +1,10 @@ # Kernel sysctl configuration file for Linux # -# Version 1.16 - 2019-10-25 +# Version 1.18 - 2019-08-07 # Michiel Klaver - IT Professional -# Modified by VirtuBox +# Modified by VirtuBox (https://github.com/VirtuBox) +# For WordOps (https://github.com/WordOps/WordOps) # -# Instructions available on https://github.com/VirtuBox/ubuntu-nginx-web-server # # Sources : # https://klaver.it/linux/sysctl.conf @@ -27,10 +27,6 @@ # http://lartc.org/howto/lartc.kernel.obscure.html # http://en.wikipedia.org/wiki/Sysctl # -# Usage -# wget -O /etc/sysctl.d/60-ubuntu-nginx-web-server.conf https://virtubox.github.io/ubuntu-nginx-web-server/files/etc/sysctl.d/60-ubuntu-nginx-web-server.conf -# -# sysctl -e -p /etc/sysctl.d/60-ubuntu-nginx-web-server.conf # For binary values, 0 is disabled, 1 is enabled. See sysctl(8) and sysctl.conf(5) for more details. # @@ -200,6 +196,8 @@ net.core.somaxconn = 32768 # Increase number of incoming connections backlog net.core.netdev_max_backlog = 16384 net.core.dev_weight = 64 +net.core.netdev_budget = 600 +net.core.netdev_budget_usecs = 4000 # Increase the maximum amount of option memory buffers net.core.optmem_max = 65535 diff --git a/wo/cli/templates/wpcommon.mustache b/wo/cli/templates/wpcommon.mustache index f646cd6..33a9e4a 100644 --- a/wo/cli/templates/wpcommon.mustache +++ b/wo/cli/templates/wpcommon.mustache @@ -56,6 +56,20 @@ location /wp-content/plugins/ewww-image-optimizer/images { deny all; } } +# enable gzip on static assets - php files are forbidden +location /wp-content/cache { +# Cache css & js files + location ~* \.(?:css(\.map)?|js(\.map)?|.html)$ { + add_header "Access-Control-Allow-Origin" "*"; + access_log off; + log_not_found off; + expires 30d; + } + location ~ \.php$ { +#Prevent Direct Access Of PHP Files From Web Browsers + deny all; + } +} # Deny access to any files with a .php extension in the uploads directory # Works in sub-directory installs and also in multisite network # Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban) diff --git a/wo/cli/templates/wprocket.mustache b/wo/cli/templates/wprocket.mustache index e73f859..14e3600 100644 --- a/wo/cli/templates/wprocket.mustache +++ b/wo/cli/templates/wprocket.mustache @@ -1,21 +1,19 @@ # WPSC NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf - # Use cached or actual file if they exists, Otherwise pass request to WordPress location / { - try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; + try_files /wp-content/cache/wp-rocket/$http_host/$cache_uri/index$https_prefix.html $uri $uri/ /index.php$is_args$args; } location ~ \.php$ { - try_files $uri =404; - include fastcgi_params; + try_files $uri =404; + include fastcgi_params; fastcgi_pass {{upstream}}; } - location ~* /cache/wp-rocket/\.html$ { - etag on; - add_header Vary "Accept-Encoding, Cookie"; + etag on; + add_header Vary "Accept-Encoding, Cookie"; access_log off; log_not_found off; - expire 10h; -} \ No newline at end of file + expires 10h; +} From 232516fab36dc4c37c9ab06c6e4421f4cc3526b6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 20:30:15 +0200 Subject: [PATCH 077/102] testing cement 2.10.12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a4b1a5..29f8528 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.8.0 +cement>=2.10.12 From 7add4eb9a68c994e4bf932bc8d185444d05879e4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 20:36:03 +0200 Subject: [PATCH 078/102] testing again with cement 2.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 29f8528..6a4b1a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cement>=2.10.12 +cement>=2.8.0 From 5ba4a4ca5a7b1bdc6a5c5b8b4a2e63ef99cedca4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 21:16:19 +0200 Subject: [PATCH 079/102] Fix redis install --- wo/cli/plugins/stack_pref.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index e432df6..92bb787 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1225,6 +1225,8 @@ def post_pref(self, apt_packages, packages): "noeviction", "maxmemory-policy " "allkeys-lru") + WOFileUtils.chown(self, '/etc/redis/redis.conf', + redis, redis) WOService.restart_service(self, 'redis-server') if (packages): From 77e1fdac730812b936a4ee61c5be779dc938b1e9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 8 Aug 2019 21:42:26 +0200 Subject: [PATCH 080/102] Fix redis --- wo/cli/plugins/stack_pref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 92bb787..1d08dfe 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1226,7 +1226,7 @@ def post_pref(self, apt_packages, packages): "maxmemory-policy " "allkeys-lru") WOFileUtils.chown(self, '/etc/redis/redis.conf', - redis, redis) + 'redis', 'redis', recursive=False) WOService.restart_service(self, 'redis-server') if (packages): From ae2936282378fc3ce147f903783617c246fe1ada Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 12 Aug 2019 22:21:29 +0200 Subject: [PATCH 081/102] Apply redis fix from master branch --- wo/cli/plugins/stack_pref.py | 51 +++++++++++++++--------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 1d08dfe..0b2923c 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1188,24 +1188,7 @@ def post_pref(self, apt_packages, packages): "maxmemory {0}" .format (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable to " - "allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - Log.debug( - self, "Setting tcp-backlog variable to " - "in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "tcp-backlog 511", - "tcp-backlog 32768") - WOService.restart_service(self, 'redis-server') else: Log.debug(self, "Setting maxmemory variable to {0} " "in redis.conf" @@ -1215,19 +1198,27 @@ def post_pref(self, apt_packages, packages): "# maxmemory ", "maxmemory {0}" .format - (int(wo_ram*1024*1024*0.1))) - Log.debug( - self, "Setting maxmemory-policy variable " - "to allkeys-lru in redis.conf") - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory-policy " - "noeviction", - "maxmemory-policy " - "allkeys-lru") - WOFileUtils.chown(self, '/etc/redis/redis.conf', - 'redis', 'redis', recursive=False) - WOService.restart_service(self, 'redis-server') + (int(wo_ram*1024*1024*0.2))) + + Log.debug( + self, "Setting maxmemory-policy variable to " + "allkeys-lru in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "# maxmemory-policy " + "noeviction", + "maxmemory-policy " + "allkeys-lru") + Log.debug( + self, "Setting tcp-backlog variable to " + "in redis.conf") + WOFileUtils.searchreplace(self, + "/etc/redis/redis.conf", + "tcp-backlog 511", + "tcp-backlog 32768") + WOFileUtils.chown(self, '/etc/redis/redis.conf', + 'redis', 'redis', recursive=False) + WOService.restart_service(self, 'redis-server') if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): From 1dc564f94e58390fe1fad766cefb56836026ef7b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 13 Aug 2019 03:46:15 +0200 Subject: [PATCH 082/102] Add wprocket to cache list --- README.md | 4 ++-- wo/cli/plugins/clean.py | 20 ++------------------ wo/cli/plugins/site_functions.py | 2 +- wo/cli/plugins/stack_services.py | 20 ++++++++++---------- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 9605769..ce0bff3 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ MIT Commits GitHub release -
- +
WordOps install +codacy Badge Twitter Badge Slack diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 589bf80..7d7acfa 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -26,8 +26,6 @@ class WOCleanController(CementBaseController): dict(help='Clean all cache', action='store_true')), (['--fastcgi'], dict(help='Clean FastCGI cache', action='store_true')), - (['--memcached'], - dict(help='Clean MemCached', action='store_true')), (['--opcache'], dict(help='Clean OpCache', action='store_true')), (['--redis'], @@ -43,14 +41,11 @@ class WOCleanController(CementBaseController): pargs.redis)): self.clean_fastcgi() if pargs.all: - self.clean_memcached() self.clean_fastcgi() self.clean_opcache() self.clean_redis() if pargs.fastcgi: self.clean_fastcgi() - if pargs.memcached: - self.clean_memcached() if pargs.opcache: self.clean_opcache() if pargs.redis: @@ -65,23 +60,12 @@ class WOCleanController(CementBaseController): else: Log.info(self, "Redis is not installed") - @expose(hide=True) - def clean_memcached(self): - try: - if(WOAptGet.is_installed(self, "memcached")): - WOService.restart_service(self, "memcached") - Log.info(self, "Cleaning MemCached") - else: - Log.info(self, "Memcached not installed") - except Exception as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Unable to restart Memcached", False) - @expose(hide=True) def clean_fastcgi(self): if(os.path.isdir("/var/run/nginx-cache")): Log.info(self, "Cleaning NGINX FastCGI cache") WOShellExec.cmd_exec(self, "rm -rf /var/run/nginx-cache/*") + WOService.restart_service(self, 'nginx') else: Log.error(self, "Unable to clean FastCGI cache", False) @@ -89,7 +73,7 @@ class WOCleanController(CementBaseController): def clean_opcache(self): try: Log.info(self, "Cleaning opcache") - urllib.request.urlopen(" https://127.0.0.1:22222/cache" + urllib.request.urlopen("https://127.0.0.1:22222/cache" "/opcache/opgui.php?reset=1").read() except Exception as e: Log.debug(self, "{0}".format(e)) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d9233a1..f710447 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1098,7 +1098,7 @@ def detSitePar(opts): if val and key in ['html', 'php', 'mysql', 'wp', 'wpsubdir', 'wpsubdomain', 'php73']: typelist.append(key) - elif val and key in ['wpfc', 'wpsc', 'wpredis']: + elif val and key in ['wpfc', 'wpsc', 'wpredis', 'wprocket']: cachelist.append(key) if len(typelist) > 1 or len(cachelist) > 1: diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 86796bc..f0794c1 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -53,8 +53,8 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") if pargs.mysql: - if ((WOVariables.wo_mysql_host is "localhost") or - (WOVariables.wo_mysql_host is "127.0.0.1")): + if ((WOVariables.wo_mysql_host == "localhost") or + (WOVariables.wo_mysql_host == "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or WOAptGet.is_installed(self, 'percona-server-server-5.6') or WOAptGet.is_installed(self, 'mariadb-server')): @@ -139,8 +139,8 @@ class WOStackStatusController(CementBaseController): # mysql if pargs.mysql: - if ((WOVariables.wo_mysql_host is "localhost") or - (WOVariables.wo_mysql_host is "127.0.0.1")): + if ((WOVariables.wo_mysql_host == "localhost") or + (WOVariables.wo_mysql_host == "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or WOAptGet.is_installed(self, 'percona-server-server-5.6') or WOAptGet.is_installed(self, 'mariadb-server')): @@ -223,8 +223,8 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") if pargs.mysql: - if ((WOVariables.wo_mysql_host is "localhost") or - (WOVariables.wo_mysql_host is "127.0.0.1")): + if ((WOVariables.wo_mysql_host == "localhost") or + (WOVariables.wo_mysql_host == "127.0.0.1")): if ((WOAptGet.is_installed(self, 'mysql-server') or WOAptGet.is_installed(self, 'percona-server-server-5.6') or @@ -306,8 +306,8 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") if pargs.mysql: - if ((WOVariables.wo_mysql_host is "localhost") or - (WOVariables.wo_mysql_host is "127.0.0.1")): + if ((WOVariables.wo_mysql_host == "localhost") or + (WOVariables.wo_mysql_host == "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or WOAptGet.is_installed(self, 'percona-server-server-5.6') or WOAptGet.is_installed(self, 'mariadb-server')): @@ -389,8 +389,8 @@ class WOStackStatusController(CementBaseController): Log.info(self, "PHP7.3-FPM is not installed") if pargs.mysql: - if ((WOVariables.wo_mysql_host is "localhost") or - (WOVariables.wo_mysql_host is "127.0.0.1")): + if ((WOVariables.wo_mysql_host == "localhost") or + (WOVariables.wo_mysql_host == "127.0.0.1")): if (WOAptGet.is_installed(self, 'mysql-server') or WOAptGet.is_installed(self, 'percona-server-server-5.6') or WOAptGet.is_installed(self, 'mariadb-server')): From a248eac0c9f39f7b9be1a037fdb836714ce62ff9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 13 Aug 2019 09:46:21 +0200 Subject: [PATCH 083/102] Move Gzip & Brotli config outside of nginx.conf --- CHANGELOG.md | 2 + install | 21 ++++------- wo/cli/plugins/stack_pref.py | 33 ++++++++++++---- wo/cli/templates/brotli.mustache | 48 ++++++++++++++++++++++++ wo/cli/templates/gzip.mustache | 30 +++++++++++++++ wo/cli/templates/nginx-core.mustache | 56 ---------------------------- 6 files changed, 114 insertions(+), 76 deletions(-) create mode 100644 wo/cli/templates/brotli.mustache create mode 100644 wo/cli/templates/gzip.mustache diff --git a/CHANGELOG.md b/CHANGELOG.md index 345ad34..7dbb655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed +- Gzip enabled again by default with configuration in /etc/nginx/conf.d/gzip.conf +- Brotli configuration moved in /etc/nginx/conf.d/brotli.conf (easier to disable in case of issues) - Moving package configuration in a new plugin stack_pref.py - Set WordOps backend password length from 16 to 24 - Upgrade framework cement to 2.6.0 diff --git a/install b/install index fabad86..74ead1f 100755 --- a/install +++ b/install @@ -475,11 +475,6 @@ wo_upgrade_nginx() { # stop nginx service nginx stop - # prevent apt preference to block install - [ -f /etc/apt/preferences.d/nginx-block ] && { - mv /etc/apt/preferences.d/nginx-block "$HOME/nginx-block" - } - # install new nginx package if [ -n "$CHECK_NGINX_EE" ] || [ -n "$CHECK_NGINX_WO" ]; then if [ -x /usr/local/bin/wo ]; then @@ -539,12 +534,8 @@ wo_upgrade_nginx() { systemctl start nginx fi - # set back apt preference - [ -f "$HOME/nginx-block" ] && { - mv "$HOME/nginx-block" /etc/apt/preferences.d/nginx-block - } - - } >> "$wo_install_log" 2>&1 + } \ + >> "$wo_install_log" 2>&1 } @@ -732,8 +723,10 @@ else if [ -z "$wo_preserve_config" ]; then if [ -n "$(command -v nginx)" ]; then if [ ! -f /etc/nginx/common/release ] || ! grep -q "v3.9.7" /etc/nginx/common/release; then - wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log - wo_upgrade_nginx | tee -ai $wo_install_log + if [ ! -f /etc/apt/preferences.d/nginx-block ]; then + wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log + wo_upgrade_nginx | tee -ai $wo_install_log + fi fi fi fi @@ -833,5 +826,7 @@ else wo_lib_echo "WordOps Documentation : https://docs.wordops.net" wo_lib_echo "WordOps Community Forum : https://community.wordops.net" echo + wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/" + echo fi diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 0b2923c..268a4b5 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -180,13 +180,32 @@ def post_pref(self, apt_packages, packages): data = dict(tls13=True) else: data = dict(tls13=False) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/nginx.conf') - wo_nginx = open('/etc/nginx/nginx.conf', - encoding='utf-8', mode='w') - self.app.render( - (data), 'nginx-core.mustache', out=wo_nginx) - wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/nginx.conf') + wo_nginx = open('/etc/nginx/nginx.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'nginx-core.mustache', out=wo_nginx) + wo_nginx.close() + + if not os.path.isfile('/etc/nginx/conf.d/gzip.conf'): + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/gzip.conf') + wo_nginx = open('/etc/nginx/conf.d/gzip.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'gzip.mustache', out=wo_nginx) + wo_nginx.close() + + if not os.path.isfile('/etc/nginx/conf.d/brotli.conf'): + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/conf.d/brotli.conf') + wo_nginx = open('/etc/nginx/conf.d/brotli.conf', + encoding='utf-8', mode='w') + self.app.render( + (data), 'brotli.mustache', out=wo_nginx) + wo_nginx.close() # Fix for white screen death with NGINX PLUS if not WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', diff --git a/wo/cli/templates/brotli.mustache b/wo/cli/templates/brotli.mustache new file mode 100644 index 0000000..7dc08c7 --- /dev/null +++ b/wo/cli/templates/brotli.mustache @@ -0,0 +1,48 @@ + ## + # Brotli Settings + ## + + brotli on; + brotli_static on; + brotli_buffers 16 8k; + brotli_min_length 64000; + brotli_comp_level 4; + brotli_types + application/atom+xml + application/geo+json + application/javascript + application/json + application/ld+json + application/manifest+json + application/rdf+xml + application/rss+xml + application/vnd.ms-fontobject + application/wasm + application/x-font-opentype + application/x-font-truetype + application/x-font-ttf + application/x-javascript + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + application/xml+rss + font/eot + font/opentype + font/otf + image/bmp + image/svg+xml + image/vnd.microsoft.icon + image/x-icon + image/x-win-bitmap + text/cache-manifest + text/calendar + text/css + text/javascript + text/markdown + text/plain + text/vcard + text/vnd.rim.location.xloc + text/vtt + text/x-component + text/x-cross-domain-policy + text/xml; \ No newline at end of file diff --git a/wo/cli/templates/gzip.mustache b/wo/cli/templates/gzip.mustache new file mode 100644 index 0000000..62b60ed --- /dev/null +++ b/wo/cli/templates/gzip.mustache @@ -0,0 +1,30 @@ + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; \ No newline at end of file diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index 4201f52..dd1fb4f 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -106,62 +106,6 @@ http { '$http_host "$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" "$server_protocol"'; - ## - # Gzip Settings - ## - - # mitigation for CRIME/BREACH attacks - gzip off; - - ## - # Brotli Settings - ## - - brotli on; - brotli_static on; - brotli_buffers 16 8k; - brotli_min_length 64000; - brotli_comp_level 4; - brotli_types - application/atom+xml - application/geo+json - application/javascript - application/json - application/ld+json - application/manifest+json - application/rdf+xml - application/rss+xml - application/vnd.ms-fontobject - application/wasm - application/x-font-opentype - application/x-font-truetype - application/x-font-ttf - application/x-javascript - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - application/xml+rss - font/eot - font/opentype - font/otf - image/bmp - image/svg+xml - image/vnd.microsoft.icon - image/x-icon - image/x-win-bitmap - text/cache-manifest - text/calendar - text/css - text/javascript - text/markdown - text/plain - text/vcard - text/vnd.rim.location.xloc - text/vtt - text/x-component - text/x-cross-domain-policy - text/xml; - ## # Virtual Host Configs ## From 5895ee02673da6e4722f7d62196beabd53c6c2e9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 13 Aug 2019 09:58:25 +0200 Subject: [PATCH 084/102] Update travis and changelog --- .travis.yml | 1 + CHANGELOG.md | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3cd9119..0fb937d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ script: - sudo wp --allow-root --info - sudo wo info - sudo cat /etc/nginx/nginx.conf + - sudo tree -L 2 /etc/nginx - sudo cat /etc/mysql/my.cnf - sudo wo stack upgrade --nginx --no-prompt - sudo wo stack upgrade --php --no-prompt diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dbb655..f1722d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Changed + +- Do not force Nginx upgrade if a custom Nginx package compiled with nginx-ee is detected +- Gzip enabled again by default with configuration in /etc/nginx/conf.d/gzip.conf +- Brotli configuration moved in /etc/nginx/conf.d/brotli.conf (easier to disable in case of issues) +- Moving package configuration in a new plugin stack_pref.py + ### v3.9.7.2 - 2019-08-12 #### Fixed @@ -21,9 +28,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed -- Gzip enabled again by default with configuration in /etc/nginx/conf.d/gzip.conf -- Brotli configuration moved in /etc/nginx/conf.d/brotli.conf (easier to disable in case of issues) -- Moving package configuration in a new plugin stack_pref.py - Set WordOps backend password length from 16 to 24 - Upgrade framework cement to 2.6.0 - Upgrade PyMySQL to 0.9.3 From 26806a292bb595523cf25cf118e69b6f517ed475 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 13 Aug 2019 17:43:49 +0200 Subject: [PATCH 085/102] Cleanup conf --- CHANGELOG.md | 6 +++ wo/cli/templates/brotli.mustache | 1 - wo/cli/templates/fail2ban-forbidden.mustache | 1 - wo/cli/templates/gzip.mustache | 50 ++++++++++---------- wo/cli/templates/locations.mustache | 50 +------------------- 5 files changed, 32 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1722d5..2295fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- Allow web browser caching for json and webmanifest files +- nginx-core.mustache template used to render nginx.conf during stack setup + #### Changed - Do not force Nginx upgrade if a custom Nginx package compiled with nginx-ee is detected - Gzip enabled again by default with configuration in /etc/nginx/conf.d/gzip.conf - Brotli configuration moved in /etc/nginx/conf.d/brotli.conf (easier to disable in case of issues) - Moving package configuration in a new plugin stack_pref.py +- Cleanup templates by removing all doublons (with/without php7) and replacing them with variables ### v3.9.7.2 - 2019-08-12 diff --git a/wo/cli/templates/brotli.mustache b/wo/cli/templates/brotli.mustache index 7dc08c7..6a8ae82 100644 --- a/wo/cli/templates/brotli.mustache +++ b/wo/cli/templates/brotli.mustache @@ -5,7 +5,6 @@ brotli on; brotli_static on; brotli_buffers 16 8k; - brotli_min_length 64000; brotli_comp_level 4; brotli_types application/atom+xml diff --git a/wo/cli/templates/fail2ban-forbidden.mustache b/wo/cli/templates/fail2ban-forbidden.mustache index 6f708bf..6488363 100644 --- a/wo/cli/templates/fail2ban-forbidden.mustache +++ b/wo/cli/templates/fail2ban-forbidden.mustache @@ -1,4 +1,3 @@ [Definition] failregex = ^ \[error\] \d+#\d+: .* forbidden .*, client: , .*$ - ignoreregex = diff --git a/wo/cli/templates/gzip.mustache b/wo/cli/templates/gzip.mustache index 62b60ed..144b5be 100644 --- a/wo/cli/templates/gzip.mustache +++ b/wo/cli/templates/gzip.mustache @@ -2,29 +2,29 @@ # Gzip Settings ## - gzip on; - gzip_disable "msie6"; + gzip on; + gzip_disable "msie6"; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component - text/xml - text/javascript; \ No newline at end of file + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component + text/xml + text/javascript; \ No newline at end of file diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index f944a40..5773adc 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -7,35 +7,11 @@ location = /favicon.ico { expires max; } # Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json)$ { +location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest)$ { add_header "Access-Control-Allow-Origin" "*"; access_log off; log_not_found off; expires max; - gzip on; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component - text/xml - text/javascript; } # Cache css & js files location ~* \.(?:css(\.map)?|js(\.map)?)$ { @@ -43,30 +19,6 @@ location ~* \.(?:css(\.map)?|js(\.map)?)$ { access_log off; log_not_found off; expires 30d; - gzip on; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_buffers 16 8k; - gzip_http_version 1.1; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component - text/xml - text/javascript; } # Security settings for better privacy # Deny hidden files From 937e61305ac4edf22b8cb181fbecce011b52a888 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 14 Aug 2019 13:56:37 +0200 Subject: [PATCH 086/102] Fix HSTS and update changelog --- .travis.yml | 1 + CHANGELOG.md | 14 ++++++++++---- README.md | 2 +- wo/cli/plugins/site_functions.py | 2 +- wo/cli/plugins/stack.py | 12 ++++++------ wo/cli/plugins/stack_pref.py | 6 +++--- wo/cli/templates/gzip.mustache | 25 +++++++++++++++++++++++-- wo/cli/templates/nginx-core.mustache | 2 +- 8 files changed, 46 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0fb937d..c9a4f97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,7 @@ script: - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc - sudo wo site create wpsc-php73.net --wpsc --php73 && sudo wo site create wpfc-php73.net --wpfc --php73 + - sudo wo site create wprocket.net --wprocket && sudo wo site create wprocket-php73.net --wprocket --php73 - sudo wo site create wpredis.net --wpredis && sudo wo site create wpredis-php73.net --wpredis --php73 - sudo wo site create wpsubdir1.com --wpsubdir && sudo wo site create wpsubdir-php73.com --wpsubdir --php73 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2295fb9..73d1ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,14 +12,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow web browser caching for json and webmanifest files - nginx-core.mustache template used to render nginx.conf during stack setup +- APT Packages configuration step with `wo stack upgrade` to apply new configurations +- Cloudflare restore real_ip configuration +- WP-Rocket support with the flag `--wprocket` #### Changed - Do not force Nginx upgrade if a custom Nginx package compiled with nginx-ee is detected - Gzip enabled again by default with configuration in /etc/nginx/conf.d/gzip.conf -- Brotli configuration moved in /etc/nginx/conf.d/brotli.conf (easier to disable in case of issues) +- Brotli configuration moved in /etc/nginx/conf.d/brotli.conf.disabled (disabled by default) - Moving package configuration in a new plugin stack_pref.py - Cleanup templates by removing all doublons (with/without php7) and replacing them with variables +- Updated Nginx to v1.16.1 in response to HTTP/2 vulnerabilites discovered +- Disable temporary adding swap feature (not working) + +#### Fixed + +- Error in HSTS header syntax ### v3.9.7.2 - 2019-08-12 @@ -29,9 +38,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.7.1 - 2019-08-09 -- APT Packages configuration step with `wo stack upgrade` to apply new configurations -- Cloudflare restore real_ip configuration - #### Changed - Set WordOps backend password length from 16 to 24 diff --git a/README.md b/README.md index ce0bff3..2e69c14 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ - **Easy to install** : One step automated installer with migration from EasyEngine v3 support - **Fast deployment** : Fast and automated WordPress, Nginx, PHP, MySQL & Redis installation -- **Custom Nginx build** : Nginx 1.16.0 - TLS v1.3 Cloudflare HTTP/2 HPACK & Brotli support +- **Custom Nginx build** : Nginx 1.16.1 - TLS v1.3 Cloudflare HTTP/2 HPACK & Brotli support - **Up-to-date** : PHP 7.2 & 7.3, MariaDB 10.3 & Redis 5.0 - **Secured** : Hardened WordPress security with strict Nginx location directives - **Powerful** : Optimized Nginx configurations with multiple cache backends support diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index f710447..4cd1d27 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1534,7 +1534,7 @@ def setupHsts(self, wo_domain_name): hstsconf.write("more_set_headers " "\"Strict-Transport-Security: " "max-age=31536000; " - "'includeSubDomains; " + "includeSubDomains; " "preload\";") hstsconf.close() return 0 diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 9e7996e..35bbcf5 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -397,12 +397,12 @@ class WOStackController(CementBaseController): Log.debug(self, "Calling pre_pref") pre_pref(self, apt_packages) if (apt_packages): - meminfo = (os.popen('/bin/cat /proc/meminfo ' - '| grep MemTotal').read()).split(":") - memsplit = re.split(" kB", meminfo[1]) - wo_mem = int(memsplit[0]) - if (wo_mem < 4000000): - WOSwap.add(self) + # meminfo = (os.popen('/bin/cat /proc/meminfo ' + # '| grep MemTotal').read()).split(":") + # memsplit = re.split(" kB", meminfo[1]) + # wo_mem = int(memsplit[0]) + # if (wo_mem < 4000000): + # WOSwap.add(self) Log.info(self, "Updating apt-cache, please wait...") WOAptGet.update(self) Log.info(self, "Installing packages, please wait...") diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 268a4b5..6cbf504 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -198,10 +198,10 @@ def post_pref(self, apt_packages, packages): (data), 'gzip.mustache', out=wo_nginx) wo_nginx.close() - if not os.path.isfile('/etc/nginx/conf.d/brotli.conf'): + if not os.path.isfile('/etc/nginx/conf.d/brotli.conf.disabled'): Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/conf.d/brotli.conf') - wo_nginx = open('/etc/nginx/conf.d/brotli.conf', + 'file /etc/nginx/conf.d/brotli.conf.disabled') + wo_nginx = open('/etc/nginx/conf.d/brotli.conf.disabled', encoding='utf-8', mode='w') self.app.render( (data), 'brotli.mustache', out=wo_nginx) diff --git a/wo/cli/templates/gzip.mustache b/wo/cli/templates/gzip.mustache index 144b5be..ff591fa 100644 --- a/wo/cli/templates/gzip.mustache +++ b/wo/cli/templates/gzip.mustache @@ -12,19 +12,40 @@ gzip_http_version 1.1; gzip_types application/atom+xml + application/geo+json application/javascript application/json + application/ld+json + application/manifest+json + application/rdf+xml application/rss+xml application/vnd.ms-fontobject + application/wasm + application/x-font-opentype + application/x-font-truetype application/x-font-ttf + application/x-javascript application/x-web-app-manifest+json application/xhtml+xml application/xml + application/xml+rss + font/eot font/opentype + font/otf + image/bmp image/svg+xml + image/vnd.microsoft.icon image/x-icon + image/x-win-bitmap + text/cache-manifest + text/calendar text/css + text/javascript + text/markdown text/plain + text/vcard + text/vnd.rim.location.xloc + text/vtt text/x-component - text/xml - text/javascript; \ No newline at end of file + text/x-cross-domain-policy + text/xml; \ No newline at end of file diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index dd1fb4f..615cf7e 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -66,7 +66,7 @@ http { ssl_session_cache shared:SSL:50m; ssl_session_tickets off; ssl_prefer_server_ciphers on; - {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; + {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; ssl_protocols TLSv1.2 TLSv1.3;{{/tls13}} ssl_ecdh_curve X25519:P-521:P-384:P-256; # Previous TLS v1.2 configuration From c32a43dee32949cd32504bc73c5d82ed7fdce4d2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 14 Aug 2019 16:10:46 +0200 Subject: [PATCH 087/102] Update travis --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9a4f97..fa612dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,6 @@ addons: git: quiet: true -cache: - apt: true - before_install: - rm -rf ~/.gnupg From 279112c38aee3875931e82eaa6f279e4a6481943 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 14 Aug 2019 18:42:19 +0200 Subject: [PATCH 088/102] Additional cache exception for Woocommerce --- wo/cli/templates/map-wp.mustache | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 00d21da..0364037 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -19,6 +19,7 @@ map $http_cookie $cookie_no_cache { "~*woocommerce_cart_hash" 1; "~*wptouch_switch_toogle" 1; "~*comment_author_email_" 1; + "~*wptouch_switch_toggle" 1; "~*edd" 1; } @@ -44,6 +45,13 @@ map $request_uri $uri_no_cache { "~*/wc-api/*" 1; "~*/logout/" 1; "~*/lost-password/" 1; + "~*/panier/" 1; + "~*/mon-compte/" 1; + "~*/embed" 1; + "~*/commande/" 1; + "~*/resetpass/" 1; + + } # do not cache requests with query strings From a0101e2760840759cfc41643fd0ddc49e8b6d0ed Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 00:42:49 +0200 Subject: [PATCH 089/102] Few improvements * Fix netdata config files owner * Increase net backlog in sysctl.conf * force wo-kernel tweak update during upgrade --- install | 27 +++++++++++++-------------- wo/cli/plugins/stack_pref.py | 13 ++++++------- wo/cli/templates/map-wp.mustache | 2 -- wo/cli/templates/sysctl.mustache | 4 ++-- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/install b/install index 74ead1f..e565596 100755 --- a/install +++ b/install @@ -182,7 +182,7 @@ wo_install_dep() { } if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies - DEBIAN_FRONTEND=noninteractive apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 + DEBIAN_FRONTEND=noninteractive apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades > /dev/null 2>&1 else # install dependencies DEBIAN_FRONTEND=noninteractive apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 @@ -645,21 +645,20 @@ wo_remove_ee_cron() { wo_tweak_kernel() { - if [ ! -f /etc/sysctl.d/60-wo-tweaks.conf ]; then - if [ "$WO_ARCH" = "x86_64" ]; then - rm -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf - wget -qO /etc/sysctl.d/60-wo-tweaks.conf https://raw.githubusercontent.com/WordOps/WordOps/master/wo/cli/templates/sysctl.mustache - if [ "$wo_distro_version" = "bionic" ] || [ "$wo_distro_version" = "disco" ] || [ "$wo_distro_version" = "buster" ]; then - modprobe tcp_bbr && echo 'tcp_bbr' >> /etc/modules-load.d/bbr.conf - echo -e '\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_notsent_lowat = 16384' >> /etc/sysctl.d/60-wo-tweaks.conf - else - modprobe tcp_htcp && echo 'tcp_htcp' >> /etc/modules-load.d/htcp.conf - echo 'net.ipv4.tcp_congestion_control = htcp' >> /etc/sysctl.d/60-wo-tweaks.conf - fi - # apply sysctl tweaks - sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf + if [ "$WO_ARCH" = "x86_64" ]; then + rm -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf + wget -qO /etc/sysctl.d/60-wo-tweaks.conf https://raw.githubusercontent.com/WordOps/WordOps/"$wo_branch"/wo/cli/templates/sysctl.mustache + if [ "$wo_distro_version" = "bionic" ] || [ "$wo_distro_version" = "disco" ] || [ "$wo_distro_version" = "buster" ]; then + modprobe tcp_bbr && echo 'tcp_bbr' >> /etc/modules-load.d/bbr.conf + echo -e '\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_notsent_lowat = 16384' >> /etc/sysctl.d/60-wo-tweaks.conf + else + modprobe tcp_htcp && echo 'tcp_htcp' >> /etc/modules-load.d/htcp.conf + echo 'net.ipv4.tcp_congestion_control = htcp' >> /etc/sysctl.d/60-wo-tweaks.conf fi + # apply sysctl tweaks + sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf fi + if [ ! -x /opt/wo-kernel.sh ]; then { # download and setup wo-kernel systemd service to apply kernel tweaks for netdata and redis on server startup diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 6cbf504..4284d47 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -126,12 +126,11 @@ def pre_pref(self, apt_packages): # add nginx repository if set(WOVariables.wo_nginx).issubset(set(apt_packages)): + Log.info(self, "Adding repository for NGINX, please wait...") if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for NGINX, please wait...") WORepo.add(self, ppa=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding ppa for Nginx') else: - Log.info(self, "Adding repository for NGINX, please wait...") WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) Log.debug(self, 'Adding repository for Nginx') WORepo.add_key(self, WOVariables.wo_nginx_key) @@ -139,12 +138,11 @@ def pre_pref(self, apt_packages): # add php repository if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or set(WOVariables.wo_php).issubset(set(apt_packages))): + Log.info(self, "Adding repository for PHP, please wait...") if (WOVariables.wo_distro == 'ubuntu'): - Log.info(self, "Adding repository for PHP, please wait...") Log.debug(self, 'Adding ppa for PHP') WORepo.add(self, ppa=WOVariables.wo_php_repo) else: - Log.info(self, "Adding repository for PHP, please wait...") # Add repository for php if (WOVariables.wo_platform_codename == 'buster'): php_pref = ("Package: *\nPin: origin " @@ -163,14 +161,11 @@ def pre_pref(self, apt_packages): if WOVariables.wo_distro == 'ubuntu': Log.debug(self, 'Adding ppa for redis') WORepo.add(self, ppa=WOVariables.wo_redis_repo) - else: - Log.debug(self, 'Adding repo_url of redis for debian') def post_pref(self, apt_packages, packages): """Post activity after installation of packages""" if (apt_packages): - # Nginx configuration if set(WOVariables.wo_nginx).issubset(set(apt_packages)): # Nginx main configuration @@ -1363,6 +1358,10 @@ def post_pref(self, apt_packages, packages): Log.debug(self, "{0}".format(e)) Log.info( self, "fail to setup mysql user for netdata") + WOFileUtils.chown(self, '/opt/netdata', + 'netdata', + 'netdata', + recursive=True) WOService.restart_service(self, 'netdata') # WordOps Dashboard diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index 0364037..dd43421 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -50,8 +50,6 @@ map $request_uri $uri_no_cache { "~*/embed" 1; "~*/commande/" 1; "~*/resetpass/" 1; - - } # do not cache requests with query strings diff --git a/wo/cli/templates/sysctl.mustache b/wo/cli/templates/sysctl.mustache index 271298c..f4e9440 100644 --- a/wo/cli/templates/sysctl.mustache +++ b/wo/cli/templates/sysctl.mustache @@ -196,8 +196,8 @@ net.core.somaxconn = 32768 # Increase number of incoming connections backlog net.core.netdev_max_backlog = 16384 net.core.dev_weight = 64 -net.core.netdev_budget = 600 -net.core.netdev_budget_usecs = 4000 +net.core.netdev_budget = 1200 +net.core.netdev_budget_usecs = 8000 # Increase the maximum amount of option memory buffers net.core.optmem_max = 65535 From 0c4e95985dc45865adf037a2d69648ffe5431625 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 02:29:18 +0200 Subject: [PATCH 090/102] Add ntp and unattended upgrades --- CHANGELOG.md | 2 ++ install | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73d1ea4..366cc7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - APT Packages configuration step with `wo stack upgrade` to apply new configurations - Cloudflare restore real_ip configuration - WP-Rocket support with the flag `--wprocket` +- Install unattended-upgrade and enable automated security updates +- Enable time synchronization with ntp #### Changed diff --git a/install b/install index e565596..e1d628a 100755 --- a/install +++ b/install @@ -175,17 +175,21 @@ fi wo_install_dep() { { - + export DEBIAN_FRONTEND=noninteractive [ -z "$wo_travis" ] && { # update server packages - DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --option=Dpkg::options::=--force-unsafe-io --assume-yes --quiet + apt-get dist-upgrade --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --option=Dpkg::options::=--force-unsafe-io --assume-yes --quiet } if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies - DEBIAN_FRONTEND=noninteractive apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades > /dev/null 2>&1 + apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ + build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz \ + gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 else # install dependencies - DEBIAN_FRONTEND=noninteractive apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 + apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ + build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar \ + software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key @@ -193,6 +197,13 @@ wo_install_dep() { fi locale-gen en + # enable unattended upgades + cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades + # set default ntp pools + sed -e 's/^#NTP=/NTP=time.cloudflare.com 0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf + # enable ntp + timedatectl set-ntp 1 + } >> "$wo_install_log" 2>&1 } From c45ae9755a6943e9b4ee4ef613957d24e659b8e6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 11:08:51 +0200 Subject: [PATCH 091/102] Update wordops-dashboard version --- wo/cli/plugins/stack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 35bbcf5..bf97953 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -335,8 +335,7 @@ class WOStackController(CementBaseController): self, "Setting packages variable for WO-Dashboard") packages = packages + \ [["https://github.com/WordOps/" - "wordops-dashboard/releases/" - "download/v1.0/wo-dashboard.tar.gz", + "wordops-dashboard/archive/master.tar.gz", "/var/lib/wo/tmp/wo-dashboard.tar.gz", "WordOps Dashboard"], ["https://github.com/soerennb/" From d7a2708f4d3890801fa2775537ab1ff7bca3b478 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 11:47:39 +0200 Subject: [PATCH 092/102] Update wordops-dashboard --- wo/cli/plugins/stack.py | 5 +++-- wo/core/variables.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index bf97953..4745ec7 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -334,8 +334,9 @@ class WOStackController(CementBaseController): Log.debug( self, "Setting packages variable for WO-Dashboard") packages = packages + \ - [["https://github.com/WordOps/" - "wordops-dashboard/archive/master.tar.gz", + [["https://github.com/WordOps/wordops-dashboard/" + "releases/download/v{0}/wordops-dashboard.tar.gz" + .format(WOVariables.wo_dashboard), "/var/lib/wo/tmp/wo-dashboard.tar.gz", "WordOps Dashboard"], ["https://github.com/soerennb/" diff --git a/wo/core/variables.py b/wo/core/variables.py index ddefb2a..b738706 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -16,6 +16,7 @@ class WOVariables(): wo_adminer = "4.7.2" wo_phpmyadmin = "4.9.0.1" wo_extplorer = "2.1.13" + wo_dashboard = "1.1" # Get WPCLI path wo_wpcli_path = '/usr/local/bin/wp' From aa10b89b6ba16c7896b38f7aafddda37c22776e6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 12:02:05 +0200 Subject: [PATCH 093/102] Add --wprocket to bash completion --- config/bash_completion.d/wo_auto.rc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index b8facb7..c550992 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -159,13 +159,13 @@ _wo_complete() "create") COMPREPLY=( $(compgen \ - -W "--user --pass --email --html --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --proxy= --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le -le=subdomain -le=wildcard --dns --dns=dns_cf --dns=dns_do" \ + -W "--user --pass --email --html --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --proxy= --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le -le=subdomain -le=wildcard --dns --dns=dns_cf --dns=dns_do" \ -- $cur) ) ;; "update") COMPREPLY=( $(compgen \ - -W "--password --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le -le=subdomain -le=wildcard --dns --dns=dns_cf --dns=dns_do" \ + -W "--password --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le -le=subdomain -le=wildcard --dns --dns=dns_cf --dns=dns_do" \ -- $cur) ) ;; "delete") @@ -211,9 +211,9 @@ _wo_complete() "--wp") if [ ${COMP_WORDS[1]} != "debug" ]; then if [ ${COMP_WORDS[2]} == "create" ]; then - retlist="--wp --wpsc --wpfc --user --email --pass --wpredis --letsencrypt -le --letsencrypt=subdomain --letsencrypt=wildcard --dns --dns=dns_cf --dns=dns_do --php73" + retlist="--wp --wpsc --wpfc --user --email --pass --wpredis --wprocket --letsencrypt -le --letsencrypt=subdomain --letsencrypt=wildcard --dns --dns=dns_cf --dns=dns_do --php73" elif [ ${COMP_WORDS[2]} == "update" ]; then - retlist="--wp --wpfc --wpsc --php73 --php73=off --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le -le=off -le=wildcard --dns --dns=dns_cf --dns=dns_do" + retlist="--wp --wpfc --wpsc --php73 --php73=off --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le -le=off -le=wildcard --dns --dns=dns_cf --dns=dns_do" else retlist="" fi @@ -230,9 +230,9 @@ _wo_complete() "--wpsubdir" | "--wpsubdomain") if [ ${COMP_WORDS[1]} != "debug" ]; then if [ ${COMP_WORDS[2]} == "create" ]; then - retlist="--wpsc --wpfc --user --email --pass --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --php73 --dns --dns=dns_cf --dns=dns_do" + retlist="--wpsc --wpfc --user --email --pass --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --php73 --dns --dns=dns_cf --dns=dns_do" elif [ ${COMP_WORDS[2]} == "update" ]; then - retlist="--wpfc --wpsc --php73 --php73=off --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le --dns --dns=dns_cf --dns=dns_do" + retlist="--wpfc --wpsc --php73 --php73=off --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le --dns --dns=dns_cf --dns=dns_do" else retlist="" fi @@ -246,9 +246,9 @@ _wo_complete() -- $cur) ) ;; - "--wpredis" | "--wpfc" | "--wpsc" | "--wpsubdir" | "--wpsubdomain" | "--user" | "--pass" | "--email" | "--wp") + "--wpredis --wprocket" | "--wpfc" | "--wpsc" | "--wpsubdir" | "--wpsubdomain" | "--user" | "--pass" | "--email" | "--wp") if [ ${COMP_WORDS[2]} == "create" ]; then - retlist="--user --pass --email --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --php73 --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --dns --dns=dns_cf --dns=dns_do" + retlist="--user --pass --email --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --wprocket --php73 --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --dns --dns=dns_cf --dns=dns_do" else retlist="" fi @@ -259,9 +259,9 @@ _wo_complete() -- $cur) ) ;; - "--wpredis" | "--wpfc") + "--wpredis --wprocket" | "--wpfc") if [ ${COMP_WORDS[2]} == "update" ]; then - retlist="--password --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le --dns --dns=dns_cf --dns=dns_do" + retlist="--password --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew --letsencrypt=clean -le --dns --dns=dns_cf --dns=dns_do" else retlist="" fi @@ -314,7 +314,7 @@ _wo_complete() elif [ ${COMP_WORDS[2]} == "delete" ]; then retlist="--db --files --force" elif [ ${COMP_WORDS[2]} == "update" ]; then - retlist="--password --php --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew" + retlist="--password --php --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=off --letsencrypt=renew" else retlist="" fi @@ -363,7 +363,7 @@ _wo_complete() case "$mprev" in "--user" | "--email" | "--pass") if [ ${COMP_WORDS[2]} == "create" ]; then - retlist="--user --pass --email --html --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --dns --dns=dns_cf --dns=dns_do" + retlist="--user --pass --email --html --php --php73 --mysql --wp --wpsubdir --wpsubdomain --wpfc --wpsc --wpredis --wprocket --letsencrypt --letsencrypt=subdomain --letsencrypt=wildcard -le --dns --dns=dns_cf --dns=dns_do" fi ret="${retlist[@]/$prev}" COMPREPLY=( $(compgen \ From cdb195bd03528c2137e5e92642bb8c0eb7d9b191 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 17:19:52 +0200 Subject: [PATCH 094/102] Revert Commit --- CHANGELOG.md | 3 ++ install | 60 ++++++++++++++++++++++++++++++------ setup.py | 9 +++--- wo/cli/plugins/stack_pref.py | 16 ++++++++-- wo/core/variables.py | 2 +- 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366cc7c..df82f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +### v3.9.7.3 - 2019-08-15 + #### Added - Allow web browser caching for json and webmanifest files @@ -17,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - WP-Rocket support with the flag `--wprocket` - Install unattended-upgrade and enable automated security updates - Enable time synchronization with ntp +- Additional cache exception for woocommerce #### Changed diff --git a/install b/install index e1d628a..04fb043 100755 --- a/install +++ b/install @@ -10,7 +10,7 @@ # Version 3.9.7.2 - 2019-08-12 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" -readonly wo_version_new="3.9.7.2" +readonly wo_version_new="3.9.7.3" # CONTENTS # --- # 1. VARIABLES AND DECLARATIONS @@ -124,11 +124,11 @@ readonly wo_install_log=/var/log/wo/install.log readonly wo_linux_distro=$(lsb_release -is) readonly wo_distro_version=$(lsb_release -sc) readonly wo_distro_id=$(lsb_release -rs) -TIME_FORMAT='%d-%b-%Y-%H%M%S' -TIME=$(date +"$TIME_FORMAT") -NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" -EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" -WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" +readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' +readonly TIME=$(date +"$TIME_FORMAT") +readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" +readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" +readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" WO_ARCH="$(uname -m)" if [ -x /usr/local/bin/ee ]; then @@ -183,13 +183,13 @@ wo_install_dep() { if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz \ - gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz \ + gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar \ - software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar \ + software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key @@ -696,6 +696,46 @@ wo_uninstall() { rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates >> /var/log/wo/install.log 2>&1 } +wo_ufw_setup() { + + CURRENT_SSH_PORT=$(grep "Port" /etc/ssh/sshd_config | awk -F " " '{print $2}') + + if [ ! -d /etc/ufw ]; then + apt-get install ufw -y + fi + + # define firewall rules + + ufw logging low + ufw default allow outgoing + ufw default deny incoming + + # default ssh port + ufw allow 22 + + # custom ssh port + if [ "$CURRENT_SSH_PORT" != "22" ]; then + ufw allow "$CURRENT_SSH_PORT" + fi + + # dns + ufw allow 53 + + # nginx + ufw allow http + ufw allow https + + # ntp + ufw allow 123 + + # wordops backend + ufw allow 22222 + + # enable ufw + echo "y" | ufw enable + +} >> $wo_install_log + ### # 4 - WO MAIN SETUP ### diff --git a/setup.py b/setup.py index fd240d0..cdf6c7e 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ import shutil conf = [] templates = [] -long_description = '''WordOps is the commandline tool to manage your - Websites based on WordPress and Nginx with easy to use - commands''' +long_description = '''WordOps An essential toolset that eases WordPress + site and server administration. It provide the ability + to Install a high performance WordPress stack + with a few keystrokes''' for name in glob.glob('config/plugins.d/*.conf'): conf.insert(1, name) @@ -56,7 +57,7 @@ if not os.path.isfile('/root/.gitconfig'): shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') setup(name='wo', - version='3.9.7.2', + version='3.9.7.3', description=long_description, long_description=long_description, classifiers=[], diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 4284d47..9b3a112 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -593,6 +593,19 @@ def post_pref(self, apt_packages, packages): else: WOService.restart_service(self, 'nginx') + # add rule for Nginx with UFW + if WOAptGet.is_installed(self, 'ufw'): + try: + WOShellExec.cmd_exec(self, "/usr/bin/ufw allow " + "http") + WOShellExec.cmd_exec(self, "/usr/bin/ufw allow " + "https") + WOShellExec.cmd_exec(self, "/usr/bin/ufw allow " + "22222") + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to add UFW rule") + # create nginx configuration for redis if set(WOVariables.wo_redis).issubset(set(apt_packages)): if os.path.isdir('/etc/nginx/common'): @@ -1070,7 +1083,6 @@ def post_pref(self, apt_packages, packages): comment='MySQL optimization cronjob ' 'added by WordOps') WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - WOService.restart_service(self, 'mysql') # create fail2ban configuration files if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): @@ -1166,7 +1178,7 @@ def post_pref(self, apt_packages, packages): # add rule for proftpd with UFW if WOAptGet.is_installed(self, 'ufw'): try: - WOShellExec.cmd_exec(self, "ufw allow " + WOShellExec.cmd_exec(self, "/usr/bin/ufw allow " "49000:50000/tcp") except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) diff --git a/wo/core/variables.py b/wo/core/variables.py index b738706..0b0a472 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -10,7 +10,7 @@ class WOVariables(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.7.2" + wo_version = "3.9.7.3" # WordOps packages versions wo_wp_cli = "2.2.0" wo_adminer = "4.7.2" From 25c1d8a19bfe23b72afa5efc5673669feeb50c46 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 19:59:23 +0200 Subject: [PATCH 095/102] Add Cache-Enabler support --- README.md | 6 ++ wo/cli/plugins/site.py | 60 +++++++++--- wo/cli/plugins/site_functions.py | 107 ++------------------- wo/cli/plugins/stack_pref.py | 16 +++ wo/cli/templates/map-wp.mustache | 2 +- wo/cli/templates/virtualconf-php7.mustache | 2 +- wo/cli/templates/virtualconf.mustache | 2 +- wo/cli/templates/wpce.mustache | 19 ++++ wo/cli/templates/wprocket.mustache | 2 +- 9 files changed, 99 insertions(+), 117 deletions(-) create mode 100644 wo/cli/templates/wpce.mustache diff --git a/README.md b/README.md index 2e69c14..89eedde 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,8 @@ wo site create example.com --wp # install wordpress without any wo site create example.com --wp --php73 # install wordpress with PHP 7.3 without any page caching wo site create example.com --wpfc # install wordpress + nginx fastcgi_cache wo site create example.com --wpredis # install wordpress + nginx redis_cache +wo site create example.com --wprocket # install wordpress with WP-Rocket plugin +wo site create example.com --wpce # install wordpress with Cache-enabler plugin wo site create example.com --wpsc # install wordpress with wp-super-cache plugin ``` @@ -97,6 +99,8 @@ wo site create example.com --wpsubdir # install wpmu-subdirectory wit wo site create example.com --wpsubdir --wpsc # install wpmu-subdirectory with wp-super-cache plugin wo site create example.com --wpsubdir --wpfc # install wpmu-subdirectory + nginx fastcgi_cache wo site create example.com --wpsubdir --wpredis # install wpmu-subdirectory + nginx redis_cache +wo site create example.com --wpsubdir --wprocket # install wpmu-subdirectory + WP-Rocket plugin +wo site create example.com --wpsubdir --wpce # install wpmu-subdirectory + Cache-Enabler plugin ``` ### WordPress multisite with subdomain @@ -106,6 +110,8 @@ wo site create example.com --wpsubdomain # install wpmu-subdomain wit wo site create example.com --wpsubdomain --wpsc # install wpmu-subdomain with wp-super-cache plugin wo site create example.com --wpsubdomain --wpfc # install wpmu-subdomain + nginx fastcgi_cache wo site create example.com --wpsubdomain --wpredis # install wpmu-subdomain + nginx redis_cache +wo site create example.com --wpsubdomain --wprocket # install wpmu-subdomain + WP-Rocket plugin +wo site create example.com --wpsubdomain --wpce # install wpmu-subdomain + Cache-Enabler plugin ``` ### Non-WordPress sites diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 41538a0..0f19a81 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -336,6 +336,9 @@ class WOSiteCreateController(CementBaseController): (['--wprocket'], dict(help="create WordPress single/multi site with WP-Rocket", action='store_true')), + (['--wpce'], + dict(help="create WordPress single/multi site with Cache-Enabler", + action='store_true')), (['--wpredis'], dict(help="create WordPress single/multi site " "with redis cache", @@ -426,7 +429,7 @@ class WOSiteCreateController(CementBaseController): if stype == 'proxy': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, wprocket=False, + wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['proxy'] = True @@ -437,7 +440,7 @@ class WOSiteCreateController(CementBaseController): if self.app.pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=False, php73=True, wp=False, - wpfc=False, wpsc=False, wprocket=False, + wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) data['basic'] = True @@ -445,7 +448,7 @@ class WOSiteCreateController(CementBaseController): if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=True, basic=False, php73=False, wp=False, - wpfc=False, wpsc=False, wprocket=False, + wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -457,7 +460,7 @@ class WOSiteCreateController(CementBaseController): data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, wprocket=False, + wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', @@ -487,6 +490,7 @@ class WOSiteCreateController(CementBaseController): if ((not self.app.pargs.wpfc) and (not self.app.pargs.wpsc) and (not self.app.pargs.wprocket) and + (not self.app.pargs.wpce) and (not self.app.pargs.wpredis)): data['basic'] = True @@ -496,7 +500,6 @@ class WOSiteCreateController(CementBaseController): data['basic'] = False self.app.pargs.wpredis = True - # Check rerequired packages are installed or not wo_auth = site_package_check(self, stype) @@ -836,6 +839,8 @@ class WOSiteUpdateController(CementBaseController): dict(help="update to wpsc cache", action='store_true')), (['--wprocket'], dict(help="update to WP-Rocket cache", action='store_true')), + (['--wpce'], + dict(help="update to Cache-Enabler cache", action='store_true')), (['--wpredis'], dict(help="update to redis cache", action='store_true')), (['-le', '--letsencrypt'], @@ -879,7 +884,7 @@ class WOSiteUpdateController(CementBaseController): if not (pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpsubdir or pargs.wpsubdomain or pargs.wpfc or pargs.wpsc or - pargs.wprocket or + pargs.wprocket or pargs.wpce or pargs.wpredis or pargs.letsencrypt or pargs.hsts or pargs.dns or pargs.force): Log.error(self, "Please provide options to update sites.") @@ -958,7 +963,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.password and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or - pargs.wprocket or + pargs.wprocket or pargs.wpce or pargs.wpsubdir or pargs.wpsubdomain or pargs.hsts)): try: @@ -971,7 +976,7 @@ class WOSiteUpdateController(CementBaseController): if (pargs.hsts and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or pargs.wp or pargs.wpfc or pargs.wpsc or - pargs.wprocket or + pargs.wprocket or parge.wpce or pargs.wpsubdir or pargs.wpsubdomain or pargs.password)): try: @@ -1013,16 +1018,16 @@ class WOSiteUpdateController(CementBaseController): if stype == 'php': data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, wprocket=False, multisite=False, - wpsubdir=False, webroot=wo_site_webroot, + wpsc=False, wpredis=False, wprocket=False, wpce=False, + multisite=False, wpsubdir=False, webroot=wo_site_webroot, currsitetype=oldsitetype, currcachetype=oldcachetype) elif stype in ['mysql', 'wp', 'wpsubdir', 'wpsubdomain']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=True, wp=False, wpfc=False, - wpsc=False, wpredis=False, wprocket=False, multisite=False, - wpsubdir=False, webroot=wo_site_webroot, + wpsc=False, wpredis=False, wprocket=False, wpce=False, + multisite=False,wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', wo_db_host='', currsitetype=oldsitetype, currcachetype=oldcachetype) @@ -1076,30 +1081,42 @@ class WOSiteUpdateController(CementBaseController): data['wpsc'] = False data['wpredis'] = False data['wprocket'] = False + data['wpce'] = False elif oldcachetype == 'wpfc': data['basic'] = False data['wpfc'] = True data['wpsc'] = False data['wpredis'] = False data['wprocket'] = False + data['wpce'] = False elif oldcachetype == 'wpsc': data['basic'] = False data['wpfc'] = False data['wpsc'] = True data['wpredis'] = False data['wprocket'] = False + data['wpce'] = False elif oldcachetype == 'wpredis': data['basic'] = False data['wpfc'] = False data['wpsc'] = False data['wpredis'] = True data['wprocket'] = False + data['wpce'] = False elif oldcachetype == 'wprocket': data['basic'] = False data['wpfc'] = False data['wpsc'] = False data['wpredis'] = False data['wprocket'] = True + data['wpce'] = False + elif oldcachetype == 'wpce': + data['basic'] = False + data['wpfc'] = False + data['wpsc'] = False + data['wpredis'] = False + data['wprocket'] = False + data['wpce'] = True if pargs.php73 == 'on': data['php73'] = True @@ -1262,6 +1279,11 @@ class WOSiteUpdateController(CementBaseController): data['basic'] = False cache = 'wprocket' + if pargs.wpce and data['currcachetype'] != 'wpce': + data['wpce'] = True + data['basic'] = False + cache = 'wpce' + if (php73 is old_php73) and (stype == oldsitetype and cache == oldcachetype): return 1 @@ -1502,7 +1524,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - if ((oldcachetype in ['wpsc', 'basic', 'wpredis', 'wprocket'] and + if ((oldcachetype in ['wpsc', 'basic', 'wpredis', 'wprocket', 'wpce'] and (data['wpfc'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpfc'])): try: @@ -1541,7 +1563,7 @@ class WOSiteUpdateController(CementBaseController): "and please try again") return 1 - elif ((oldcachetype in ['wpsc', 'basic', 'wpfc', 'wprocket'] and + elif ((oldcachetype in ['wpsc', 'basic', 'wpfc', 'wprocket', 'wpce'] and (data['wpredis'])) or (oldsitetype == 'wp' and data['multisite'] and data['wpredis'])): @@ -1659,6 +1681,16 @@ class WOSiteUpdateController(CementBaseController): "`tail /var/log/wo/wordops.log` and please try again") return 1 + if oldcachetype == 'wpce' and not data['wpce']: + try: + uninstallwp_plugin(self, 'cache-enabler', data) + except SiteError as e: + Log.debug(self, str(e)) + Log.info(self, Log.FAIL + "Update site failed." + "Check the log for details: " + "`tail /var/log/wo/wordops.log` and please try again") + return 1 + # Service Nginx Reload if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 4cd1d27..376e867 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -16,6 +16,7 @@ from wo.core.git import WOGit from wo.core.logging import Log from wo.core.mysql import * from wo.core.services import WOService +from wo.cli.plugins.stack_pref import pre_pref, post_pref from wo.core.shellexec import CommandExecutionError, WOShellExec from wo.core.sslutils import SSL from wo.core.variables import WOVariables @@ -584,6 +585,10 @@ def setupwordpress(self, data): if data['wpredis']: installwp_plugin(self, 'redis-cache', data) + """Install Cache-Enabler""" + if data['wpce']: + installwp_plugin(self, 'cache-enabler', data) + wp_creds = dict(wp_user=wo_wp_user, wp_pass=wo_wp_pass, wp_email=wo_wp_email) @@ -785,7 +790,7 @@ def site_package_check(self, stype): Log.info(self, "NGINX PLUS Detected ...") apt = ["nginx-plus"] + WOVariables.wo_nginx # apt_packages = apt_packages + WOVariables.wo_nginx - stack.post_pref(apt, packages) + stack.post_pref(self, apt, packages) elif WOAptGet.is_installed(self, 'nginx'): Log.info(self, "WordOps detected a previously" "installed Nginx package. " @@ -794,7 +799,7 @@ def site_package_check(self, stype): "https://github.com/WordOps/WordOps/issues/ \n") apt = ["nginx"] + WOVariables.wo_nginx # apt_packages = apt_packages + WOVariables.wo_nginx - stack.post_pref(apt, packages) + post_pref(self, apt, packages) else: apt_packages = apt_packages + WOVariables.wo_nginx else: @@ -853,39 +858,6 @@ def site_package_check(self, stype): if not WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + WOVariables.wo_redis - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/common/redis-php72.conf")): - - data = dict(upstream="php72") - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}") - - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d/redis.conf")): - with open("/etc/nginx/conf.d/redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis '$remote_addr" - " $upstream_response_time " - "$srcache_fetch_status [$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" \"$http_user_agent\"';\n") - if self.app.pargs.php73: Log.debug(self, "Setting apt_packages variable for PHP 7.3") if not WOAptGet.is_installed(self, 'php7.3-fpm'): @@ -895,72 +867,9 @@ def site_package_check(self, stype): else: apt_packages = apt_packages + WOVariables.wo_php73 - if (os.path.isdir("/etc/nginx/common") and - not os.path.isfile("/etc/nginx/common/locations-wo.conf")): - data = dict(upstream="php73") - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/php73.conf') - wo_nginx = open('/etc/nginx/common/php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'php.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpcommon-php73.conf') - wo_nginx = open('/etc/nginx/common/wpcommon-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpcommon.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpfc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpfc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpfc.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wpsc-php73.conf') - wo_nginx = open('/etc/nginx/common/wpsc-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wpsc.mustache', - out=wo_nginx) - wo_nginx.close() - - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/wprocket-php73.conf') - wo_nginx = open('/etc/nginx/common/wprocket-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'wprocket.mustache', - out=wo_nginx) - wo_nginx.close() - - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/common/redis-php73.conf")): - data = dict(upstream="php73") - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - return(stack.install(apt_packages=apt_packages, packages=packages, disp_msg=False)) - def updatewpuserpassword(self, wo_domain, wo_site_webroot): wo_wp_user = '' @@ -1098,7 +1007,7 @@ def detSitePar(opts): if val and key in ['html', 'php', 'mysql', 'wp', 'wpsubdir', 'wpsubdomain', 'php73']: typelist.append(key) - elif val and key in ['wpfc', 'wpsc', 'wpredis', 'wprocket']: + elif val and key in ['wpfc', 'wpsc', 'wpredis', 'wprocket', 'wpce']: cachelist.append(key) if len(typelist) > 1 or len(cachelist) > 1: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 9b3a112..257fa62 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -334,6 +334,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpce-php72.conf') + wo_nginx = open('/etc/nginx/common/wpce-php72.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpce.mustache', + out=wo_nginx) + wo_nginx.close() + # PHP 7.3 conf if os.path.isdir("/etc/nginx/common"): data = dict(upstream="php73") @@ -378,6 +386,14 @@ def post_pref(self, apt_packages, packages): out=wo_nginx) wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/wpce-php73.conf') + wo_nginx = open('/etc/nginx/common/wpce-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'wpce.mustache', + out=wo_nginx) + wo_nginx.close() + Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') wo_nginx = open('/etc/nginx/common/redis-php73.conf', diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index dd43421..efb3060 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -64,7 +64,7 @@ map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cac 0000 0; } -# map $skip_cache with $cache_uri for --wpsc stack +# map $skip_cache with $cache_uri for --wpsc --wpce & --wprocket stack map $skip_cache $cache_uri { 0 $request_uri; default 'null cache'; diff --git a/wo/cli/templates/virtualconf-php7.mustache b/wo/cli/templates/virtualconf-php7.mustache index fa137cb..09a99bd 100644 --- a/wo/cli/templates/virtualconf-php7.mustache +++ b/wo/cli/templates/virtualconf-php7.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf;{{/wprocket}} + {{^static}}include {{#basic}}common/php73.conf;{{/basic}}{{#wpfc}}common/wpfc-php73.conf;{{/wpfc}} {{#wpsc}}common/wpsc-php73.conf;{{/wpsc}}{{#wpredis}}common/redis-php73.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php73.conf;{{/wprocket}}{{#wpce}}common/wpce-php73.conf;{{/wpce}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php73.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/virtualconf.mustache b/wo/cli/templates/virtualconf.mustache index 19a1afb..098d6cc 100644 --- a/wo/cli/templates/virtualconf.mustache +++ b/wo/cli/templates/virtualconf.mustache @@ -49,7 +49,7 @@ server { } {{/static}} - {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf;{{/wprocket}} + {{^static}}include {{#basic}}common/php72.conf;{{/basic}}{{#wpfc}}common/wpfc-php72.conf;{{/wpfc}}{{#wpsc}}common/wpsc-php72.conf;{{/wpsc}}{{#wpredis}}common/redis-php72.conf;{{/wpredis}}{{#wprocket}}common/wprocket-php72.conf;{{/wprocket}}{{#wpce}}common/wpce-php72.conf;{{/wpce}} {{#wpsubdir}}include common/wpsubdir.conf;{{/wpsubdir}}{{/static}} {{#wp}}include common/wpcommon-php72.conf;{{/wp}} include common/locations-wo.conf;{{/proxy}} diff --git a/wo/cli/templates/wpce.mustache b/wo/cli/templates/wpce.mustache new file mode 100644 index 0000000..e345a2b --- /dev/null +++ b/wo/cli/templates/wpce.mustache @@ -0,0 +1,19 @@ +# WPCE NGINX CONFIGURATION - WO v3.9.7 +# DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE +# $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf +# Use cached or actual file if they exists, Otherwise pass request to WordPress +location / { + try_files /wp-content/cache/cache-enabler/${http_host}${cache_uri}index.html $uri $uri/ /index.php$is_args$args; +} +location ~ \.php$ { + try_files $uri =404; + include fastcgi_params; + fastcgi_pass {{upstream}}; +} +location ~* /cache/cache-enabler/\.html$ { + etag on; + add_header Vary "Accept-Encoding, Cookie"; + access_log off; + log_not_found off; + expires 10h; +} diff --git a/wo/cli/templates/wprocket.mustache b/wo/cli/templates/wprocket.mustache index 14e3600..8e01d9d 100644 --- a/wo/cli/templates/wprocket.mustache +++ b/wo/cli/templates/wprocket.mustache @@ -1,4 +1,4 @@ -# WPSC NGINX CONFIGURATION - WO v3.9.7 +# WPROCKET NGINX CONFIGURATION - WO v3.9.7 # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # $cache_uri variable set in /etc/nginx/conf.d/map-wp.conf # Use cached or actual file if they exists, Otherwise pass request to WordPress From 0bf84fdb99cc9b764b78fd047beca1bba449eebc Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 20:00:00 +0200 Subject: [PATCH 096/102] add wpce to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fa612dd..74c8087 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ script: - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc - sudo wo site create wpsc-php73.net --wpsc --php73 && sudo wo site create wpfc-php73.net --wpfc --php73 - sudo wo site create wprocket.net --wprocket && sudo wo site create wprocket-php73.net --wprocket --php73 + - sudo wo site create wpce.net --wpce && sudo wo site create wpce-php73.net --wpce --php73 - sudo wo site create wpredis.net --wpredis && sudo wo site create wpredis-php73.net --wpredis --php73 - sudo wo site create wpsubdir1.com --wpsubdir && sudo wo site create wpsubdir-php73.com --wpsubdir --php73 From 840e353a54a558cddb45db660104c57b7cb3c214 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 15 Aug 2019 23:46:16 +0200 Subject: [PATCH 097/102] Add --mysqlclient and cleanup code --- wo/cli/plugins/stack.py | 20 +++--- wo/cli/plugins/stack_pref.py | 103 +++++++++++---------------- wo/cli/templates/nginx-core.mustache | 6 ++ wo/core/variables.py | 47 +++++------- 4 files changed, 73 insertions(+), 103 deletions(-) diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 4745ec7..cf95508 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -63,6 +63,9 @@ class WOStackController(CementBaseController): dict(help='Install PHP 7.3 stack', action='store_true')), (['--mysql'], dict(help='Install MySQL stack', action='store_true')), + (['--mysqlclient'], + dict(help='Install MySQL client for remote MySQL server', + action='store_true')), (['--wpcli'], dict(help='Install WPCLI stack', action='store_true')), (['--phpmyadmin'], @@ -107,12 +110,10 @@ class WOStackController(CementBaseController): if ((not pargs.web) and (not pargs.admin) and (not pargs.nginx) and (not pargs.php) and (not pargs.mysql) and (not pargs.wpcli) and - (not pargs.phpmyadmin) and - (not pargs.composer) and - (not pargs.netdata) and - (not pargs.dashboard) and - (not pargs.fail2ban) and - (not pargs.security) and + (not pargs.phpmyadmin) and (not pargs.composer) and + (not pargs.netdata) and (not pargs.dashboard) and + (not pargs.fail2ban) and (not pargs.security) + and (not pargs.mysqlclient) and (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.phpredisadmin) and @@ -218,9 +219,10 @@ class WOStackController(CementBaseController): "/usr/bin/mysqltuner", "MySQLTuner"]] - else: - Log.debug(self, "MySQL connection is already alive") - Log.info(self, "MySQL connection is already alive") + if pargs.mysqlclient: + Log.debug(self, "Setting apt_packages variable " + "for MySQL Client") + apt_packages = apt_packages + WOVariables.wo_mysql_client # WP-CLI if pargs.wpcli: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 257fa62..4e8cb00 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -25,7 +25,8 @@ from wo.core.variables import WOVariables def pre_pref(self, apt_packages): """Pre settings to do before installation packages""" - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): + if (set(WOVariables.wo_mysql).issubset(set(apt_packages)) or + set(WOVariables.wo_mysql_client).issubset(set(apt_packages))): # add mariadb repository excepted on raspbian and ubuntu 19.04 if (not WOVariables.wo_distro == 'raspbian'): Log.info(self, "Adding repository for MySQL, please wait...") @@ -42,71 +43,47 @@ def pre_pref(self, apt_packages): keyserver="keyserver.ubuntu.com") WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver="keyserver.ubuntu.com") + if set(WOVariables.wo_mysql).issubset(set(apt_packages)): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) # configure MySQL non-interactive install if (not WOVariables.wo_distro == 'raspbian'): - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") + mariadb_ver = '10.3' else: - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") + mariadb_ver = '10.1' - Log.debug(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.1 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") + Log.debug(self, "Pre-seeding MySQL") + Log.debug(self, "echo \"mariadb-server-{0} " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections" + .format(mariadb_ver)) + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(mariadb_ver, chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") + + Log.debug(self, "echo \"mariadb-server-{0} " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections" + .format(mariadb_ver)) + try: + WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(mariadb_ver, chars=chars), + log=False) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error("Failed to initialize MySQL package") # generate my.cnf root credentials mysql_config = """ [client] @@ -695,7 +672,7 @@ def post_pref(self, apt_packages, packages): # Parse /etc/php/7.2/fpm/php-fpm.conf data = dict(pid="/run/php/php7.2-fpm.pid", error_log="/var/log/php/7.2/fpm.log", - include="/etc/php/7.2/fpm/pool.d/*.conf") + include="/etc/php/7.2/fpm/pool.d/*.conf") Log.debug(self, "writting php7.2 configuration into " "/etc/php/7.2/fpm/php-fpm.conf") wo_php_fpm = open('/etc/php/7.2/fpm/php-fpm.conf', @@ -858,8 +835,8 @@ def post_pref(self, apt_packages, packages): # Parse /etc/php/7.3/fpm/php-fpm.conf data = dict(pid="/run/php/php7.3-fpm.pid", - error_log="/var/log/php7.3-fpm.log", - include="/etc/php/7.3/fpm/pool.d/*.conf") + error_log="/var/log/php7.3-fpm.log", + include="/etc/php/7.3/fpm/pool.d/*.conf") Log.debug(self, "writting php 7.3 configuration into " "/etc/php/7.3/fpm/php-fpm.conf") wo_php_fpm = open('/etc/php/7.3/fpm/php-fpm.conf', diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index 615cf7e..be1ea21 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -40,6 +40,12 @@ http { reset_timedout_connection on; more_set_headers "X-Powered-By : WordOps"; + open_file_cache max=50000 inactive=60s; + open_file_cache_errors off; + open_file_cache_min_uses 2; + open_file_cache_valid 120s; + open_log_file_cache max=10000 inactive=30s min_uses=2; + # Limit Request limit_req_status 403; limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s; diff --git a/wo/core/variables.py b/wo/core/variables.py index 0b0a472..ed9ce1b 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -110,39 +110,26 @@ class WOVariables(): # PHP repo and packages if wo_distro == 'ubuntu': wo_php_repo = "ppa:ondrej/php" - wo_php = ["php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap", - "php7.2-readline", "php7.2-common", "php7.2-recode", - "php7.2-cli", "php7.2-mbstring", - "php7.2-bcmath", "php7.2-mysql", "php7.2-opcache", - "php7.2-zip", "php7.2-xml", "php7.2-soap"] - wo_php73 = ["php7.3-fpm", "php7.3-curl", "php7.3-gd", "php7.3-imap", - "php7.3-readline", "php7.3-common", "php7.3-recode", - "php7.3-cli", "php7.3-mbstring", - "php7.3-bcmath", "php7.3-mysql", "php7.3-opcache", - "php7.3-zip", "php7.3-xml", "php7.3-soap"] - wo_php_extra = ["php-memcached", "php-imagick", - "graphviz", "php-xdebug", "php-msgpack", "php-redis"] wo_php_key = '' else: wo_php_repo = ( "deb https://packages.sury.org/php/ {codename} main" .format(codename=wo_platform_codename)) - wo_php = ["php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap", - "php7.2-readline", "php7.2-common", "php7.2-recode", - "php7.2-cli", "php7.2-mbstring", - "php7.2-bcmath", "php7.2-mysql", "php7.2-opcache", - "php7.2-zip", "php7.2-xml", "php7.2-soap"] - wo_php73 = ["php7.3-fpm", "php7.3-curl", "php7.3-gd", "php7.3-imap", - "php7.3-readline", "php7.3-common", "php7.3-recode", - "php7.3-cli", "php7.3-mbstring", - "php7.3-bcmath", "php7.3-mysql", "php7.3-opcache", - "php7.3-zip", "php7.3-xml", "php7.3-soap"] - wo_php_extra = ["php-memcached", "php-imagick", - "graphviz", "php-xdebug", "php-msgpack", - "php-redis", "php-mysql"] - wo_php_key = 'AC0E47584A7A714D' + wo_php = ["php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap", + "php7.2-readline", "php7.2-common", "php7.2-recode", + "php7.2-cli", "php7.2-mbstring", + "php7.2-bcmath", "php7.2-mysql", "php7.2-opcache", + "php7.2-zip", "php7.2-xml", "php7.2-soap"] + wo_php73 = ["php7.3-fpm", "php7.3-curl", "php7.3-gd", "php7.3-imap", + "php7.3-readline", "php7.3-common", "php7.3-recode", + "php7.3-cli", "php7.3-mbstring", + "php7.3-bcmath", "php7.3-mysql", "php7.3-opcache", + "php7.3-zip", "php7.3-xml", "php7.3-soap"] + wo_php_extra = ["php-memcached", "php-imagick", + "graphviz", "php-xdebug", "php-msgpack", "php-redis"] + # MySQL repo and packages if wo_distro == 'ubuntu': wo_mysql_repo = ("deb [arch=amd64,ppc64el] " @@ -155,11 +142,9 @@ class WOVariables(): "10.3/debian {codename} main" .format(codename=wo_platform_codename)) - if wo_platform_codename == 'jessie': - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb"] + wo_mysql = ["mariadb-server", "percona-toolkit", "python3-mysqldb"] + + wo_mysql_client = ["mariadb-client", "python3-mysqldb"] wo_fail2ban = ["fail2ban"] From 3e339984061fc67a96ab50212ded0c223515d22e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Aug 2019 00:03:43 +0200 Subject: [PATCH 098/102] Refactor nginx upgrade process --- CHANGELOG.md | 1 + install | 65 +++++++++++++++++++--------------------------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df82f8c..6f20a75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Cleanup templates by removing all doublons (with/without php7) and replacing them with variables - Updated Nginx to v1.16.1 in response to HTTP/2 vulnerabilites discovered - Disable temporary adding swap feature (not working) +- `wo stack upgrade --nginx` is now able to apply new configurations during `wo update`, it highly reduce upgrade duration #### Fixed diff --git a/install b/install index 04fb043..615bba0 100755 --- a/install +++ b/install @@ -465,58 +465,41 @@ wo_upgrade_nginx() { fi # chec if the package nginx-ee is installed CHECK_NGINX_EE=$(dpkg --list | grep nginx-ee) - CHECK_NGINX_WO=$(dpkg --list | grep nginx-wo) CHECK_PHP72=$(command -v php-fpm7.2) # add new Nginx repository - if [ "$wo_linux_distro" = "Ubuntu" ]; then - if [ ! -f /etc/apt/sources.list.d/wordops-ubuntu-nginx-wo-"$(lsb_release -sc)".list ]; then - add-apt-repository ppa:wordops/nginx-wo -y -u - fi + if [ "$wo_distro_version" == "jessie" ]; then + # import the respository key for updates + curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_8.0/Release.key | apt-key add - else - if [ "$wo_distro_version" == "jessie" ]; then - # import the respository key for updates - curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_8.0/Release.key | apt-key add - - else - curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_9.0/Release.key | apt-key add - - fi - + curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_9.0/Release.key | apt-key add - fi - # stop nginx - service nginx stop - # install new nginx package - if [ -n "$CHECK_NGINX_EE" ] || [ -n "$CHECK_NGINX_WO" ]; then + if [ -n "$CHECK_NGINX_EE" ]; then if [ -x /usr/local/bin/wo ]; then - if [ -n "$CHECK_NGINX_EE" ]; then - # remove previous package - apt-mark unhold nginx-ee nginx-common nginx-custom - apt-get --assume-yes purge nginx-ee nginx-common nginx-custom --allow-change-held-packages - # remove previous php-fpm pool configuration - if [ -n "$CHECK_PHP72" ]; then - apt-get purge php7.2-fpm -y -qq - rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} - fi - elif [ -n "$CHECK_NGINX_WO" ]; then - apt-mark unhold nginx-wo nginx-common nginx-custom - apt-get --assume-yes purge nginx-wo nginx-common nginx-custom --allow-change-held-packages - if [ -n "$CHECK_PHP72" ]; then - apt-get purge php7.2-fpm -y -qq - rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} - fi + # stop nginx + service nginx stop + # remove previous package + apt-mark unhold nginx-ee nginx-common nginx-custom + apt-get --assume-yes purge nginx-ee nginx-common nginx-custom --allow-change-held-packages + # remove previous php-fpm pool configuration + if [ -n "$CHECK_PHP72" ]; then + apt-get purge php7.2-fpm -y -qq + rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} fi if [ -d /etc/nginx ]; then rm -rf /etc/nginx fi - /usr/local/bin/wo stack install --nginx --php - rm -f /etc/nginx/common/acl.conf /etc/nginx/conf.d/{map-wp-cache.conf,map-wp.conf,fascgi.conf,blockips.conf} /etc/nginx/htpasswd-wo - fi - fi + rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo + /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ + fi + else + /usr/local/bin/wo stack upgrade --nginx + fi # restore sites and configuration - /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ [ -f /etc/nginx/htpasswd-ee ] && { mv /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* @@ -772,11 +755,9 @@ else fi if [ -z "$wo_preserve_config" ]; then if [ -n "$(command -v nginx)" ]; then - if [ ! -f /etc/nginx/common/release ] || ! grep -q "v3.9.7" /etc/nginx/common/release; then - if [ ! -f /etc/apt/preferences.d/nginx-block ]; then - wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log - wo_upgrade_nginx | tee -ai $wo_install_log - fi + if [ ! -f /etc/apt/preferences.d/nginx-block ]; then + wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log + wo_upgrade_nginx | tee -ai $wo_install_log fi fi fi From 7cb223b02a79162d3c18224b114f1fd3577c98c0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Aug 2019 00:15:34 +0200 Subject: [PATCH 099/102] Fix missing --force and cleanup code --- install | 2 +- wo/cli/plugins/stack.py | 33 +++++++++++++++++---------------- wo/cli/plugins/stack_upgrade.py | 31 +++++++++++-------------------- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/install b/install index 615bba0..aca2a05 100755 --- a/install +++ b/install @@ -497,7 +497,7 @@ wo_upgrade_nginx() { fi else - /usr/local/bin/wo stack upgrade --nginx + /usr/local/bin/wo stack upgrade --nginx --force fi # restore sites and configuration [ -f /etc/nginx/htpasswd-ee ] && { mv /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index cf95508..3320a6d 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -104,6 +104,7 @@ class WOStackController(CementBaseController): def install(self, packages=[], apt_packages=[], disp_msg=True): """Start installation of packages""" self.msg = [] + empty_packages = [] pargs = self.app.pargs try: # Default action for stack installation @@ -395,25 +396,25 @@ class WOStackController(CementBaseController): except Exception as e: Log.debug(self, "{0}".format(e)) - if (apt_packages) or (packages): + if (apt_packages): Log.debug(self, "Calling pre_pref") pre_pref(self, apt_packages) - if (apt_packages): - # meminfo = (os.popen('/bin/cat /proc/meminfo ' - # '| grep MemTotal').read()).split(":") - # memsplit = re.split(" kB", meminfo[1]) - # wo_mem = int(memsplit[0]) - # if (wo_mem < 4000000): - # WOSwap.add(self) - Log.info(self, "Updating apt-cache, please wait...") - WOAptGet.update(self) - Log.info(self, "Installing packages, please wait...") - WOAptGet.install(self, apt_packages) - if (packages): - Log.debug(self, "Downloading following: {0}".format(packages)) - WODownload.download(self, packages) + # meminfo = (os.popen('/bin/cat /proc/meminfo ' + # '| grep MemTotal').read()).split(":") + # memsplit = re.split(" kB", meminfo[1]) + # wo_mem = int(memsplit[0]) + # if (wo_mem < 4000000): + # WOSwap.add(self) + Log.info(self, "Updating apt-cache, please wait...") + WOAptGet.update(self) + Log.info(self, "Installing packages, please wait...") + WOAptGet.install(self, apt_packages) + post_pref(self, apt_packages, empty_packages) + if (packages): + Log.debug(self, "Downloading following: {0}".format(packages)) + WODownload.download(self, packages) Log.debug(self, "Calling post_pref") - post_pref(self, apt_packages, packages) + post_pref(self, empty_packages, packages) if disp_msg: if (self.msg): diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 5b84e22..35f7bf0 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -49,6 +49,9 @@ class WOStackUpgradeController(CementBaseController): (['--no-prompt'], dict(help="Upgrade Packages without any prompt", action='store_true')), + (['--force'], + dict(help="Force Packages upgrade without any prompt", + action='store_true')), ] @expose(hide=True) @@ -157,17 +160,15 @@ class WOStackUpgradeController(CementBaseController): Log.error(self, "Composer isn't installed") if len(packages) or len(apt_packages): - - Log.info(self, "During package update process non nginx-cached" - " parts of your site may remain down") - # Check prompt - if (not pargs.no_prompt): - start_upgrade = input("Do you want to continue:[y/N]") - if start_upgrade != "Y" and start_upgrade != "y": - Log.error(self, "Not starting package update") - - Log.info(self, "Updating packages, please wait...") if len(apt_packages): + Log.info(self, "Your site may be down for few seconds if " + "you are upgrading Nginx, PHP-FPM, MariaDB or Redis") + # Check prompt + if ((not pargs.no_prompt) and (not pargs.force)): + start_upgrade = input("Do you want to continue:[y/N]") + if start_upgrade != "Y" and start_upgrade != "y": + Log.error(self, "Not starting package update") + Log.info(self, "Updating APT packages, please wait...") # apt-get update WOAptGet.update(self) if set(WOVariables.wo_php).issubset(set(apt_packages)): @@ -180,16 +181,6 @@ class WOStackUpgradeController(CementBaseController): WOAptGet.install(self, apt_packages) post_pref(self, apt_packages, empty_packages) # Post Actions after package updates - if (set(WOVariables.wo_nginx).issubset(set(apt_packages))): - WOService.restart_service(self, 'nginx') - if set(WOVariables.wo_php).issubset(set(apt_packages)): - WOService.restart_service(self, 'php7.2-fpm') - if set(WOVariables.wo_php73).issubset(set(apt_packages)): - WOService.restart_service(self, 'php7.3-fpm') - if set(WOVariables.wo_mysql).issubset(set(apt_packages)): - WOService.restart_service(self, 'mysql') - if set(WOVariables.wo_redis).issubset(set(apt_packages)): - WOService.restart_service(self, 'redis-server') if len(packages): if pargs.wpcli: From 62b4e839cc8cf28230f4d66f79f7b34f9c118590 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Aug 2019 00:19:48 +0200 Subject: [PATCH 100/102] Update changelog --- CHANGELOG.md | 3 ++- wo/cli/plugins/stack_pref.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f20a75..f1bf59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - nginx-core.mustache template used to render nginx.conf during stack setup - APT Packages configuration step with `wo stack upgrade` to apply new configurations - Cloudflare restore real_ip configuration -- WP-Rocket support with the flag `--wprocket` +- WP-Rocket plugin support with the flag `--wprocket` +- Cache-Enabler plugin support with the flag `--wpce` - Install unattended-upgrade and enable automated security updates - Enable time synchronization with ntp - Additional cache exception for woocommerce diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 4e8cb00..adccc55 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -380,7 +380,7 @@ def post_pref(self, apt_packages, packages): wo_nginx.close() with open("/etc/nginx/common/release", - "a") as release_file: + "w") as release_file: release_file.write("v{0}" .format(WOVariables.wo_version)) release_file.close() From d19f6714f35057ca4ba49f40c10069a7cf8d92ef Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Aug 2019 00:20:59 +0200 Subject: [PATCH 101/102] Prepare for release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1bf59b..ecbddff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] -### v3.9.7.3 - 2019-08-15 +### v3.9.7.3 - 2019-08-16 #### Added From a2d1481d27731f0c264cac1ac0394617353dcfe7 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Aug 2019 00:26:08 +0200 Subject: [PATCH 102/102] Fix two error in site_functions --- wo/cli/plugins/site_functions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 376e867..a3d110d 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -870,6 +870,7 @@ def site_package_check(self, stype): return(stack.install(apt_packages=apt_packages, packages=packages, disp_msg=False)) + def updatewpuserpassword(self, wo_domain, wo_site_webroot): wo_wp_user = '' @@ -1225,10 +1226,11 @@ def site_url_https(self, domain): Log.info(self, "Checking if site url already " "use https, please wait...") WOFileUtils.chdir(self, '{0}/htdocs/'.format(wo_site_webroot)) - wo_siteurl = WOShellExec.cmd_exec_stdout(self, - "php {0} option get siteurl " - .format(WOVariables.wo_wpcli_path) + - "--allow-root --quiet") + wo_siteurl = \ + WOShellExec.cmd_exec_stdout(self, + "php {0} option get siteurl " + .format(WOVariables.wo_wpcli_path) + + "--allow-root --quiet") test_url = re.split(":", wo_siteurl) if not (test_url[0] == 'https'): try: