From 42e856173f7e7213287e8be5a372ba39a2c995ca Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 26 Aug 2019 18:05:26 +0200 Subject: [PATCH] Several new features - cht.sh stack : linux online cheatsheet. Usage : `cheat `. Example for tar : `cheat tar` - ClamAV anti-virus with weekly cronjob to update signatures database - Internal function to add daily cronjobs - Additional comment to detect previous configuration tuning (MariaDB & Redis) - Domain/Subdomain detection based on public domain suffixes list - Increase Nginx & MariaDB systemd open_files limits - Cronjob to update Cloudflare IPs list --- CHANGELOG.md | 16 ++ install | 40 ++++- wo/cli/plugins/site.py | 226 +++++++++++++++------------- wo/cli/plugins/site_functions.py | 6 +- wo/cli/plugins/stack.py | 53 ++++++- wo/cli/plugins/stack_pref.py | 122 ++++++++++----- wo/cli/plugins/stack_upgrade.py | 2 +- wo/cli/templates/cf-update.sh | 23 +++ wo/cli/templates/freshclam.mustache | 11 ++ wo/core/cron.py | 14 ++ wo/core/domainvalidate.py | 27 ++++ wo/core/services.py | 24 ++- 12 files changed, 397 insertions(+), 167 deletions(-) create mode 100644 wo/cli/templates/cf-update.sh create mode 100644 wo/cli/templates/freshclam.mustache diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c5a67..d610049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- cht.sh stack : linux online cheatsheet. Usage : `cheat `. Example for tar : `cheat tar` +- ClamAV anti-virus with weekly cronjob to update signatures database +- Internal function to add daily cronjobs +- Additional comment to detect previous configuration tuning (MariaDB & Redis) +- Domain/Subdomain detection based on public domain suffixes list +- Increase Nginx & MariaDB systemd open_files limits +- Cronjob to update Cloudflare IPs list + +#### Changed + +- eXplorer filemanager isn't installed with WordOps dashboard anymore, and a flag `--extplorer` is available. But it's still installed when running the command `wo stack install` +- Template rendering function now check for a .custom file before overwriting a configuration by default. +- flag `--letsencrypt=subdomain` is not required anymore, you can use `--letsencrypt` or `-le` + ### v3.9.8.3 - 2019-08-21 #### Changed diff --git a/install b/install index 6ae2bd7..a13716b 100755 --- a/install +++ b/install @@ -494,7 +494,7 @@ wo_upgrade_nginx() { # install new nginx package if [ -n "$CHECK_NGINX_EE" ]; then if [ -x /usr/local/bin/wo ]; then - [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block .;} + [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } # stop nginx service nginx stop # remove previous package @@ -545,7 +545,7 @@ wo_upgrade_nginx() { systemctl stop nginx systemctl start nginx fi - [ -f ./nginx-block ] && { mv nginx-block /etc/apt/preferences.d/nginx-block ;} + [ -f /var/lib/wo/tmp/nginx-block ] && { mv /var/lib/wo/tmp/nginx-block /etc/apt/preferences.d/nginx-block; } } \ >> "$wo_install_log" 2>&1 @@ -655,7 +655,6 @@ wo_tweak_kernel() { # apply sysctl tweaks sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf fi - } wo_systemd_tweak() { @@ -678,6 +677,37 @@ wo_systemd_tweak() { } +wo_domain_suffix() { + curl -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat +} + +wo_mariadb_tweak() { + # increase mariadb open_files_limit + { + if [ -d /etc/systemd/system/mariadb.service.d ] && [ ! -f /etc/systemd/system/mariadb.service.d/limits.conf ]; then + echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/mariadb.service.d/limits.conf + systemctl daemon-reload + service mysql restart + fi + } >> /var/log/wo/install.log 2>&1 +} + +wo_nginx_tweak() { + # increase nginx open_files_limit + { + if [ -x /usr/sbin/nginx ]; then + if [ ! -d /etc/systemd/system/nginx.service.d ]; then + mkdir -p /etc/systemd/system/nginx.service.d + if [ ! -f /etc/systemd/system/nginx.service.d/limits.conf ]; then + echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/nginx.service.d/limits.conf + systemctl daemon-reload + nginx -t && service nginx restart + fi + fi + fi + } >> /var/log/wo/install.log 2>&1 +} + wo_clean() { rm -rf /usr/local/lib/python3.*/dist-packages/wo-* } @@ -794,6 +824,8 @@ else wo_lib_echo "Adding systemd service tweak" | tee -ai $wo_install_log wo_systemd_tweak | tee -ai $wo_install_log fi + wo_nginx_tweak | tee -ai $wo_install_log + wo_mysql_tweak | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log wo_update_wp_cli | tee -ai $wo_install_log else @@ -836,6 +868,8 @@ else wo_lib_echo "Adding systemd service tweak" | tee -ai $wo_install_log wo_systemd_tweak | tee -ai $wo_install_log fi + wo_nginx_tweak | tee -ai $wo_install_log + wo_mysql_tweak | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log wo_git_init | tee -ai $wo_install_log wo_update_wp_cli | tee -ai $wo_install_log diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 8455856..52188c3 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -4,7 +4,7 @@ from cement.core import handler, hook from wo.core.sslutils import SSL from wo.core.variables import WOVariables from wo.core.shellexec import WOShellExec -from wo.core.domainvalidate import ValidateDomain +from wo.core.domainvalidate import ValidateDomain, GetDomainlevel from wo.core.fileutils import WOFileUtils from wo.cli.plugins.site_functions import * from wo.core.services import WOService @@ -45,18 +45,19 @@ class WOSiteController(CementBaseController): @expose(help="Enable site example.com") def enable(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'could not input site name') - self.app.pargs.site_name = self.app.pargs.site_name.strip() + pargs.site_name = pargs.site_name.strip() # validate domain name - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) # check if site exists if not check_domain_exists(self, wo_domain): @@ -83,17 +84,18 @@ class WOSiteController(CementBaseController): @expose(help="Disable site example.com") def disable(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'could not input site name') - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) # check if site exists if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -124,16 +126,18 @@ class WOSiteController(CementBaseController): @expose(help="Get example.com information") def info(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'could not input site name') - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + wo_domain_type = GetDomainlevel(wo_domain) wo_db_name = '' wo_db_user = '' wo_db_pass = '' @@ -143,7 +147,6 @@ class WOSiteController(CementBaseController): if os.path.isfile('/etc/nginx/sites-available/{0}' .format(wo_domain)): siteinfo = getSiteInfo(self, wo_domain) - sitetype = siteinfo.site_type cachetype = siteinfo.cache_type wo_site_webroot = siteinfo.site_path @@ -179,8 +182,9 @@ class WOSiteController(CementBaseController): @expose(help="Monitor example.com logs") def log(self): - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs = self.app.pargs + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) wo_site_webroot = getSiteInfo(self, wo_domain).site_path if not check_domain_exists(self, wo_domain): @@ -191,17 +195,18 @@ class WOSiteController(CementBaseController): @expose(help="Display Nginx configuration of example.com") def show(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'could not input site name') # TODO Write code for wo site edit command here - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -221,17 +226,18 @@ class WOSiteController(CementBaseController): @expose(help="Change directory to site webroot") def cd(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'Unable to read input, please try again') - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -261,17 +267,18 @@ class WOSiteEditController(CementBaseController): @expose(hide=True) def default(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, 'Unable to read input, Please try again') - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) @@ -381,43 +388,43 @@ class WOSiteCreateController(CementBaseController): def default(self): pargs = self.app.pargs if pargs.php72: - self.app.pargs.php = True + pargs.php = True # self.app.render((data), 'default.mustache') # Check domain name validation data = dict() host, port = None, None try: - stype, cache = detSitePar(vars(self.app.pargs)) + stype, cache = detSitePar(vars(pargs)) except RuntimeError as e: Log.debug(self, str(e)) Log.error(self, "Please provide valid options to creating site") - if stype is None and self.app.pargs.proxy: + if stype is None and pargs.proxy: stype, cache = 'proxy', '' - proxyinfo = self.app.pargs.proxy[0].strip() + proxyinfo = pargs.proxy[0].strip() if not proxyinfo: Log.error(self, "Please provide proxy server host information") proxyinfo = proxyinfo.split(':') host = proxyinfo[0].strip() port = '80' if len(proxyinfo) < 2 else proxyinfo[1].strip() - elif stype is None and not self.app.pargs.proxy: + elif stype is None and not pargs.proxy: stype, cache = 'html', 'basic' - elif stype and self.app.pargs.proxy: + elif stype and pargs.proxy: Log.error(self, "proxy should not be used with other site types") - if not self.app.pargs.site_name: + if not pargs.site_name: try: - while not self.app.pargs.site_name: + while not pargs.site_name: # preprocessing before finalize site name - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.debug(self, str(e)) Log.error(self, "Unable to input site name, Please try again!") - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) - + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + wo_domain_type = GetDomainlevel(wo_domain) if not wo_domain.strip(): Log.error("Invalid domain name, " "Provide valid domain name") @@ -442,7 +449,7 @@ class WOSiteCreateController(CementBaseController): data['port'] = port data['basic'] = True - if self.app.pargs.php73: + if pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, static=False, basic=False, php73=True, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, @@ -475,9 +482,9 @@ class WOSiteCreateController(CementBaseController): data['wp'] = True data['basic'] = False data[cache] = True - data['wp-user'] = self.app.pargs.user - data['wp-email'] = self.app.pargs.email - data['wp-pass'] = self.app.pargs.wppass + data['wp-user'] = pargs.user + data['wp-email'] = pargs.email + data['wp-pass'] = pargs.wppass if stype in ['wpsubdir', 'wpsubdomain']: data['multisite'] = True if stype == 'wpsubdir': @@ -485,25 +492,25 @@ class WOSiteCreateController(CementBaseController): else: pass - if data and self.app.pargs.php73: + if data and pargs.php73: data['php73'] = True php73 = 1 elif data: data['php73'] = False php73 = 0 - if ((not self.app.pargs.wpfc) and - (not self.app.pargs.wpsc) and - (not self.app.pargs.wprocket) and - (not self.app.pargs.wpce) and - (not self.app.pargs.wpredis)): + if ((not pargs.wpfc) and + (not pargs.wpsc) and + (not pargs.wprocket) and + (not pargs.wpce) and + (not pargs.wpredis)): data['basic'] = True if (cache == 'wpredis'): cache = 'wpredis' data['wpredis'] = True data['basic'] = False - self.app.pargs.wpredis = True + pargs.wpredis = True # Check rerequired packages are installed or not wo_auth = site_package_check(self, stype) @@ -619,7 +626,7 @@ class WOSiteCreateController(CementBaseController): "and please try again") # Setup WordPress if Wordpress site - if (data['wp'] and (not self.app.pargs.vhostonly)): + if (data['wp'] and (not pargs.vhostonly)): try: wo_wp_creds = setupwordpress(self, data) # Add database information for site into database @@ -644,7 +651,7 @@ class WOSiteCreateController(CementBaseController): "`tail /var/log/wo/wordops.log` " "and please try again") - if (data['wp'] and (self.app.pargs.vhostonly)): + if (data['wp'] and (pargs.vhostonly)): try: wo_wp_creds = setupwordpress(self, data) # Add database information for site into database @@ -745,7 +752,7 @@ class WOSiteCreateController(CementBaseController): for msg in wo_auth: Log.info(self, Log.ENDC + msg, log=False) - if data['wp'] and (not self.app.pargs.vhostonly): + if data['wp'] and (not pargs.vhostonly): Log.info(self, Log.ENDC + "WordPress admin user :" " {0}".format(wo_wp_creds['wp_user']), log=False) Log.info(self, Log.ENDC + "WordPress admin user password : {0}" @@ -759,32 +766,32 @@ class WOSiteCreateController(CementBaseController): Log.error(self, "Check the log for details: " "`tail /var/log/wo/wordops.log` and please try again") - if self.app.pargs.letsencrypt: + if pargs.letsencrypt: data['letsencrypt'] = True letsencrypt = True - if self.app.pargs.dns: + if pargs.dns: wo_acme_dns = pargs.dns if data['letsencrypt'] is True: - if self.app.pargs.letsencrypt == "subdomain": - if self.app.pargs.dns: + if pargs.letsencrypt == "subdomain": + if pargs.dns: setupLetsEncrypt(self, wo_domain, True, False, True, wo_acme_dns) else: setupLetsEncrypt(self, wo_domain, True) httpsRedirect(self, wo_domain) - elif self.app.pargs.letsencrypt == "wildcard": + elif pargs.letsencrypt == "wildcard": setupLetsEncrypt(self, wo_domain, False, True, True, wo_acme_dns) httpsRedirect(self, wo_domain, True, True) else: - if self.app.pargs.dns: + if pargs.dns: setupLetsEncrypt(self, wo_domain, False, False, True, wo_acme_dns) else: setupLetsEncrypt(self, wo_domain) httpsRedirect(self, wo_domain) - if self.app.pargs.hsts: + if pargs.hsts: setupHsts(self, wo_domain) site_url_https(self, wo_domain) @@ -915,6 +922,7 @@ class WOSiteUpdateController(CementBaseController): self.doupdatesite(pargs) def doupdatesite(self, pargs): + pargs = self.app.pargs letsencrypt = False php73 = None @@ -950,7 +958,7 @@ class WOSiteUpdateController(CementBaseController): (wo_domain, wo_www_domain, ) = ValidateDomain(pargs.site_name) wo_site_webroot = WOVariables.wo_webroot + wo_domain - + wo_domain_type = GetDomainlevel(wo_domain) check_site = getSiteInfo(self, wo_domain) if check_site is None: @@ -1342,27 +1350,30 @@ class WOSiteUpdateController(CementBaseController): " http://{0}".format(wo_domain)) return 0 - if self.app.pargs.letsencrypt: - if self.app.pargs.dns: + if pargs.letsencrypt: + if pargs.dns: wo_acme_dns = pargs.dns if data['letsencrypt'] is True: + if (wo_domain_type == 'subdomain' and + pargs.letsencrypt != 'wildcard'): + pargs.letsencrypt == 'subdomain' if not os.path.isfile("{0}/conf/nginx/ssl.conf.disabled" .format(wo_site_webroot)): - if self.app.pargs.letsencrypt == "on": - if self.app.pargs.dns: + if pargs.letsencrypt == "on": + if pargs.dns: setupLetsEncrypt(self, wo_domain, False, False, True, wo_acme_dns) else: setupLetsEncrypt(self, wo_domain) httpsRedirect(self, wo_domain) - elif self.app.pargs.letsencrypt == "subdomain": - if self.app.pargs.dns: + elif pargs.letsencrypt == "subdomain": + if pargs.dns: setupLetsEncrypt(self, wo_domain, True, False, True, wo_acme_dns) else: setupLetsEncrypt(self, wo_domain, True) httpsRedirect(self, wo_domain) - elif self.app.pargs.letsencrypt == "wildcard": + elif pargs.letsencrypt == "wildcard": setupLetsEncrypt(self, wo_domain, False, True, True, wo_acme_dns) httpsRedirect(self, wo_domain, True, True) @@ -1391,7 +1402,7 @@ class WOSiteUpdateController(CementBaseController): ".PLEASE renew soon . ") elif data['letsencrypt'] is False: - if self.app.pargs.letsencrypt == "off": + if pargs.letsencrypt == "off": if os.path.isfile("{0}/conf/nginx/ssl.conf" .format(wo_site_webroot)): Log.info(self, 'Setting Nginx configuration') @@ -1407,8 +1418,8 @@ class WOSiteUpdateController(CementBaseController): '{0}/conf/nginx/' 'hsts.conf.disabled' .format(wo_site_webroot)) - elif (self.app.pargs.letsencrypt == "clean" or - self.app.pargs.letsencrypt == "purge"): + elif (pargs.letsencrypt == "clean" or + pargs.letsencrypt == "purge"): removeAcmeConf(self, wo_domain) if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " @@ -1798,16 +1809,18 @@ class WOSiteDeleteController(CementBaseController): @expose(help="Delete website configuration and files") @expose(hide=True) def default(self): - if not self.app.pargs.site_name: + pargs = self.app.pargs + if not pargs.site_name: try: - while not self.app.pargs.site_name: - self.app.pargs.site_name = (input('Enter site name : ') - .strip()) + while not pargs.site_name: + pargs.site_name = (input('Enter site name : ') + .strip()) except IOError as e: Log.error(self, 'could not input site name') - self.app.pargs.site_name = self.app.pargs.site_name.strip() - (wo_domain, wo_www_domain) = ValidateDomain(self.app.pargs.site_name) + pargs.site_name = pargs.site_name.strip() + (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) + wo_domain_type = GetDomainlevel(wo_domain) wo_db_name = '' wo_prompt = '' wo_nginx_prompt = '' @@ -1818,9 +1831,9 @@ class WOSiteDeleteController(CementBaseController): if not check_domain_exists(self, wo_domain): Log.error(self, "site {0} does not exist".format(wo_domain)) - if ((not self.app.pargs.db) and (not self.app.pargs.files) and - (not self.app.pargs.all)): - self.app.pargs.all = True + if ((not pargs.db) and (not pargs.files) and + (not pargs.all)): + pargs.all = True # Gather information from wo-db for wo_domain check_site = getSiteInfo(self, wo_domain) @@ -1834,18 +1847,18 @@ class WOSiteDeleteController(CementBaseController): wo_mysql_grant_host = self.app.config.get('mysql', 'grant-host') if wo_db_name == 'deleted': mark_db_deleted = True - if self.app.pargs.all: - self.app.pargs.db = True - self.app.pargs.files = True + if pargs.all: + pargs.db = True + pargs.files = True else: - if self.app.pargs.all: + if pargs.all: mark_db_deleted = True - self.app.pargs.files = True + pargs.files = True # Delete website database - if self.app.pargs.db: + if pargs.db: if wo_db_name != 'deleted' and wo_db_name != '': - if not self.app.pargs.no_prompt: + if not pargs.no_prompt: wo_db_prompt = input('Are you sure, you want to delete' ' database [y/N]: ') else: @@ -1870,9 +1883,9 @@ class WOSiteDeleteController(CementBaseController): ) # Delete webroot - if self.app.pargs.files: + if pargs.files: if wo_site_webroot != 'deleted': - if not self.app.pargs.no_prompt: + if not pargs.no_prompt: wo_web_prompt = input('Are you sure, you want to delete ' 'webroot [y/N]: ') else: @@ -1891,7 +1904,7 @@ class WOSiteDeleteController(CementBaseController): mark_webroot_deleted = True Log.info(self, "Webroot seems to be already deleted") - if not self.app.pargs.force: + if not pargs.force: if (mark_webroot_deleted and mark_db_deleted): # TODO Delete nginx conf removeNginxConf(self, wo_domain) @@ -1925,15 +1938,16 @@ class WOSiteListController(CementBaseController): @expose(help="Lists websites") def default(self): + pargs = self.app.pargs sites = getAllsites(self) if not sites: pass - if self.app.pargs.enabled: + if pargs.enabled: for site in sites: if site.is_enabled: Log.info(self, "{0}".format(site.sitename)) - elif self.app.pargs.disabled: + elif pargs.disabled: for site in sites: if not site.is_enabled: Log.info(self, "{0}".format(site.sitename)) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 7b30f21..54774f6 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1279,8 +1279,10 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', if os.path.isfile('/etc/nginx/sites-available/{0}' .format(domain)): removeNginxConf(self, domain) - if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' - .format(domain)): + if (os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' + .format(domain)) or + os.path.isdir('/etc/letsencrypt/live/{0}' + .format(domain))): removeAcmeConf(self, domain) if webroot: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 3ddc4ee..07b5a1c 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -87,8 +87,12 @@ class WOStackController(CementBaseController): dict(help='Install Adminer stack', action='store_true')), (['--fail2ban'], dict(help='Install Fail2ban stack', action='store_true')), + (['--clamav'], + dict(help='Install ClamAV stack', action='store_true')), (['--utils'], dict(help='Install Utils stack', action='store_true')), + (['--cheat'], + dict(help='Install cht.sh stack', action='store_true')), (['--redis'], dict(help='Install Redis', action='store_true')), (['--phpredisadmin'], @@ -124,11 +128,12 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.mariabackup) and + (not pargs.cheat) and (not pargs.clamav) and (not pargs.phpredisadmin) and (not pargs.php73)): pargs.web = True pargs.admin = True - pargs.security = True + pargs.fail2ban = True if pargs.all: pargs.web = True @@ -151,9 +156,11 @@ class WOStackController(CementBaseController): pargs.netdata = True pargs.dashboard = True pargs.phpredisadmin = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True + pargs.clamav = True # Redis if pargs.redis: @@ -220,11 +227,13 @@ class WOStackController(CementBaseController): if not WOShellExec.cmd_exec(self, "mysqladmin ping"): apt_packages = apt_packages + WOVariables.wo_mysql + # mysqlclient if pargs.mysqlclient: Log.debug(self, "Setting apt_packages variable " "for MySQL Client") apt_packages = apt_packages + WOVariables.wo_mysql_client + # mariabackup if pargs.mariabackup: if not WOAptGet.is_installed(self, 'mariadb-backup'): Log.debug(self, "Setting apt_packages variable " @@ -254,6 +263,15 @@ class WOStackController(CementBaseController): Log.debug(self, "Fail2ban already installed") Log.info(self, "Fail2ban already installed") + # ClamAV + if pargs.clamav: + Log.debug(self, "Setting apt_packages variable for ClamAV") + if not WOAptGet.is_installed(self, 'fail2ban'): + apt_packages = apt_packages + ["clamav"] + else: + Log.debug(self, "Fail2ban already installed") + Log.info(self, "Fail2ban already installed") + # proftpd if pargs.proftpd: Log.debug(self, "Setting apt_packages variable for ProFTPd") @@ -324,7 +342,7 @@ class WOStackController(CementBaseController): "htdocs/db/adminer/adminer.css" .format(WOVariables.wo_webroot), "Adminer theme"]] - + # mysqltuner if pargs.mysqltuner: Log.debug(self, "Setting packages variable for MySQLTuner ") packages = packages + [["https://raw." @@ -356,16 +374,24 @@ class WOStackController(CementBaseController): "releases/download/v{0}/wordops-dashboard.tar.gz" .format(WOVariables.wo_dashboard), "/var/lib/wo/tmp/wo-dashboard.tar.gz", - "WordOps Dashboard"], - ["https://github.com/soerennb/" - "extplorer/archive/v{0}.tar.gz" - .format(WOVariables.wo_extplorer), - "/var/lib/wo/tmp/extplorer.tar.gz", - "eXtplorer"]] + "WordOps Dashboard"]] else: Log.debug(self, "WordOps dashboard already installed") Log.info(self, "WordOps dashboard already installed") + if pargs.dashboard: + if not os.path.isdir('/var/www/22222/htdocs/files'): + Log.debug(self, "Setting packages variable for eXtplorer") + packages = packages + \ + [["https://github.com/soerennb/" + "extplorer/archive/v{0}.tar.gz" + .format(WOVariables.wo_extplorer), + "/var/lib/wo/tmp/extplorer.tar.gz", + "eXtplorer"]] + else: + Log.debug(self, "eXtplorer is already installed") + Log.info(self, "eXtplorer is already installed") + # UTILS if pargs.utils: Log.debug(self, "Setting packages variable for utils") @@ -408,6 +434,17 @@ class WOStackController(CementBaseController): '/var/lib/wo/tmp/anemometer.tar.gz', 'Anemometer'] ] + if pargs.cheat: + if (not os.path.isfile('/usr/local/bin/cht.sh') and + not os.path.isfile('/usr/bin/cht.sh')): + Log.debug(self, "Setting packages variable for cht.sh") + [["https://cht.sh/:cht.sh", + "/usr/local/bin/cht.sh", + "Cht.sh"]] + else: + Log.debug(self, "cht.sh is already installed") + Log.info(self, "cht.sh is already installed") + except Exception as e: Log.debug(self, "{0}".format(e)) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index cc2899e..f4d48b0 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -202,6 +202,15 @@ def post_pref(self, apt_packages, packages): WOTemplate.tmpl_render(self, '{0}/cloudflare.conf'.format(ngxcnf), 'cloudflare.mustache', data) + Log.debug("Setting up freshclam cronjob") + WOTemplate.tmpl_render(self, '/opt/cf-update.sh', + 'cf-update.mustache', + data, overwrite=False) + WOFileUtils.chmod(self, "/opt/cf-update.sh", 0o775) + WOCron.setcron_weekly(self, '/opt/cf-update.sh ' + '> /dev/null 2>&1', + comment='Cloudflare IP refresh cronjob ' + 'added by WordOps') WOTemplate.tmpl_render(self, '{0}/map-wp-fastcgi-cache.conf'.format( @@ -471,7 +480,7 @@ def post_pref(self, apt_packages, packages): WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") WOService.reload_service(self, 'nginx') - server_ip = requests.get('http://v4.wordops.eu') + if set(["nginx"]).issubset(set(apt_packages)): print("WordOps backend configuration was successful\n" "You can access it on : https://{0}:22222" @@ -657,8 +666,8 @@ def post_pref(self, apt_packages, packages): WOFileUtils.chown(self, "{0}22222/htdocs" .format(ngxroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + 'www-data', + 'www-data', recursive=True) WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.2-fpm') @@ -829,8 +838,8 @@ def post_pref(self, apt_packages, packages): WOFileUtils.chown(self, "{0}22222/htdocs" .format(ngxroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, recursive=True) + 'www-data', + 'www-data', recursive=True) WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") WOService.restart_service(self, 'php7.3-fpm') @@ -845,17 +854,34 @@ def post_pref(self, apt_packages, packages): encoding='utf-8', mode='w') config_file.write(config) config_file.close() - else: + elif (not WOFileUtils.grep(self, "/etc/mysql/my.cnf", "WordOps")): + with open("/etc/mysql/my.cnf", + "a") as mysql_file: + mysql_file.write("\n# WordOps v3.9.8\n") wo_ram = psutil.virtual_memory().total / (1024 * 1024) + # set InnoDB variable depending on the RAM available wo_ram_innodb = int(wo_ram*0.3) wo_ram_log_buffer = int(wo_ram_innodb*0.25) wo_ram_log_size = int(wo_ram_log_buffer*0.5) # replacing default values Log.debug(self, "Tuning MySQL configuration") + # set innodb_buffer_pool_instances depending + # on the amount of RAM + if (wo_ram_innodb > 1000) and (wo_ram_innodb < 64000): + wo_innodb_instance = int( + wo_ram_innodb/1000) + elif (wo_ram_innodb < 1000): + wo_innodb_instance = int(1) + elif (wo_ram_innodb > 64000): + wo_innodb_instance = int(64) WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "innodb_buffer_pool_size = 256M", - "innodb_buffer_pool_size = {0}M" - .format(wo_ram_innodb)) + "innodb_buffer_pool_size " + "= {0}M\n" + "innodb_buffer_pool_instances " + "= {1}\n" + .format(wo_ram_innodb, + wo_innodb_instance)) WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "innodb_log_buffer_size = 8M", "innodb_log_buffer_size = {0}M" @@ -870,11 +896,6 @@ def post_pref(self, apt_packages, packages): "= 600", "wait_timeout " "= 120") - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "skip-external-locking", - "skip-external-locking\n" - "skip-name-resolve = 1\n") - # disabling mariadb binlog WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", @@ -924,15 +945,8 @@ def post_pref(self, apt_packages, packages): "table_open_cache = 16000") WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", "max_allowed_packet = 16M", - "max_allowed_packet = 64M") - if (wo_ram_innodb > 1000) and (wo_ram_innodb < 64000): - wo_innodb_instance = int(wo_ram_innodb/1000) - WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", - "# * Security Features", - "innodb_buffer_pool_instances " - "= {0}\n" - .format(wo_innodb_instance) + - "# * Security Features") + "max_allowed_packet = 64M\n" + "skip-name-resolve=1\n") WOService.stop_service(self, 'mysql') WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', @@ -1083,7 +1097,11 @@ def post_pref(self, apt_packages, packages): # enable systemd service Log.debug(self, "Enabling redis systemd service") WOShellExec.cmd_exec(self, "systemctl enable redis-server") - if os.path.isfile("/etc/redis/redis.conf"): + if (os.path.isfile("/etc/redis/redis.conf") and + not WOFileUtils.grep(self, "/etc/mysql/my.cnf", "WordOps")): + with open("/etc/redis/redis.conf", + "a") as redis_file: + redis_file.write("\n# WordOps v3.9.8\n") wo_ram = psutil.virtual_memory().total / (1024 * 1024) if wo_ram < 1024: Log.debug(self, "Setting maxmemory variable to " @@ -1127,6 +1145,18 @@ def post_pref(self, apt_packages, packages): 'redis', 'redis', recursive=False) WOService.restart_service(self, 'redis-server') + # Redis configuration + if set(["clamav"]).issubset(set(apt_packages)): + Log.debug("Setting up freshclam cronjob") + WOTemplate.tmpl_render(self, '/opt/freshclam.sh', + 'freshclam.mustache', + data, overwrite=False) + WOFileUtils.chmod(self, "/opt/freshclam.sh", 0o775) + WOCron.setcron_weekly(self, '/opt/freshclam.sh ' + '> /dev/null 2>&1', + comment='ClamAV freshclam cronjob ' + 'added by WordOps') + if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): Log.debug(self, "Setting Privileges" @@ -1190,8 +1220,8 @@ def post_pref(self, apt_packages, packages): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) # composer install and phpmyadmin update @@ -1211,8 +1241,8 @@ def post_pref(self, apt_packages, packages): "/var/www/22222/htdocs/db/pma/") WOFileUtils.chown(self, '{0}22222/htdocs/db/pma' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) if any('/usr/bin/mysqltuner' == x[1] @@ -1291,8 +1321,8 @@ def post_pref(self, apt_packages, packages): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) # Extplorer FileManager @@ -1314,8 +1344,8 @@ def post_pref(self, apt_packages, packages): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) # webgrind @@ -1359,8 +1389,8 @@ def post_pref(self, apt_packages, packages): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) # anemometer if any('/var/lib/wo/tmp/anemometer.tar.gz' == x[1] @@ -1419,10 +1449,20 @@ def post_pref(self, apt_packages, packages): out=wo_anemometer) wo_anemometer.close() + # pt-query-advisor if any('/usr/bin/pt-query-advisor' == x[1] for x in packages): WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + # cht.sh + if any('/usr/local/bin/cht.sh' == x[1] + for x in packages): + WOFileUtils.chmod(self, "/usr/local/bin/cht.sh", 0o775) + if not WOFileUtils.grep(self, "~/.bashrc", "cheat"): + with open("~/.bashrc", + "a") as wo_bashrc: + wo_bashrc.write("\nalias cheat='cht.sh'\n") + # phpredisadmin if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] for x in packages): @@ -1436,21 +1476,21 @@ def post_pref(self, apt_packages, packages): .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) if os.path.isfile("/usr/local/bin/composer"): - WOShellExec.cmd_exec(self, "sudo -u www-data -H " - "composer " - "create-project -n -s dev " + WOShellExec.cmd_exec(self, "/usr/local/bin/composer" + "create-project --no-plugins " + "--no-scripts -n -s dev " "erik-dubbelboer/php-redis-admin " "/var/www/22222/htdocs/cache" "/redis/phpRedisAdmin ") Log.debug(self, 'Setting Privileges of webroot permission to ' - '{0}22222/htdocs/cache/file ' + '{0}22222/htdocs/cache/redis' .format(WOVariables.wo_webroot)) WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, + 'www-data', + 'www-data', recursive=True) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index b46858c..7a2566a 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -64,7 +64,7 @@ class WOStackUpgradeController(CementBaseController): packages = [] nginx_packages = [] empty_packages = [] - pargs = pargs = self.app.pargs + pargs = self.app.pargs if ((not pargs.web) and (not pargs.nginx) and (not pargs.php) and (not pargs.php73) and diff --git a/wo/cli/templates/cf-update.sh b/wo/cli/templates/cf-update.sh new file mode 100644 index 0000000..562b451 --- /dev/null +++ b/wo/cli/templates/cf-update.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# WordOps bash script to update cloudflare IP + +CURL_BIN=$(command -v curl) +CF_IPV4=$($CURL_BIN -sL https://www.cloudflare.com/ips-v4) +CF_IPV6=$($CURL_BIN -sL https://www.cloudflare.com/ips-v6) + +echo -e '# WordOps (wo) set visitors real ip with Cloudflare\n' > /etc/nginx/conf.d/cloudflare.conf +echo "####################################" +echo "Adding Cloudflare IPv4" +echo "####################################" +for cf_ip4 in $CF_IPV4; do + echo "set_real_ip_from $cf_ip4;" >> /etc/nginx/conf.d/cloudflare.conf +done +echo "####################################" +echo "Adding Cloudflare IPv6" +echo "####################################" +for cf_ip6 in $CF_IPV6; do + echo "set_real_ip_from $cf_ip6;" >> /etc/nginx/conf.d/cloudflare.conf +done +echo 'real_ip_header CF-Connecting-IP;' >> /etc/nginx/conf.d/cloudflare.conf + +nginx -t && service nginx reload \ No newline at end of file diff --git a/wo/cli/templates/freshclam.mustache b/wo/cli/templates/freshclam.mustache new file mode 100644 index 0000000..e8bbd34 --- /dev/null +++ b/wo/cli/templates/freshclam.mustache @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# WordOps ClamAV freshclam script +# script path after installation /opt/freshcham.sh + +if [ -x /etc/init.d/clamav-freshclam ]; then +{ + /etc/init.d/clamav-freshclam stop + freshclam + /etc/init.d/clamav-freshclam start +} >> /var/log/wo/clamav.log 2>&1 +fi diff --git a/wo/core/cron.py b/wo/core/cron.py index 02e9e81..fd4ec70 100644 --- a/wo/core/cron.py +++ b/wo/core/cron.py @@ -21,6 +21,20 @@ class WOCron(): "\\\"; } | crontab -\"") Log.debug(self, "Cron set") + def setcron_daily(self, cmd, comment='Cron set by WordOps', user='root', + min=0, hour=12): + if not WOShellExec.cmd_exec(self, "crontab -l " + "| grep -q \'{0}\'".format(cmd)): + + WOShellExec.cmd_exec(self, "/bin/bash -c \"crontab -l " + "2> /dev/null | {{ cat; echo -e" + " \\\"" + "\\n@daily" + "{0}".format(cmd) + + " # {0}".format(comment) + + "\\\"; } | crontab -\"") + Log.debug(self, "Cron set") + def remove_cron(self, cmd): if WOShellExec.cmd_exec(self, "crontab -l " "| grep -q \'{0}\'".format(cmd)): diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index f0c5985..750622e 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -1,5 +1,6 @@ """WordOps domain validation module.""" from urllib.parse import urlparse +import os def ValidateDomain(url): @@ -22,3 +23,29 @@ def ValidateDomain(url): final_domain = domain_name return (final_domain, domain_name) + + +def GetDomainlevel(domain): + """ + This function returns the domain type : domain, subdomain, + """ + domain_name = domain.split('.') + if domain_name[0] == 'www': + domain_name = domain_name[1:] + if os.path.isfile("/var/lib/wo/public_suffix_list.dat"): + # Read mode opens a file for reading only. + Suffix_file = open( + "/var/lib/wo/public_suffix_list.dat", "r") + # Read all the lines into a list. + for domain_suffix in Suffix_file: + if (str(domain_suffix).strip()) == ('.'.join(testing_domain[1:])): + domain_type = 'domain' + break + elif (str(domain_suffix).strip()) == ('.'.join(testing_domain[2:]): + domain_type='subdomain' + break + else: + domain_type='other' + Suffix_file.close() + + return (domain_type) diff --git a/wo/core/services.py b/wo/core/services.py index e47aca1..bd545ea 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -17,8 +17,12 @@ class WOService(): """ try: if service_name in ['nginx', 'php5-fpm']: - service_cmd = ('{0} -t && service {0} start' - .format(service_name)) + # Check Nginx configuration before executing command + sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + output, error_output = sub.communicate() + if 'emerg' not in str(error_output): + service_cmd = ('service {0} start'.format(service_name)) else: service_cmd = ('service {0} start'.format(service_name)) @@ -64,8 +68,12 @@ class WOService(): """ try: if service_name in ['nginx', 'php5-fpm']: - service_cmd = ('{0} -t && service {0} restart' - .format(service_name)) + # Check Nginx configuration before executing command + sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + output, error_output = sub.communicate() + if 'emerg' not in str(error_output): + service_cmd = ('service {0} restart'.format(service_name)) else: service_cmd = ('service {0} restart'.format(service_name)) @@ -90,8 +98,12 @@ class WOService(): """ try: if service_name in ['nginx', 'php5-fpm']: - service_cmd = ('{0} -t && service {0} reload' - .format(service_name)) + # Check Nginx configuration before executing command + sub = subprocess.Popen('nginx -t', stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + output, error_output = sub.communicate() + if 'emerg' not in str(error_output): + service_cmd = ('service {0} restart'.format(service_name)) else: service_cmd = ('service {0} reload'.format(service_name))