diff --git a/.gitignore b/.gitignore index 601587c..b79b87e 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ local/ man/ \.pytest_cache/ + +\.vscode/ + +\.noseids diff --git a/.travis.yml b/.travis.yml index 82851da..3356f34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,20 @@ +sudo: required +dist: xenial + language: bash notifications: slack: wordops:MyZBNbI7JfhbAi3YyFckMdaa -dist: xenial +addons: + apt: + update: true + +git: + quiet: true + +cache: + apt: true before_install: - rm -rf ~/.gnupg @@ -13,12 +24,6 @@ before_script: - sudo bash -c 'echo example.com > /etc/hostname' - sudo apt-get -qq purge mysql* graphviz* - sudo apt-get -qq autoremove --purge -addons: - apt: - update: true - -git: - quiet: true script: - lsb_release -a @@ -28,32 +33,20 @@ script: - sudo apt-get install -y --force-yes git python3-setuptools python3-dev python3-apt ccze tree - sudo bash install -b $TRAVIS_BRANCH --travis - sudo wo --help && sudo wo stack install + - sudo wo stack upgrade --netdata --no-prompt + - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql && sudo wo site create proxy.com --proxy=127.0.0.1:3000 + - sudo wo site create wp1.com --wp && sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc + - sudo wo site create wpsc-php73.net --wpsc --php73 && sudo wo site create wpfc-php73.net --wpfc --php73 + - sudo wo site create wpredis.net --wpredis && sudo wo site create wpredis-php73.net --wpredis --php73 + - sudo wo site create wpsubdir1.com --wpsubdir && sudo wo site create wpsubdir-php73.com --wpsubdir --php73 - - sudo wo site create html.net --html && sudo wo site create php.com --php && sudo wo site create mysql.com --mysql || sudo tail -n50 /var/log/wo/wordops.log - - sudo wo site create proxy.com --proxy=127.0.0.1:3000 || sudo tail -n50 /var/log/wo/wordops.log - - sudo wo site create wp1.com --wp || sudo tail -n50 /var/log/wo/wordops.log + - sudo wo site create wpsubdirwpsc1.com --wpsubdir --wpsc && sudo wo site create wpsubdirwpsc2.com --wpsubdir --wpfc && sudo wo site create wpsubdirwpsc1-php73.com --wpsubdir --wpsc --php73 && sudo wo site create wpsubdirwpsc2-php73.com --wpsubdir --wpfc --php73 + - sudo wo site create wpsubdomain1.com --wpsubdomain && sudo wo site create wpsubdomain1-php73.com --wpsubdomain --php73 && sudo wo site create wpsubdomainwpsc.org --wpsubdomain --wpsc && sudo wo site create wpsubdomainwpfc.org --wpsubdomain --wpfc && sudo wo site create wpsubdomainwpfc2.in --wpfc --wpsubdomain - - sudo wo site create wpsc1.net --wpsc && sudo wo site create wpfc1.com --wpfc || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsc-php73.net --wpsc --php73 && sudo wo site create wpfc-php73.net --wpfc --php73 || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpredis.net --wpredis && sudo wo site create wpredis-php73.net --wpredis --php73 || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsubdir1.com --wpsubdir && sudo wo site create wpsubdir-php73.com --wpsubdir --php73 || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsubdirwpsc1.com --wpsubdir --wpsc && sudo wo site create wpsubdirwpsc2.com --wpsubdir --wpfc || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsubdirwpsc1-php73.com --wpsubdir --wpsc --php73 && sudo wo site create wpsubdirwpsc2-php73.com --wpsubdir --wpfc --php73 || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsubdomain1.com --wpsubdomain && sudo wo site create wpsubdomain1-php73.com --wpsubdomain --php73 || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create wpsubdomainwpsc.org --wpsubdomain --wpsc && sudo wo site create wpsubdomainwpfc.org --wpsubdomain --wpfc && sudo wo site create wpsubdomainwpfc2.in --wpfc --wpsubdomain || sudo tail -n50 /var/log/wo/wordops.log - - - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql || sudo tail -n50 /var/log/wo/wordops.log - - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis || sudo tail -n50 /var/log/wo/wordops.log + - sudo wo site create 1.com --html && sudo wo site create 2.com --php && sudo wo site create 3.com --mysql + - sudo wo site update 1.com --wp && sudo wo site update 2.com --php73 && sudo wo site update 3.com --php73 && sudo wo site update 1.com --wpfc && sudo wo site update 1.com --wpsc && sudo wo site update 1.com --wpredis - sudo wp --allow-root --info - - sudo wo info || sudo tail -n50 /var/log/wo/wordops.log + - sudo wo info - sudo tree -L 2 /etc/nginx - - sudo cat /var/www/wpredis.net/wp-config.php - - sudo wo update --travis || sudo tail -n50 /var/log/wo/wordops.log + - sudo wo update --travis - sudo wo stack status \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cda1f0..49ca4ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,40 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Releases +### v3.9.x - [Unreleased] + --- -### v3.9.6 - [Unreleased] +### v3.9.5.4 - 2019-07-13 + +#### Added + +- New Nginx package on Ubuntu with TLS v1.3 support (OpenSSL 1.1.1c) +- Netdata upgrade with `wo stack upgrade --netdata` +- Netdata stack remove/purge + +#### Changed + +- phpRedisAdmin is now installed with the stack "--admin" +- Remove memcached - not required anymore + +#### Fixed + +- phpRedisAdmin installation +- Duplicated locations /robots.txt after upgrade to v3.9.5.3 +- Let's Encrypt stack "wo site update --letsencrypt/--letsencrypt=off" +- pt-query-advisor dead link +- Netdata persistant configuration + +### v3.9.5.3 - 2019-06-18 + +#### Added + +- Argument `--preserve` with the command `wo update` to keep current Nginx configuration + +#### Fixed + +- Nginx upgrade failure when running wo update ### v3.9.5.2 - 2019-06-17 diff --git a/README.md b/README.md index f32fe4d..9a9f7d1 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ - **Easy to install** : One step automated installer with migration from EasyEngine v3 support - **Fast deployment** : Fast and automated WordPress, Nginx, PHP, MySQL & Redis installation -- **Up-to-date** : Nginx 1.16.0 with Brotli support, PHP 7.2 & 7.3, MariaDB 10.3 & Redis 5.0 +- **Up-to-date** : Nginx 1.16.0 with TLS v1.3 & Brotli support, PHP 7.2 & 7.3, MariaDB 10.3 & Redis 5.0 - **Secured** : Hardened WordPress security with strict Nginx location directives - **Powerful** : Optimized Nginx configurations with multiple cache backends support - **SSL** : Let's Encrypt SSL certificates handled by acme.sh diff --git a/install b/install index 9073a21..842d006 100755 --- a/install +++ b/install @@ -7,10 +7,10 @@ # Copyright (c) 2019 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- -# Version 3.9.5.3 - 2019-06-18 +# Version 3.9.5.4 - 2019-07-09 # ------------------------------------------------------------------------- readonly wo_version_old="2.2.3" -readonly wo_version_new="3.9.5.3" +readonly wo_version_new="3.9.5.4" # CONTENTS # --- # 1. VARIABLES AND DECLARATIONS @@ -178,14 +178,10 @@ wo_install_dep() { DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confold" -y dist-upgrade } > /dev/null 2>&1 if [ "$wo_linux_distro" == "Ubuntu" ]; then - # add nginx repository gpg key - wget https://download.opensuse.org/repositories/home:virtubox:WordOps/xUbuntu_"$wo_distro_id"/Release.key -O Release.key - apt-key add - < Release.key - rm -f Release.key # install dependencies DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confold" -y install build-essential curl gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz gnupg2 cron ccze rsync tree haveged ufw > /dev/null 2>&1 else - # add nginx repository gpg ke + # add nginx repository gpg key wget https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_9.0/Release.key -O Release.key apt-key add - < Release.key rm -f Release.key @@ -334,17 +330,14 @@ wo_update_wp_cli() { WP_CLI_PATH=$(command -v wp) if [ -n "$WP_CLI_PATH" ]; then rm -rf "$WP_CLI_PATH" - # Update WP-CLI to the most recent version - wget -qO /usr/local/bin/wp https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar - chmod +x /usr/local/bin/wp - else - wget -qO /usr/local/bin/wp https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar - chmod +x /usr/local/bin/wp fi + # Update WP-CLI to the most recent version + wget -qO /usr/local/bin/wp https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar + chmod +x /usr/local/bin/wp [ ! -h /usr/bin/wp ] && { ln -s /usr/local/bin/wp /usr/bin/ } - [ ! -f /etc/bash_completion.d/wp-completion.bash ] && { + [ -d /etc/bash_completion.d ] && { wget -qO /etc/bash_completion.d/wp-completion.bash https://raw.githubusercontent.com/wp-cli/wp-cli/master/utils/wp-completion.bash } } >> "$wo_install_log" 2>&1 @@ -414,11 +407,15 @@ wo_install() { } >> "$wo_install_log" 2>&1 - if [ "$wo_force_install" = "y" ]; then - [ ! -f $HOME/.gitconfig ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME" > $HOME/.gitconfig'; } + if [ -f $HOME/.gitconfig ]; then + python3 setup.py install >> $wo_install_log 2>&1 + else + if [ "$wo_force_install" = "y" ]; then + [ ! -f $HOME/.gitconfig ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME" > $HOME/.gitconfig'; } + fi + python3 setup.py install fi - python3 setup.py install } wo_upgrade_nginx() { @@ -443,18 +440,18 @@ wo_upgrade_nginx() { # add new Nginx repository if [ "$wo_linux_distro" = "Ubuntu" ]; then - wget -qO /tmp/nginx-wo.key "https://download.opensuse.org/repositories/home:virtubox:WordOps/xUbuntu_${wo_distro_id}/Release.key" + add-apt-repository ppa:wordops/nginx-wo -y -u else if [ "$wo_distro_version" == "jessie" ]; then wget -qO /tmp/nginx-wo.key https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_8.0/Release.key else wget -qO /tmp/nginx-wo.key https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_9.0/Release.key fi + # import the respository key for updates + apt-key add - < /tmp/nginx-wo.key + rm -f /tmp/nginx-wo.key + sudo apt-get update -qq fi - # import the respository key for updates - apt-key add - < /tmp/nginx-wo.key - rm -f /tmp/nginx-wo.key - sudo apt-get update -qq # stop nginx service nginx stop @@ -491,6 +488,10 @@ wo_upgrade_nginx() { # restore sites and configuration /usr/bin/rsync -auz /var/lib/wo-backup/nginx/ /etc/nginx/ + sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* # update redis.conf headers if [ -f /etc/nginx/common/redis.conf ]; then diff --git a/setup.py b/setup.py index e07ac0c..1f098e4 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ if not os.path.isfile('/root/.gitconfig'): shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') setup(name='wo', - version='3.9.5.3', + version='3.9.5.4', description=long_description, long_description=long_description, classifiers=[], diff --git a/wo/cli/plugins/maintenance.py b/wo/cli/plugins/maintenance.py index 666668d..adf945c 100644 --- a/wo/cli/plugins/maintenance.py +++ b/wo/cli/plugins/maintenance.py @@ -7,10 +7,8 @@ from wo.core.variables import WOVariables from wo.core.aptget import WOAptGet from wo.core.apt_repo import WORepo from wo.core.services import WOService -from wo.core.fileutils import WOFileUtils from wo.core.shellexec import WOShellExec -from wo.core.git import WOGit -from wo.core.download import WODownload + def wo_maintenance_hook(app): diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 73944c5..7f1597c 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -839,6 +839,9 @@ class WOSiteUpdateController(CementBaseController): action='store_true')), (['--all'], dict(help="update all sites", action='store_true')), + (['--force'], + dict(help="force letsencrypt certificate renewal", + action='store_true')), ] @expose(help="Update site type or cache") @@ -1080,10 +1083,12 @@ class WOSiteUpdateController(CementBaseController): # --letsencrypt=renew code goes here if pargs.letsencrypt == "renew" and not pargs.all: expiry_days = SSL.getExpirationDays(self, wo_domain) - min_expiry_days = 30 + min_expiry_days = 45 if check_ssl: if (expiry_days <= min_expiry_days): renewLetsEncrypt(self, wo_domain) + elif pargs.force: + renewLetsEncrypt(self, wo_domain) else: Log.error( self, "You have more than 30 days with the current " @@ -1118,8 +1123,16 @@ class WOSiteUpdateController(CementBaseController): expiry_days = SSL.getExpirationDays(self, wo_domain, True) if expiry_days < 0: return 0 - min_expiry_days = 30 + min_expiry_days = 45 if (expiry_days <= min_expiry_days): + renewLetsEncrypt(self, ee_domain) + if not WOService.reload_service(self, 'nginx'): + Log.error(self, "service nginx reload failed. " + "check issues with `nginx -t` command") + Log.info(self, "SUCCESS: Certificate was successfully " + "renewed For https://{0}".format(wo_domain)) + elif pargs.force: + renewLetsEncrypt(self, ee_domain) Log.info(self, "Certificate was successfully renewed") if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " @@ -1128,7 +1141,7 @@ class WOSiteUpdateController(CementBaseController): "renewed For https://{0}".format(wo_domain)) else: Log.info( - self, "You have more than 30 days with the current " + self, "You have more than 45 days with the current " "certificate - refusing to run.\n") if (SSL.getExpirationDays(self, wo_domain) > 0): @@ -1260,12 +1273,14 @@ class WOSiteUpdateController(CementBaseController): " http://{0}".format(wo_domain)) return 0 - if pargs.letsencrypt == "on": + if pargs.letsencrypt: if data['letsencrypt'] is True: if not os.path.isfile("{0}/conf/nginx/ssl.conf.disabled" .format(wo_site_webroot)): - setupLetsEncrypt(self, wo_domain) - + if not pargs.letsencrypt == "subdomain": + setupLetsEncrypt(self, wo_domain) + else: + setupLetsEncryptSubdomain(self, wo_domain) else: WOFileUtils.mvfile(self, "{0}/conf/nginx/ssl.conf.disabled" .format(wo_site_webroot), @@ -1316,65 +1331,6 @@ class WOSiteUpdateController(CementBaseController): Log.info(self, "Successfully Disabled SSl for Site " " http://{0}".format(wo_domain)) - if pargs.letsencrypt == "subdomain": - if data['letsencrypt'] is True: - if not os.path.isfile("{0}/conf/nginx/ssl.conf.disabled" - .format(wo_site_webroot)): - setupLetsEncryptSubdomain(self, wo_domain) - - else: - WOFileUtils.mvfile(self, "{0}/conf/nginx/ssl.conf.disabled" - .format(wo_site_webroot), - '{0}/conf/nginx/ssl.conf' - .format(wo_site_webroot)) - - httpsRedirect(self, wo_domain) - - if not WOService.reload_service(self, 'nginx'): - Log.error(self, "service nginx reload failed. " - "check issues with `nginx -t` command") - - Log.info(self, "Congratulations! Successfully" - " Configured SSL for Site " - " https://{0}".format(wo_domain)) - - if (SSL.getExpirationDays(self, wo_domain) > 0): - Log.info(self, "Your cert will expire within " + - str(SSL.getExpirationDays(self, wo_domain)) + - " days.") - else: - Log.warn( - self, "Your cert already EXPIRED !" - " PLEASE renew soon . ") - - elif data['letsencrypt'] is False: - if os.path.isfile("{0}/conf/nginx/ssl.conf" - .format(wo_site_webroot)): - Log.info(self, 'Setting Nginx configuration') - WOFileUtils.mvfile(self, "{0}/conf/nginx/ssl.conf" - .format(wo_site_webroot), - '{0}/conf/nginx/ssl.conf.disabled' - .format(wo_site_webroot)) - httpsRedirect(self, wo_domain, False) - if os.path.isfile(("{0}/conf/nginx/hsts.conf") - .format(wo_site_webroot)): - WOFileUtils.mvfile(self, "{0}/conf/nginx/" - "hsts.conf" - .format(wo_site_webroot), - '{0}/conf/nginx/hsts.conf.disabled' - .format(wo_site_webroot)) - - if not WOService.reload_service(self, 'nginx'): - Log.error(self, "service nginx reload failed. " - "check issues with `nginx -t` command") - # Log.info(self,"Removing Cron Job set for - # cert auto-renewal") - # WOCron.remove_cron(self,'wo site update {0} - # --le=renew --min_expiry_limit 30 2> \/dev\/null' - # .format(wo_domain)) - Log.info(self, "Successfully Disabled SSl for Site " - " http://{0}".format(wo_domain)) - # Add nginx conf folder into GIT WOGit.add(self, ["{0}/conf/nginx".format(wo_site_webroot)], msg="Adding letsencrypts config of site: {0}" @@ -1398,7 +1354,6 @@ class WOSiteUpdateController(CementBaseController): else: Log.error(self, "HTTPS is not configured for given " "site") - return 0 elif data['hsts'] is False: if os.path.isfile(("{0}/conf/nginx/hsts.conf") @@ -1413,7 +1368,6 @@ class WOSiteUpdateController(CementBaseController): else: Log.error(self, "HSTS is not configured for given " "site") - return 0 if stype == oldsitetype and cache == oldcachetype: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index a9c05a6..4ea0cd4 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -25,6 +25,7 @@ import pwd import grp import codecs import platform +import psutil from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_migrate import WOStackMigrateController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController @@ -188,10 +189,14 @@ class WOStackController(CementBaseController): # add nginx repository if set(WOVariables.wo_nginx).issubset(set(apt_packages)): - Log.info(self, "Adding repository for NGINX, please wait...") - WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) - Log.debug(self, 'Adding repository for Nginx') - WORepo.add_key(self, WOVariables.wo_nginx_key) + if (WOVariables.wo_platform_distro == 'ubuntu'): + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, ppa=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding ppa for Nginx') + else: + WORepo.add(self, repo_url=WOVariables.wo_nginx_repo) + Log.debug(self, 'Adding repository for Nginx') + WORepo.add_key(self, WOVariables.wo_nginx_key) # add php repository if (set(WOVariables.wo_php73).issubset(set(apt_packages)) or @@ -1067,23 +1072,24 @@ class WOStackController(CementBaseController): WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") WOService.reload_service(self, 'mysql') - # create fail2ban configuration files - if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): - if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): - data = dict() - Log.debug(self, "Setting up fail2ban jails configuration") - wo_fail2ban = open('/etc/fail2ban/jail.d/custom.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban.mustache', - out=wo_fail2ban) - wo_fail2ban.close() + # create fail2ban configuration files + if set(WOVariables.wo_fail2ban).issubset(set(apt_packages)): + if not os.path.isfile("/etc/fail2ban/jail.d/custom.conf"): + data = dict() + Log.debug(self, "Setting up fail2ban jails configuration") + wo_fail2ban = open('/etc/fail2ban/jail.d/custom.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban.mustache', + out=wo_fail2ban) + wo_fail2ban.close() - Log.debug(self, "Setting up fail2ban wp filter") - wo_fail2ban = open('/etc/fail2ban/filter.d/wo-wordpress.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'fail2ban-wp.mustache', - out=wo_fail2ban) - wo_fail2ban.close() + Log.debug(self, "Setting up fail2ban wp filter") + wo_fail2ban = open('/etc/fail2ban/filter.d/' + 'wo-wordpress.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'fail2ban-wp.mustache', + out=wo_fail2ban) + wo_fail2ban.close() if (packages): if any('/usr/local/bin/wp' == x[1] for x in packages): @@ -1163,6 +1169,7 @@ class WOStackController(CementBaseController): WOShellExec.cmd_exec(self, "sudo -u www-data -H composer " "update -n --no-dev -d " "/var/www/22222/htdocs/db/pma/") + # netdata install if any('/var/lib/wo/tmp/kickstart.sh' == x[1] for x in packages): @@ -1178,6 +1185,12 @@ class WOStackController(CementBaseController): "health_alarm_notify.conf", 'SEND_EMAIL="YES"', 'SEND_EMAIL="NO"') + # make changes persistant + WOFileUtils.copyfile(self, "/opt/netdata/usr/" + "lib/netdata/conf.d/" + "health_alarm_notify.conf", + "/opt/netdata/etc/netdata/" + "health_alarm_notify.conf") # check if mysql credentials are available if os.path.isfile('/etc/mysql/conf.d/my.cnf'): try: @@ -1192,7 +1205,7 @@ class WOStackController(CementBaseController): WOMysql.execute(self, "flush privileges;", log=False) - except StatementExcecutionError as e: + except CommandExecutionError as e: Log.info( self, "fail to setup mysql user for netdata") WOService.restart_service(self, 'netdata') @@ -1205,7 +1218,8 @@ class WOStackController(CementBaseController): Log.debug(self, "Extracting wo-dashboard.tar.gz " "to location {0}22222/htdocs/" .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/wo-dashboard.tar.gz', + WOExtract.extract(self, '/var/lib/wo/tmp/' + 'wo-dashboard.tar.gz', '{0}22222/htdocs' .format(WOVariables.wo_webroot)) if WOVariables.wo_wan != 'eth0': @@ -1245,30 +1259,14 @@ class WOStackController(CementBaseController): WOVariables.wo_php_user, recursive=True) - # phpmemcachedadmin - if any('/var/lib/wo/tmp/memcached.tar.gz' == x[1] - for x in packages): - Log.debug(self, "Extracting memcached.tar.gz to location" - " {0}22222/htdocs/cache/memcache " - .format(WOVariables.wo_webroot)) - WOExtract.extract(self, '/var/lib/wo/tmp/memcached.tar.gz', - '{0}22222/htdocs/cache/memcache' - .format(WOVariables.wo_webroot)) - Log.debug(self, "Setting Privileges to " - "{0}22222/htdocs/cache/memcache file" - .format(WOVariables.wo_webroot)) - WOFileUtils.chown(self, '{0}22222' - .format(WOVariables.wo_webroot), - WOVariables.wo_php_user, - WOVariables.wo_php_user, - recursive=True) # webgrind if any('/var/lib/wo/tmp/webgrind.tar.gz' == x[1] for x in packages): Log.debug(self, "Extracting file webgrind.tar.gz to " "location /var/lib/wo/tmp/ ") WOExtract.extract( - self, '/var/lib/wo/tmp/webgrind.tar.gz', '/var/lib/wo/tmp/') + self, '/var/lib/wo/tmp/webgrind.tar.gz', + '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/php' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directroy " @@ -1311,7 +1309,8 @@ class WOStackController(CementBaseController): Log.debug(self, "Extracting file anemometer.tar.gz to " "location /var/lib/wo/tmp/ ") WOExtract.extract( - self, '/var/lib/wo/tmp/anemometer.tar.gz', '/var/lib/wo/tmp/') + self, '/var/lib/wo/tmp/anemometer.tar.gz', + '/var/lib/wo/tmp/') if not os.path.exists('{0}22222/htdocs/db/' .format(WOVariables.wo_webroot)): Log.debug(self, "Creating directory") @@ -1363,6 +1362,7 @@ class WOStackController(CementBaseController): if any('/usr/bin/pt-query-advisor' == x[1] for x in packages): WOFileUtils.chmod(self, "/usr/bin/pt-query-advisor", 0o775) + # phpredisadmin if any('/var/lib/wo/tmp/pra.tar.gz' == x[1] for x in packages): @@ -1371,15 +1371,20 @@ class WOStackController(CementBaseController): Log.debug(self, "Creating new directory " "{0}22222/htdocs/cache/redis" .format(WOVariables.wo_webroot)) - os.makedirs('{0}22222/htdocs/cache/redis' + os.makedirs('{0}22222/htdocs/cache/redis/phpRedisAdmin' .format(WOVariables.wo_webroot)) + WOFileUtils.chown(self, '{0}22222' + .format(WOVariables.wo_webroot), + WOVariables.wo_php_user, + WOVariables.wo_php_user, + recursive=True) if os.path.isfile("/usr/local/bin/composer"): WOShellExec.cmd_exec(self, "sudo -u www-data -H " "composer " "create-project -n -s dev " "erik-dubbelboer/php-redis-admin " "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin/ ") + "/redis/phpRedisAdmin ") Log.debug(self, 'Setting Privileges of webroot permission to ' '{0}22222/htdocs/cache/file ' .format(WOVariables.wo_webroot)) @@ -1412,6 +1417,7 @@ class WOStackController(CementBaseController): if self.app.pargs.all: self.app.pargs.web = True self.app.pargs.admin = True + self.app.pargs.php73 = True if self.app.pargs.web: self.app.pargs.nginx = True @@ -1429,6 +1435,7 @@ class WOStackController(CementBaseController): self.app.pargs.utils = True self.app.pargs.netdata = True self.app.pargs.dashboard = True + self.app.pargs.phpredisadmin = True # Redis if self.app.pargs.redis: @@ -1529,20 +1536,11 @@ class WOStackController(CementBaseController): # PHPMYADMIN if self.app.pargs.phpmyadmin: Log.debug(self, "Setting packages variable for phpMyAdmin ") - if (not self.app.pargs.composer): - packages = packages + [["https://github.com/phpmyadmin/" - "phpmyadmin/archive/STABLE.tar.gz", - "/var/lib/wo/tmp/pma.tar.gz", - "phpMyAdmin"], - ["https://getcomposer.org/" - "installer", - "/var/lib/wo/tmp/composer-install", - "Composer"]] - else: - packages = packages + [["https://github.com/phpmyadmin/" - "phpmyadmin/archive/STABLE.tar.gz", - "/var/lib/wo/tmp/pma.tar.gz", - "phpMyAdmin"]] + self.app.pargs.composer = True + packages = packages + [["https://github.com/phpmyadmin/" + "phpmyadmin/archive/STABLE.tar.gz", + "/var/lib/wo/tmp/pma.tar.gz", + "phpMyAdmin"]] # Composer if self.app.pargs.composer: Log.debug(self, "Setting packages variable for Composer ") @@ -1552,8 +1550,9 @@ class WOStackController(CementBaseController): # PHPREDISADMIN if self.app.pargs.phpredisadmin: Log.debug(self, "Setting packages variable for phpRedisAdmin") - packages = packages + [["https://github.com/ErikDubbelboer/" - "phpRedisAdmin/archive/master.tar.gz", + self.app.pargs.composer = True + packages = packages + [["https://github.com/erikdubbelboer/" + "phpRedisAdmin/archive/v1.11.3.tar.gz", "/var/lib/wo/tmp/pra.tar.gz", "phpRedisAdmin"], ["https://github.com/nrk/predis/" @@ -1605,12 +1604,7 @@ class WOStackController(CementBaseController): # UTILS if self.app.pargs.utils: Log.debug(self, "Setting packages variable for utils") - packages = packages + [["https://github.com/elijaa/" - "phpmemcachedadmin/archive/" - "1.3.0.tar.gz", - '/var/lib/wo/tmp/memcached.tar.gz', - 'phpMemcachedAdmin'], - ["https://raw.githubusercontent.com" + packages = packages + [["https://raw.githubusercontent.com" "/rtCamp/eeadmin/master/cache/nginx/" "clean.php", "{0}22222/htdocs/cache/" @@ -1640,11 +1634,8 @@ class WOStackController(CementBaseController): "archive/master.tar.gz", '/var/lib/wo/tmp/webgrind.tar.gz', 'Webgrind'], - ["http://bazaar.launchpad.net/~" - "percona-toolkit-dev/percona-toolkit/" - "2.1/download/head:/ptquerydigest-" - "20110624220137-or26tn4" - "expb9ul2a-16/pt-query-digest", + ["https://www.percona.com/" + "get/pt-query-digest", "/usr/bin/pt-query-advisor", "pt-query-advisor"], ["https://github.com/box/Anemometer/" @@ -1677,14 +1668,16 @@ class WOStackController(CementBaseController): Log.debug(self, "Enabling redis systemd service") WOShellExec.cmd_exec(self, "systemctl enable redis-server") if os.path.isfile("/etc/redis/redis.conf"): - if WOVariables.wo_ram < 512: + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + wo_swap = psutil.swap_memory().total / (1024 * 1024) + if wo_ram < 512: Log.debug(self, "Setting maxmemory variable to " "{0} in redis.conf" - .format(int(WOVariables.wo_ram*1024*1024*0.1))) + .format(int(wo_ram*1024*1024*0.1))) WOShellExec.cmd_exec(self, "sed -i 's/# maxmemory" " /maxmemory {0}/'" " /etc/redis/redis.conf" - .format(int(WOVariables.wo_ram*1024*1024*0.1))) + .format(int(wo_ram*1024*1024*0.1))) Log.debug( self, "Setting maxmemory-policy variable to " "allkeys-lru in redis.conf") @@ -1697,11 +1690,11 @@ class WOStackController(CementBaseController): else: Log.debug(self, "Setting maxmemory variable to {0} " "in redis.conf" - .format(int(WOVariables.wo_ram*1024*1024*0.2))) + .format(int(wo_ram*1024*1024*0.2))) WOShellExec.cmd_exec(self, "sed -i 's/# maxmemory " "/maxmemory {0}/' " "/etc/redis/redis.conf" - .format(int(WOVariables.wo_ram*1024*1024*0.2))) + .format(int(wo_ram*1024*1024*0.2))) Log.debug( self, "Setting maxmemory-policy variable " "to allkeys-lru in redis.conf") @@ -1751,6 +1744,10 @@ class WOStackController(CementBaseController): self.app.pargs.phpmyadmin = True self.app.pargs.composer = True self.app.pargs.utils = True + self.app.pargs.netdata = True + self.app.pargs.dashboard = True + self.app.pargs.phpredisadmin = True + # NGINX if self.app.pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): @@ -1805,11 +1802,22 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing package variable of phpMyAdmin ") packages = packages + ['{0}22222/htdocs/db/pma' .format(WOVariables.wo_webroot)] + # Composer + if self.app.pargs.composer: + Log.debug(self, "Removing package variable of Composer ") + if os.path.isfile('/usr/local/bin/composer'): + packages = packages + ['/usr/local/bin/composer'] + else: + Log.warn(self, "Composer is not installed with WordOps") + # PHPREDISADMIN if self.app.pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") - packages = packages + ['{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)] + if os.path.isdir('{0}22222/htdocs/cache/redis' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/' + 'cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)] # ADMINER if self.app.pargs.adminer: Log.debug(self, "Removing package variable of Adminer ") @@ -1823,12 +1831,23 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot), '{0}22222/htdocs/cache/nginx/' 'clean.php'.format(WOVariables.wo_webroot), - '{0}22222/htdocs/cache/memcache' - .format(WOVariables.wo_webroot), '/usr/bin/pt-query-advisor', '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot)] + if self.app.pargs.netdata: + Log.debug(self, "Removing Netdata") + if os.path.isfile('/opt/netdata/usr/' + 'libexec/netdata-uninstaller.sh'): + packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] + + if self.app.pargs.dashboard: + Log.debug(self, "Removing Wo-Dashboard") + packages = packages + ['{0}22222/htdocs/assets/' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)] + if (packages) or (apt_packages): wo_prompt = input('Are you sure you to want to' ' remove from server.' @@ -1843,6 +1862,13 @@ class WOStackController(CementBaseController): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') + # Netdata uninstaller + if (set(['/var/lib/wo/tmp/' + 'kickstart.sh']).issubset(set(packages))): + WOShellExec.cmd_exec(self, "bash /opt/netdata/usr/" + "libexec/netdata-" + "uninstaller.sh -y -f") + if (packages): WOFileUtils.remove(self, packages) WOAptGet.auto_remove(self) @@ -1888,6 +1914,10 @@ class WOStackController(CementBaseController): self.app.pargs.adminer = True self.app.pargs.phpmyadmin = True self.app.pargs.utils = True + self.app.pargs.composer = True + self.app.pargs.netdata = True + self.app.pargs.dashboard = True + self.app.pargs.phpredisadmin = True # NGINX if self.app.pargs.nginx: @@ -1936,11 +1966,22 @@ class WOStackController(CementBaseController): format(WOVariables.wo_webroot)] Log.debug(self, "Purge package variable phpMyAdmin") + # Composer + if self.app.pargs.composer: + Log.debug(self, "Removing package variable of Composer ") + if os.path.isfile('/usr/local/bin/composer'): + packages = packages + ['/usr/local/bin/composer'] + else: + Log.warn(self, "Composer is not installed with WordOps") + # PHPREDISADMIN if self.app.pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") - packages = packages + ['{0}22222/htdocs/cache/redis/phpRedisAdmin' - .format(WOVariables.wo_webroot)] + if os.path.isdir('{0}22222/htdocs/cache/redis' + .format(WOVariables.wo_webroot)): + packages = packages + ['{0}22222/htdocs/' + 'cache/redis/phpRedisAdmin' + .format(WOVariables.wo_webroot)] # Adminer if self.app.pargs.adminer: Log.debug(self, "Purge package variable Adminer") @@ -1955,13 +1996,24 @@ class WOStackController(CementBaseController): .format(WOVariables.wo_webroot), '{0}22222/htdocs/cache/nginx/' 'clean.php'.format(WOVariables.wo_webroot), - '{0}22222/htdocs/cache/memcache' - .format(WOVariables.wo_webroot), '/usr/bin/pt-query-advisor', '{0}22222/htdocs/db/anemometer' .format(WOVariables.wo_webroot) ] + if self.app.pargs.netdata: + Log.debug(self, "Removing Netdata") + if os.path.isfile('/opt/netdata/usr/' + 'libexec/netdata-uninstaller.sh'): + packages = packages + ['/var/lib/wo/tmp/kickstart.sh'] + + if self.app.pargs.dashboard: + Log.debug(self, "Removing Wo-Dashboard") + packages = packages + ['{0}22222/htdocs/assets/' + .format(WOVariables.wo_webroot), + '{0}22222/htdocs/index.php' + .format(WOVariables.wo_webroot)] + if (packages) or (apt_packages): wo_prompt = input('Are you sure you to want to purge ' 'from server ' @@ -1975,6 +2027,13 @@ class WOStackController(CementBaseController): if (set(["nginx-custom"]).issubset(set(apt_packages))): WOService.stop_service(self, 'nginx') + # Netdata uninstaller + if (set(['/var/lib/wo/tmp/' + 'kickstart.sh']).issubset(set(packages))): + WOShellExec.cmd_exec(self, "bash /opt/netdata/usr/" + "libexec/netdata-" + "uninstaller.sh -y -f") + if (apt_packages): Log.info(self, "Purging packages, please wait...") WOAptGet.remove(self, apt_packages, purge=True) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 43250f8..c8f1446 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -28,8 +28,6 @@ class WOStackUpgradeController(CementBaseController): dict(help='Upgrade admin tools stack', action='store_true')), (['--nginx'], dict(help='Upgrade Nginx stack', action='store_true')), - (['--nginxmainline'], - dict(help='Upgrade Nginx Mainline stack', action='store_true')), (['--php'], dict(help='Upgrade PHP stack', action='store_true')), (['--mysql'], @@ -38,6 +36,8 @@ class WOStackUpgradeController(CementBaseController): dict(help='Upgrade WPCLI', action='store_true')), (['--redis'], dict(help='Upgrade Redis', action='store_true')), + (['--netdata'], + dict(help='Upgrade Netdata', action='store_true')), (['--no-prompt'], dict(help="Upgrade Packages without any prompt", action='store_true')), @@ -83,7 +83,7 @@ class WOStackUpgradeController(CementBaseController): if ((not self.app.pargs.web) and (not self.app.pargs.nginx) and (not self.app.pargs.php) and (not self.app.pargs.mysql) and (not self.app.pargs.all) and (not self.app.pargs.wpcli) and - (not self.app.pargs.redis)): + (not self.app.pargs.netdata) and (not self.app.pargs.redis)): self.app.pargs.web = True if self.app.pargs.all: @@ -97,6 +97,7 @@ class WOStackUpgradeController(CementBaseController): self.app.pargs.php = True self.app.pargs.mysql = True self.app.pargs.wpcli = True + self.app.pargs.netdata = True if self.app.pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): @@ -137,6 +138,13 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "WPCLI is not installed with WordOps") + if self.app.pargs.netdata: + if os.path.isdir('/opt/netdata'): + packages = packages + [['https://my-netdata.io/' + 'kickstart-static64.sh', + '/var/lib/wo/tmp/kickstart.sh', + 'Netdata']] + if len(packages) or len(apt_packages): Log.info(self, "During package update process non nginx-cached" @@ -168,12 +176,21 @@ class WOStackUpgradeController(CementBaseController): if self.app.pargs.wpcli: WOFileUtils.remove(self, ['/usr/local/bin/wp']) + if self.app.pargs.netdata: + WOFileUtils.remove(self, ['/var/lib/wo/tmp/kickstart.sh']) + Log.debug(self, "Downloading following: {0}".format(packages)) WODownload.download(self, packages) if self.app.pargs.wpcli: WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + if self.app.pargs.netdata: + Log.info(self, "Upgrading Netdata, please wait...") + WOShellExec.cmd_exec(self, "/bin/bash /var/lib/wo/tmp/" + "kickstart.sh " + "--dont-wait") + Log.info(self, "Successfully updated packages") else: self.app.args.print_help() diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 815a40f..b7a6934 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -58,6 +58,9 @@ class WOUpdateController(CementBaseController): except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") + except Exception as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") elif self.app.pargs.preserve: try: Log.info(self, "updating WordOps, please wait...") @@ -65,6 +68,9 @@ class WOUpdateController(CementBaseController): except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") + except Exception as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") else: try: Log.info(self, "updating WordOps, please wait...") @@ -72,6 +78,9 @@ class WOUpdateController(CementBaseController): except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") + except Exception as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") def load(app): diff --git a/wo/core/addswap.py b/wo/core/addswap.py index 5feb1e5..02df054 100644 --- a/wo/core/addswap.py +++ b/wo/core/addswap.py @@ -5,6 +5,7 @@ from wo.core.fileutils import WOFileUtils from wo.core.aptget import WOAptGet from wo.core.logging import Log import os +import psutil class WOSwap(): @@ -16,8 +17,11 @@ class WOSwap(): def add(self): """Swap addition with WordOps""" - if WOVariables.wo_ram < 512: - if WOVariables.wo_swap < 1000: + # Get System RAM and SWAP details + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + wo_swap = psutil.swap_memory().total / (1024 * 1024) + if wo_ram < 512: + if wo_swap < 1000: Log.info(self, "Adding SWAP file, please wait...") # Install dphys-swapfile diff --git a/wo/core/variables.py b/wo/core/variables.py index ea5e7b2..2e1b43f 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -11,7 +11,7 @@ class WOVariables(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.5.3" + wo_version = "3.9.5.4" # WordOps packages versions wo_wp_cli = "2.2.0" wo_adminer = "4.7.1" @@ -74,10 +74,6 @@ class WOVariables(): os.system( "/usr/bin/git config --global user.email {0}".format(wo_email)) - # Get System RAM and SWAP details - wo_ram = psutil.virtual_memory().total / (1024 * 1024) - wo_swap = psutil.swap_memory().total / (1024 * 1024) - # MySQL hostname wo_mysql_host = "" config = configparser.RawConfigParser() @@ -100,18 +96,8 @@ class WOVariables(): wo_nginx_repo = ("deb http://download.opensuse.org" "/repositories/home:" "/virtubox:/WordOps/xUbuntu_14.04/ /") - elif wo_platform_codename == 'xenial': - wo_nginx_repo = ("deb http://download.opensuse.org" - "/repositories/home:" - "/virtubox:/WordOps/xUbuntu_16.04/ /") - elif wo_platform_codename == 'bionic': - wo_nginx_repo = ("deb http://download.opensuse.org" - "/repositories/home:" - "/virtubox:/WordOps/xUbuntu_18.04/ /") else: - wo_nginx_repo = ("deb http://download.opensuse.org" - "/repositories/home:" - "/virtubox:/WordOps/xUbuntu_19.04/ /") + wo_nginx_repo = "ppa:wordops/nginx-wo" elif wo_platform_distro == 'debian': if wo_platform_codename == 'jessie': wo_nginx_repo = ("deb http://download.opensuse.org" @@ -141,7 +127,7 @@ class WOVariables(): "php7.3-cli", "php7.3-mbstring", "php7.3-bcmath", "php7.3-mysql", "php7.3-opcache", "php7.3-zip", "php7.3-xml", "php7.3-soap"] - wo_php_extra = ["php-memcached", "php-imagick", "memcached", + wo_php_extra = ["php-memcached", "php-imagick", "graphviz", "php-xdebug", "php-msgpack", "php-redis"] wo_php_key = '' else: @@ -158,7 +144,7 @@ class WOVariables(): "php7.3-cli", "php7.3-mbstring", "php7.3-bcmath", "php7.3-mysql", "php7.3-opcache", "php7.3-zip", "php7.3-xml", "php7.3-soap"] - wo_php_extra = ["php-memcached", "php-imagick", "memcached", + wo_php_extra = ["php-memcached", "php-imagick", "graphviz", "php-xdebug", "php-msgpack", "php-redis"] wo_php_key = 'AC0E47584A7A714D'