diff --git a/.travis.yml b/.travis.yml index 3337222..ebeaa10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,12 @@ -sudo: required +os: linux dist: bionic -language: bash +language: shell notifications: webhooks: secure: "JiGtzYplTyFg/L6Rsi7ptEQIV29O5qCWU2Zf5pLITsQrBrQO4cIXXp9G4Z+cenXjfIiqbqIgU0US3zXeIAl4g14xdfzmMYeMMwuKBpI8afMYv8MD6ldoP0MTFHQfROE6OXxKLVUvZn1R0oLLU1fzVSI0qGjNkt20cf/Lrt/reH/zS5hAI92kWI3u2zPu7Zn/g/a8MO/Y3Iv7v1PSQaVkVJVqtOK3U2GJqhIv2G1AVcaPb7Nh/V2zm2dDYBVT0UotBnlBUcUXbEMP77D9pjtWXd1/0rWuJIHixMjwUybpZqY75UMee5INynU6OZRsv029LRHAIMkWhfBkdVN/U5jhQJzui14+vRQrb5nfUMG8Cd8INojDlu6dk/ps2GzTCCXBITeMQKAouUoHD2LEbsNp17xi1K4ZlKb3+0lrOAiS4JYFE6wOo4yMlLTYoquYSqk7AuxuUS8A5OD5MYxhk9uafiTSxKFOo39KYWTSaACsPD8q1swaTSjoYm9skyZvIkIFq5bHBCYEGFe6X/NY9l5tz3hSe+TJOerCHsg+dXVuQl+pIp5nw2as9TH9ox5Vgqc9Zh4GbTDQVvdAmUpmlsZ/SKoOMCkmkB1aRNFq/7RnERIJyAEGJbauHWmjtOM4cCxesl0L0b2Eab89zQpSn7pzE8JTiJgpzCUc22p653PTaqM=" -addons: - apt: - update: true - git: quiet: true @@ -23,18 +19,19 @@ before_script: - unset LANG - sudo apt-get update --allow-releaseinfo-change -qq - sudo apt-get -qq purge mysql* graphviz* redis* - - sudo apt-get install -qq git python3-setuptools python3-dev python3-apt ccze tree - sudo apt-get -qq autoremove --purge after_script: - - sudo bash install --purge + - sudo -E bash install --purge script: - lsb_release -a - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - - sudo time bash install --travis -b "$TRAVIS_BRANCH" - - sudo time bash tests/travis.sh - - sudo wo update --travis \ No newline at end of file + - sudo -E time bash install --travis -b "$TRAVIS_BRANCH" + - sudo -E python3 -m pip install -U -r requirements.txt + - sudo -E time bash tests/travis.sh + - sudo -E wo update --travis + - sudo python3 setup.py sdist bdist_wheel \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 812e66e..b9ca680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,40 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +### v3.10.0 - 2019-10-30 + +#### Added + +- WordOps is now installed inside a wheel with pip (easier, cleaner and safer) from PyPi +- Redis 5.0.6 package backported to Debian 8/9/10 +- Custom motd to display a message if a new WordOps release is available +- Run `mysql_upgrade` during MySQL upgrade with `wo stack upgrade` to perform migration if needed +- `wo stack upgrade --ngxblocker` to update ngxblocker blocklist + +#### Changed + +- Sysctl tweaks are applied during stack install and removed from install script +- Nginx & MariaDB systemd tweaks are removed from install script and applied during stacks install/upgrade +- Initial creation of .gitconfig is displayed the first time you run the command `wo` +- Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage +- WordOps now check if a repository already exist before trying to adding it again. +- Improved SSL certificate error messages by displaying domain IP and server IP +- Version check before updating WordOps with `wo update` is now directly handled by `wo` +- Refactored WordOps download function with python3-requests +- MySQL backup path changed to `/var/lib/wo-backup/mysql` +- Do not check anymore if stack are installed with apt in `wo service` but only if there is a systemd service +- Refactored `--letsencrypt=renew`. Require the flag `--force` if certificate expiration is more than 45 days +- Improve netdata stack upgrade with install from source detection and updater fallback + +#### Fixed + +- Incorrect PHP-FPM log path is `wo log` +- force-ssl.conf not removed after removing a site +- `wo clean --opcache` not working with invalid SSL certificate +- `wo stack install --cheat` wasn't working properly previously +- `wo info` failure depending on php-fpm pool name. ConfigParser will now detect the section name. + + ### v3.9.9.4 - 2019-10-18 #### Changed diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..60115fe --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include *.py +include setup.cfg +include README.md CHANGELOG.md LICENSE +include *.txt diff --git a/README.md b/README.md index 9dc7748..fd9d101 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ MIT Commits GitHub release -
WordOps install +
PyPI - Downloads codacy Badge Twitter Badge Rocket.chat @@ -65,7 +65,7 @@ - Ubuntu 16.04 LTS (Xenial) - Ubuntu 19.04 (Disco) - Debian 9 (Stretch) -- Debian 10 (Buster) - Not ready for production +- Debian 10 (Buster) - Raspbian 9 (Stretch) - Raspbian 10 (Buster) - Testing @@ -173,6 +173,7 @@ Apps & Tools shipped with WordOps : - [ClamAV](https://github.com/Cisco-Talos/clamav-devel) - [cheat.sh](https://github.com/chubin/cheat.sh) - [ProFTPd](https://github.com/proftpd/proftpd) +- [nginx-ultimate-bad-bot-blocker](https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/) Cache Plugins supported by WordOps : diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index d5dacc0..c3146c0 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -79,7 +79,7 @@ _wo_complete() ;; "upgrade" ) COMPREPLY=( $(compgen \ - -W "--web --admin --utils --nginx --php --php73 --mysql --all --netdata --composer --phpmyadmin --dashboard --no-prompt --mysqtuner --wpcli --force" \ + -W "--web --admin --utils --nginx --php --php73 --mysql --all --netdata --composer --phpmyadmin --dashboard --mysqtuner --wpcli --force" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") diff --git a/docs/wo.8 b/docs/wo.8 index b6f23b7..357d1ea 100644 --- a/docs/wo.8 +++ b/docs/wo.8 @@ -1,27 +1,27 @@ -.TH wo 8 "WordOps (wo) version: 3.9.6.3" "Jul 26,2019" "WordOps" +.TH wo 8 "WordOps (wo) version: 3.10.0" "Oct 24,2019" "WordOps" .SH NAME .B WordOps (wo) \- Manage Nginx Based Websites. .SH SYNOPSIS wo [ --version | --help | info | stack | site | debug | update | clean | import_slow_log | log | secure | sync | maintenance ] .TP -wo stack [ install | remove | purge | migrate | upgrade] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils | --dashboard | --netdata | --fail2ban | --proftpd ] +wo stack [ install | remove | purge | migrate | upgrade ] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils | --dashboard | --netdata | --fail2ban | --proftpd ] .TP wo stack [ status | start | stop | reload | restart ] [--all | --nginx | --php | --php73 |--mysql | --web | --redis | --netdata | --fail2ban | --proftpd] .TP wo site [ list | info | show | enable | disable | edit | cd | show ] [ example.com ] .TP -wo site create example.com [ --html | --php | --php73 | --mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis | --letsencrypt/-le/--letsencrypt=wildcard][--dns/--dns=dns_cf/dns_do]] +wo site create example.com [ --html | --php | --php73 | --mysql][[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis | --wpce | --wprocket ] [ -le/--letsencrypt=wildcard ][ --dns/--dns=dns_cf/dns_dgon]] .TP -wo site update example.com [ --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ] [--password] [-le/--letsencrypt/--letsencrypt=on/off/wildcard/clean/purge] [--dns/--dns=dns_cf/dns_do]] +wo site update example.com [ --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis | --wpce | --wprocket ] [--password] [-le/--letsencrypt=on/off/wildcard/clean/purge ] [ --dns/--dns=dns_cf/dns_dgon ] .TP -wo site delete example.com [--db | --files | --all | --no-prompt | --force/-f ] +wo site delete example.com [--db | --files | --all | --no-prompt | --force ] .TP wo debug [ -i | --all=on/off |--nginx=on/off | --rewrite=on/off | --php=on/off | --fpm=on/off | --mysql=on/off ] .TP wo debug example.com [ -i | --all=on/off | --nginx=on/off | --rewrite=on/off | --wp=on/off ] .TP -wo secure [ --auth | --port | --ip ] +wo secure [ --auth | --port | --ip | --ssh | --sshport ] .SH DESCRIPTION WordOps aka wo is the opensource project developed with the purpose to automate web-server configuration. .br @@ -48,7 +48,7 @@ Display WordOps (wo) help. .TP .B install [ --all | --web | --nginx | --php | --php73 |--mysql | --redis | --adminer | --phpmyadmin | --phpredismyadmin | --wpcli | --utils | --netdata | --dashboard | --fail2ban | --proftpd ] .br -Install Nginx PHP5 MySQL Postfix stack Packages if not used with +Install Nginx PHP7.2 MariaDB SendMail Netdata Fail2Ban stack Packages if not used with .br any options.Installs specific package if used with option. .TP @@ -129,13 +129,13 @@ Disable site by Destroying softlink with site file in .br Edit NGINX configuration of site. .TP -.B create [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ]] +.B create [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ] .br Create new site according to given options. If no options provided .br create static site with html only. .TP -.B update [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis ] [--password]] +.B update [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis ] [--password ] .br Update site configuration according to specified options. .TP @@ -270,17 +270,23 @@ used with wo secure command. Update whitelist IP address .TP .B --wpsc .br -Install and activate Nginx-helper and WP Super Cache plugin. +Install and activate WP Super Cache plugin and serve pages from cache directly with Nginx. .TP .B --wpfc .br -Install and activate Nginx-helper plugin with -.br -Nginx FastCGI cache. +Install and activate Nginx-helper plugin with Nginx FastCGI cache. .TP .B --wpredis .br -Install, activate, configure Nginx-helper and Redis Object Cache Plugin, Configure NGINX for Redis Page Caching. +Install, activate, configure Nginx-helper and Redis Object Cache Plugin, Configure NGINX for Redis Full-Page Caching. +.TP +.B --wpce +.br +Install and activate Cache-enabler plugin and serve pages from cache directly with Nginx. +.TP +.B --wprocket +.br +Configure Nginx for WP-Rocket plugin to serve pages from cache directly with Nginx. .SH FILES .br /etc/wo/wo.conf diff --git a/gitconfig.py b/gitconfig.py deleted file mode 100644 index f335b14..0000000 --- a/gitconfig.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -import configparser -import os -import re -import shutil - -# WordOps git configuration management -config = configparser.ConfigParser() -config.read(os.path.expanduser("~")+'/.gitconfig') -try: - wo_user = config['user']['name'] - wo_email = config['user']['email'] -except Exception: - print("WordOps (wo) require an username & and an email " - "address to configure Git (used to save server configurations)") - print("Your informations will ONLY be stored locally") - - wo_user = input("Enter your name: ") - while wo_user == "": - print("Unfortunately, this can't be left blank") - wo_user = input("Enter your name: ") - - wo_email = input("Enter your email: ") - - while not re.match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$", - wo_email): - print("Whoops, seems like you made a typo - " - "the e-mailaddress is invalid...") - wo_email = input("Enter your email: ") - - os.system("git config --global user.name {0}".format(wo_user)) - os.system("git config --global user.email {0}".format(wo_email)) - -if not os.path.isfile('/root/.gitconfig'): - shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') diff --git a/install b/install index 6a07e1d..512615f 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.9.9.4 - 2019-10-18 +# Version 3.10.0 - 2019-10-30 # ------------------------------------------------------------------------- # CONTENTS @@ -27,6 +27,7 @@ TPUT_RESET=$(tput sgr0) TPUT_FAIL=$(tput setaf 1) TPUT_INFO=$(tput setaf 7) TPUT_ECHO=$(tput setaf 4) +TPUT_OK=$(tput setaf 2) wo_lib_echo() { @@ -62,10 +63,6 @@ while [ "$#" -gt 0 ]; do wo_branch="$2" shift ;; - -v | --version) - wo_version="$2" - shift - ;; --force) wo_force_install="y" ;; @@ -73,6 +70,9 @@ while [ "$#" -gt 0 ]; do wo_travis="y" wo_force_install="y" ;; + --mainline | --beta) + wo_branch="mainline" + ;; -s | --silent) wo_force_install="y" ;; @@ -105,44 +105,51 @@ export LC_ALL='C.UTF-8' # check if a command exist command_exists() { - command -v "$@" > /dev/null 2>&1 + command -v "$@" >/dev/null 2>&1 } # run functions and exit on failure _run() { if [ -n "$2" ]; then - wo_lib_echo "$2" + echo -ne "${TPUT_ECHO}${2}${TPUT_RESET}\t" fi - if ! { "$1" >> "$wo_install_log" 2>&1; }; then - exit 1 + if ! { "$1" >>"$wo_install_log" 2>&1; }; then + if [ -n "$2" ]; then + echo -e "${TPUT_FAIL}[KO]${TPUT_RESET}" + fi + else + if [ -n "$2" ]; then + echo -e "[${TPUT_OK}OK${TPUT_RESET}]" + fi fi } -### -# 1 - Define variables for later use -### -if [ -z "$wo_branch" ]; then - wo_branch=master -fi -readonly wo_log_dir=/var/log/wo/ -readonly wo_backup_dir=/var/lib/wo-backup/ -readonly wo_tmp_dir=/var/lib/wo/tmp -readonly wo_install_log=/var/log/wo/install.log -readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' -readonly TIME=$(date +"$TIME_FORMAT") -readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" -readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" -readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" -readonly wo_lxc=$(grep "container=lxc" /proc/1/environ) -readonly wo_wsl=$(grep "wsl" /proc/1/environ) -readonly wo_arch="$(uname -m)" +_curl() { + curl -m 10 --retry 3 -sL "$@" +} -if [ -x /usr/local/bin/ee ]; then - ee_migration=1 -elif [ -x /usr/local/bin/wo ]; then - wo_upgrade=1 -fi +wo_init_variables() { + if [ -z "$wo_branch" ]; then + if [ "$wo_travis" = "y" ]; then + wo_branch=updating-configuration + else + wo_branch=master + fi + fi + readonly wo_install_log=/var/log/wo/install.log + readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' + readonly TIME=$(date +"$TIME_FORMAT") + readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" + readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" + readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" + + if [ -x /usr/local/bin/ee ]; then + ee_migration=1 + elif [ -x /usr/local/bin/wo ]; then + wo_upgrade=1 + fi +} ### # 1 - Checking linux distro @@ -156,8 +163,8 @@ wo_check_distro() { if [ -z "$wo_force_install" ]; then if [ "$wo_linux_distro" != "Ubuntu" ] && [ "$wo_linux_distro" != "Debian" ] && [ "$wo_linux_distro" != "Raspbian" ]; then wo_lib_echo_fail "WordOps (wo) only supports Ubuntu, Debian & Raspbian at the moment." - wo_lib_echo_fail "If you are feeling adventurous, you are free to fork WordOps to support" - wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." + wo_lib_echo_fail "You can bypass this warning by adding the flag --force to the install command" + wo_lib_echo_fail "Feel free to open a pull-request if you want to add support for another Linux distributions" exit 100 else check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster") @@ -174,14 +181,18 @@ wo_check_distro() { # 1 - To prevent errors or unexpected behaviour, create the log and ACL it ### wo_dir_init() { + local wo_log_dir=/var/log/wo + local wo_backup_dir=/var/lib/wo-backup + local wo_tmp_dir=/var/lib/wo/tmp if [ ! -d "$wo_log_dir" ] || [ ! -d "$wo_backup_dir" ] || [ ! -d "$wo_tmp_dir" ]; then - mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? + mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" # create wordops log files touch /var/log/wo/{wordops.log,install.log} - chmod -R 700 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? + chmod -R 750 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" + chown -R root:adm "$wo_log_dir" fi } @@ -190,11 +201,6 @@ wo_dir_init() { # 2 - Setup the dependencies for installation #### -wo_dist_upgrade() { - # perform server packages upgrade - apt-get dist-upgrade --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --option=Dpkg::options::=--force-unsafe-io --assume-yes --quiet -} - wo_install_dep() { local wo_linux_distro wo_linux_distro=$(lsb_release -is) @@ -202,14 +208,13 @@ wo_install_dep() { # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip python3-pip python3-wheel python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \ - gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - - add-apt-repository ppa:wordops/nginx-wo -yn else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip dirmngr sudo python3-pip python3-wheel python3-apt python3-setuptools python3-dev ca-certificates sqlite3 git tar \ - software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key @@ -221,6 +226,9 @@ wo_install_dep() { if [ ! -f /etc/apt/apt.conf.d/20auto-upgrades ]; then cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades fi + # upgrade pip + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade setuptools wheel } @@ -243,17 +251,16 @@ wo_sync_db() { # Switching from EE -> WO ### if [ ! -f /var/lib/wo/dbase.db ]; then - # Create the WordOps folder - mkdir -p /var/lib/wo if [ -f /var/lib/ee/ee.db ]; then + # Make a backup of the EasyEngine database cp /var/lib/ee/ee.db /var/lib/wo/dbase-ee.db # Copy ee database cp /var/lib/ee/ee.db /var/lib/wo/dbase.db else - if [ -d /etc/nginx/sites-available ]; then + if [ -d /etc/nginx/sites-available ] && [ -d /var/www ]; then # Create an empty database for WordOps echo "CREATE TABLE sites ( @@ -410,59 +417,40 @@ wo_install_acme_sh() { # Let's Encrypt .well-known folder setup if [ ! -d /var/www/html/.well-known/acme-challenge ]; then mkdir -p /var/www/html/.well-known/acme-challenge - chown -R www-data:www-data /var/www/html /var/www/html/.well-known - chmod 750 /var/www/html /var/www/html/.well-known - else - chown -R www-data:www-data /var/www/html /var/www/html/.well-known - chmod 750 /var/www/html /var/www/html/.well-known fi -} + chown -R www-data:www-data /var/www/html /var/www/html/.well-known + chmod 750 /var/www/html /var/www/html/.well-known -wo_git_config() { - - if [ "$wo_force_install" = "y" ]; then - [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } - fi - - # .gitconfig inital setup - cd /var/lib/wo/tmp/WordOps-install || exit 1 - python3 gitconfig.py - -} - -# Download WordOps -wo_download() { - rm -f /etc/bash_completion.d/wo_auto.rc - rm -rf /var/lib/wo/tmp/WordOps-* - if [ -z "$wo_version" ]; then - curl -sL https://github.com/WordOps/WordOps/archive/${wo_branch}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_branch" /var/lib/wo/tmp/WordOps-install - else - curl -sL https://github.com/WordOps/WordOps/archive/v${wo_version}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_version" /var/lib/wo/tmp/WordOps-install - fi - return 0 } # WordOps install wo_install() { - - cd /var/lib/wo/tmp/WordOps-install || exit 1 - python3 setup.py install + if [ "$wo_branch" = "master" ]; then + python3 -m pip install --upgrade wordops + else + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" + fi + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc } # Clone Github repository if it doesn't exist wo_travis_install() { - if [ "$wo_force_install" = "y" ]; then - [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } + if [ -d ./dist ]; then + rm -rf dist fi - - if [ -f "$HOME/.gitconfig" ]; then - # install and redirect log to not print python package install - python3 setup.py install + if [ -f ./setup.py ]; then + python3 setup.py sdist bdist_wheel + python3 -m pip install --upgrade dist/*.whl + else + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc } @@ -496,37 +484,40 @@ wo_upgrade_nginx() { fi # install new nginx package - if [ -n "$CHECK_NGINX_EE" ]; then - if [ -x /usr/local/bin/wo ]; then - [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } - # stop nginx - service nginx stop - # remove previous package - apt-mark unhold nginx-ee nginx-common nginx-custom - apt-get autoremove nginx-ee nginx-common nginx-custom --allow-change-held-packages --purge -qq - # remove previous php-fpm pool configuration - if [ -n "$CHECK_PHP72" ]; then - apt-get purge php7.2-fpm -y -qq - rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} + if { + if [ -n "$CHECK_NGINX_EE" ]; then + if [ -x /usr/local/bin/wo ]; then + [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } + # stop nginx + service nginx stop + # remove previous package + apt-mark unhold nginx-ee nginx-common nginx-custom + apt-get autoremove nginx-ee nginx-common nginx-custom --allow-change-held-packages --purge -qq + # remove previous php-fpm pool configuration + if [ -n "$CHECK_PHP72" ]; then + apt-get purge php7.2-fpm -y -qq + rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} + fi + if [ -d /etc/nginx ]; then + rm -rf /etc/nginx + fi + /usr/local/bin/wo stack install --nginx --php + rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo + /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ + /usr/local/bin/wo stack upgrade --nginx --force fi - if [ -d /etc/nginx ]; then - rm -rf /etc/nginx - fi - /usr/local/bin/wo stack install --nginx --php - rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo - /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ - /usr/local/bin/wo stack upgrade --nginx --force fi + }; then + # restore sites and configuration + [ -f /etc/nginx/htpasswd-ee ] && { cp -f /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } + sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/htpasswd-ee/htpasswd-wo/" /etc/nginx/common/acl.conf + sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf fi - # restore sites and configuration - [ -f /etc/nginx/htpasswd-ee ] && { cp -f /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } - sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/htpasswd-ee/htpasswd-wo/" /etc/nginx/common/acl.conf - sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf # update redis.conf headers if [ -f /etc/nginx/common/redis.conf ]; then @@ -549,30 +540,26 @@ wo_upgrade_nginx() { systemctl start nginx fi [ -f /var/lib/wo/tmp/nginx-block ] && { mv /var/lib/wo/tmp/nginx-block /etc/apt/preferences.d/nginx-block; } - + return 0 } wo_update_latest() { # Move ~/.my.cnf to /etc/mysql/conf.d/my.cnf - if [ ! -f /etc/mysql/conf.d/my.cnf ]; then - # create conf.d folder if not exist - [ ! -d /etc/mysql/conf.d ] && { - mkdir -p /etc/mysql/conf.d - chmod 755 /etc/mysql/conf.d - } - if [ -f "$HOME/.my.cnf" ]; then - cp -f "$HOME/.my.cnf" /etc/mysql/conf.d/my.cnf - chmod 600 /etc/mysql/conf.d/my.cnf - - elif [ -f /root/.my.cnf ]; then - cp -f /root/.my.cnf /etc/mysql/conf.d/my.cnf - chmod 600 /etc/mysql/conf.d/my.cnf - fi - else - if [ ! -f /root/.my.cnf ]; then - cp /etc/mysql/conf.d/my.cnf /root/.my.cnf - chmod 600 /root/.my.cnf + if [ -d /etc/mysql ]; then + if [ ! -f /etc/mysql/conf.d/my.cnf ]; then + # create conf.d folder if not exist + [ ! -d /etc/mysql/conf.d ] && { + mkdir -p /etc/mysql/conf.d + chmod 755 /etc/mysql/conf.d + } + if [ -f /root/.my.cnf ]; then + cp -f /root/.my.cnf /etc/mysql/conf.d/my.cnf + chmod 600 /etc/mysql/conf.d/my.cnf + elif [ -f "$HOME/.my.cnf" ]; then + cp -f "$HOME/.my.cnf" /etc/mysql/conf.d/my.cnf + chmod 600 /etc/mysql/conf.d/my.cnf + fi fi fi } @@ -603,100 +590,36 @@ wo_remove_ee_cron() { } -wo_tweak_kernel() { - local wo_distro_version - wo_distro_version=$(lsb_release -sc) - if [ "$wo_arch" = "x86_64" ] && [ -z "$wo_lxc" ] && [ -z "$wo_wsl" ]; then - rm -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf - wget -qO /etc/sysctl.d/60-wo-tweaks.conf https://raw.githubusercontent.com/WordOps/WordOps/"$wo_branch"/wo/cli/templates/sysctl.mustache - if [ "$wo_distro_version" = "bionic" ] || [ "$wo_distro_version" = "disco" ] || [ "$wo_distro_version" = "buster" ]; then - modprobe tcp_bbr && echo 'tcp_bbr' >> /etc/modules-load.d/bbr.conf - echo -e '\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_notsent_lowat = 16384' >> /etc/sysctl.d/60-wo-tweaks.conf - else - modprobe tcp_htcp && echo 'tcp_htcp' >> /etc/modules-load.d/htcp.conf - echo 'net.ipv4.tcp_congestion_control = htcp' >> /etc/sysctl.d/60-wo-tweaks.conf - fi - # apply sysctl tweaks - sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf - fi -} - -wo_systemd_tweak() { - - if [ ! -x /opt/wo-kernel.sh ]; then - # download and setup wo-kernel systemd service to apply kernel tweaks for netdata and redis on server startup - wget -qO /opt/wo-kernel.sh https://raw.githubusercontent.com/WordOps/WordOps/updating-configuration/wo/cli/templates/wo-kernel-script.mustache - chmod +x /opt/wo-kernel.sh - wget -qO /lib/systemd/system/wo-kernel.service https://raw.githubusercontent.com/WordOps/WordOps/updating-configuration/wo/cli/templates/wo-kernel-service.mustache - systemctl enable wo-kernel.service - systemctl start wo-kernel.service - fi - - LIMIT_CHECK=$(grep "500000" /etc/security/limits.conf) - if [ -z "$LIMIT_CHECK" ]; then - echo -e "* hard nofile 500000\n* soft nofile 500000\nroot hard nofile 500000\nroot soft nofile 500000\n" >> /etc/security/limits.conf - fi - -} - wo_domain_suffix() { - curl -m 10 --retry 3 -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat -} - -wo_mariadb_tweak() { - # increase mariadb open_files_limit - if [ -d /etc/systemd/system/mariadb.service.d ] && [ ! -f /etc/systemd/system/mariadb.service.d/limits.conf ]; then - echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/mariadb.service.d/limits.conf - systemctl daemon-reload - service mysql restart - fi -} - -wo_nginx_tweak() { - # increase nginx open_files_limit - if [ ! -d /etc/systemd/system/nginx.service.d ]; then - mkdir -p /etc/systemd/system/nginx.service.d - if [ ! -f /etc/systemd/system/nginx.service.d/limits.conf ]; then - echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/nginx.service.d/limits.conf - systemctl daemon-reload - nginx -t && service nginx restart - fi - fi + _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' >/var/lib/wo/public_suffix_list.dat } wo_clean() { - rm -rf /usr/local/lib/python3.*/dist-packages/wo-* /usr/local/lib/python3.*/dist-packages/wordops-* + rm -rf /usr/local/lib/python3.*/dist-packages/wo-*.egg /usr/local/lib/python3.*/dist-packages/wordops-*.egg } wo_uninstall() { + if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then + python3 -m pip uninstall wordops -y + fi rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates } -wo_cheat_install() { - curl -sL https://cht.sh/:cht.sh > /usr/local/bin/cht.sh - chmod +x /usr/local/bin/cht.sh - [ ! -h /usr/local/bin/cheat ] && { - rm -f /usr/local/bin/cheat - ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat - } - curl -sL https://cheat.sh/:bash_completion > /etc/bash_completion.d/cht.sh -} - wo_clean_repo() { # remove old EasyEngine Nginx repository if [ -f /etc/apt/sources.list.d/ee-repo.list ]; then cp -f /etc/apt/sources.list.d/ee-repo.list /etc/apt/sources.list.d/ee-repo.list.save - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save > /etc/apt/sources.list.d/ee-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save >/etc/apt/sources.list.d/ee-repo.list fi if [ -f /etc/apt/sources.list.d/wo-repo.list ]; then local wo_linux_distro wo_linux_distro=$(lsb_release -is) cp -f /etc/apt/sources.list.d/wo-repo.list /etc/apt/sources.list.d/wo-repo.list.save if [ "$wo_linux_distro" = "Ubuntu" ]; then - grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list else - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list fi fi } @@ -709,26 +632,31 @@ wo_init() { if [ -z "$wo_travis" ]; then if ! { - apt-get update --allow-releaseinfo-change -qq > /dev/null 2>&1 + apt-get update --allow-releaseinfo-change -qq >/dev/null 2>&1 }; then - apt-get update -qq > /dev/null 2>&1 + apt-get update -qq >/dev/null 2>&1 fi if ! command_exists curl; then - apt-get -y install curl -qq > /dev/null 2>&1 + apt-get -y install curl -qq >/dev/null 2>&1 fi if ! command_exists lsb_release; then - apt-get install lsb-release -qq + apt-get install lsb-release -qq > /dev/null 2>&1 + fi + if ! command_exists jq; then + apt-get install jq -qq > /dev/null 2>&1 fi fi - + if [ "$wo_force_install" = "y" ]; then + [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } + fi if [ -f ./setup.py ]; then - readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print$2}' 2>&1) + readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print $2}' 2>&1) else - readonly wo_version_new=$(curl -sL https://wops.cc/setup.py 2>&1 | grep "version='" | awk -F "'" '{print$2}' 2>&1) + readonly wo_version_new=$(curl -m 5 --retry 3 -sL https://api.github.com/repos/WordOps/WordOps/releases/latest 2>&1 | jq -r '.tag_name' ) fi echo "" - wo_lib_echo "Welcome to WordOps install/update script v${wo_version_new}" + wo_lib_echo "Welcome to WordOps install/update script ${wo_version_new}" echo "" } @@ -737,6 +665,17 @@ wo_init() { # 4 - WO MAIN SETUP ### +# create required directories +wo_dir_init +# install lsb_release, curl and display header +wo_init +# define main variables +wo_init_variables +# remove old repositories +_run wo_clean_repo +# check distribution support +wo_check_distro + # wo uninstall script if [ "$wo_purge" = "y" ]; then _run wo_backup_wo "Backing-up WO install" @@ -744,18 +683,8 @@ if [ "$wo_purge" = "y" ]; then wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else - wo_clean_repo - wo_init - wo_check_distro - wo_dir_init # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then - if [ -z "$wo_force_install" ]; then - if { wo -v 2>&1 | grep -q "$wo_version_new"; }; then - wo_lib_error "You already have WordOps $wo_version_new" 1 - fi - fi - _run wo_backup_wo "Backing-up WO install" _run wo_clean # 2 - Migration from EEv3 elif [ -x /usr/local/bin/ee ]; then @@ -767,14 +696,13 @@ else fi _run wo_backup_ee "Backing-up EE install" _run wo_remove_ee_cron "Removing EasyEngine cronjob" - _run wo_sync_db "Syncing WO database" fi _run wo_install_dep "Installing wo dependencies" _run wo_timesync # skip steps if travis if [ -z "$wo_travis" ]; then - _run wo_download "Downloading WordOps" - wo_git_config + #_run wo_download "Downloading WordOps" + _run wo_sync_db _run wo_install "Installing WordOps" else _run wo_travis_install "Installing WordOps" @@ -783,24 +711,13 @@ else _run wo_upgrade_nginx "Upgrading Nginx" _run wo_clean_ee "Cleaning previous EasyEngine install" fi - _run wo_install_acme_sh - _run wo_tweak_kernel "Applying Kernel tweaks" - if [ ! -f /opt/wo-kernel.sh ]; then - _run wo_systemd_tweak "Adding systemd service tweak" - fi - if [ -x /usr/sbin/nginx ]; then - _run wo_nginx_tweak - fi - if [ -d /etc/systemd/system/mariadb.service.d ]; then - _run wo_mariadb_tweak - fi - _run wo_cheat_install "Running post-install steps" + _run wo_install_acme_sh "Running post-install steps" _run wo_domain_suffix _run wo_update_wp_cli _run wo_update_latest _run secure_wo_db - wo sync >> $wo_install_log 2>&1 + wo sync >>$wo_install_log 2>&1 if [ "$ee_migration" = "1" ]; then echo @@ -810,10 +727,10 @@ else elif [ "$wo_upgrade" = "1" ]; then wo_lib_echo "WordOps (wo) upgrade to $wo_version_new was succesfull!" echo - wo_lib_echo "To upgrade WordOps stacks use the command:" - wo_lib_echo_info "wo stack upgrade --all" + wo_lib_echo "To upgrade WordOps web stacks use the command:" + wo_lib_echo_info "wo stack upgrade" echo - wo_lib_echo "To update all other server packages use the command:" + wo_lib_echo "To update all other packages use the command:" wo_lib_echo_info "wo maintenance" else wo_lib_echo "WordOps (wo) installed successfully" @@ -830,6 +747,7 @@ else echo wo_lib_echo "WordOps Documentation : https://docs.wordops.net" wo_lib_echo "WordOps Community Forum : https://community.wordops.net" + wo_lib_echo "WordOps Community Chat : https://chat.wordops.net" echo wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/" echo diff --git a/setup.cfg b/setup.cfg index 4ee5196..514f780 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,4 +12,17 @@ cover-html-dir=coverage_report/ where=tests/ [metadata] -license-file = LICENSE \ No newline at end of file +license-file = LICENSE + +[flake8] +ignore = F405,W504,S322,S404,S603,s607,s602 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist +max-complexity = 10 \ No newline at end of file diff --git a/setup.py b/setup.py index 7c960cf..21a9b89 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ import glob import os +import sys from setuptools import find_packages, setup - # read the contents of your README file this_directory = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: @@ -27,8 +27,8 @@ if os.geteuid() == 0: os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.9.9.4', - description='WordPress & server administration toolset', + version='3.10.0', + description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', classifiers=[ diff --git a/snapcraft.yaml b/snapcraft.yaml deleted file mode 100644 index eb46364..0000000 --- a/snapcraft.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: test-wordops -version: git -summary: WordOps -description: | - WordOps is an essential toolset that eases WordPress - site and server administration. It provide the ability - to install a high performance WordPress stack - with a few keystrokes. -confinement: devmode -base: core18 - -parts: - test-wordops: - plugin: python - python-version: python3 - source: . - stage-packages: - - cement - - python-apt - -apps: - test-wordops: - command: wo diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index 63939cb..fbecf58 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -7,64 +7,43 @@ class CliTestCaseStack(test.WOTestCase): def test_wo_cli(self): self.app.setup() self.app.run() - self.app.close() - - def test_wo_cli_stack_install(self): - self.app = get_test_app(argv=['stack', 'install']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_web(self): - self.app = get_test_app(argv=['stack', 'install', '--web']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_admin(self): - self.app = get_test_app(argv=['stack', 'install', '--admin']) - self.app.setup() - self.app.run() - self.app.close() def test_wo_cli_stack_install_nginx(self): self.app = get_test_app(argv=['stack', 'install', '--nginx']) self.app.setup() self.app.run() - self.app.close() def test_wo_cli_stack_install_php(self): self.app = get_test_app(argv=['stack', 'install', '--php']) self.app.setup() self.app.run() - self.app.close() + + def test_wo_cli_stack_install_php73(self): + self.app = get_test_app(argv=['stack', 'install', '--php73']) + self.app.setup() + self.app.run() def test_wo_cli_stack_install_mysql(self): self.app = get_test_app(argv=['stack', 'install', '--mysql']) self.app.setup() self.app.run() - self.app.close() def test_wo_cli_stack_install_wpcli(self): self.app = get_test_app(argv=['stack', 'install', '--wpcli']) self.app.setup() self.app.run() - self.app.close() def test_wo_cli_stack_install_phpmyadmin(self): self.app = get_test_app(argv=['stack', 'install', '--phpmyadmin']) self.app.setup() self.app.run() - self.app.close() def test_wo_cli_stack_install_adminer(self): self.app = get_test_app(argv=['stack', 'install', '--adminer']) self.app.setup() self.app.run() - self.app.close() def test_wo_cli_stack_install_utils(self): self.app = get_test_app(argv=['stack', 'install', '--utils']) self.app.setup() self.app.run() - self.app.close() diff --git a/tests/cli/plugins/test_example.py b/tests/cli/plugins/test_example.py index 0f9d21d..dc101f8 100644 --- a/tests/cli/plugins/test_example.py +++ b/tests/cli/plugins/test_example.py @@ -1,9 +1,9 @@ """Tests for Example Plugin.""" -from wo.utils import test +from wo.utils.test import WOTestCase -class ExamplePluginTestCase(test.WOTestCase): +class ExamplePluginTestCase(WOTestCase): def test_load_example_plugin(self): self.app.setup() self.app.plugin.load_plugin('example') diff --git a/tests/travis.sh b/tests/travis.sh index 3391d08..6ebaa72 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -28,7 +28,7 @@ exit_script() { echo -e "${CGREEN}#############################################${CEND}" echo -e ' stack install ' echo -e "${CGREEN}#############################################${CEND}" -stack_list='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw ngxblocker' +stack_list='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw ngxblocker cheat' for stack in $stack_list; do echo -ne " Installing $stack [..]\r" if { @@ -145,7 +145,7 @@ if [ -z "$1" ]; then echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo stack upgrade ' echo -e "${CGREEN}#############################################${CEND}" - stack_upgrade='nginx php php73 mysql redis netdata dashboard phpmyadmin composer' + stack_upgrade='nginx php php73 mysql redis netdata dashboard phpmyadmin composer ngxblocker' for stack in $stack_upgrade; do echo -ne " Upgrading $stack [..]\r" if { @@ -225,11 +225,12 @@ echo -e ' various informations ' echo -e "${CGREEN}#############################################${CEND}" wp --allow-root --info wo site info wp.net +wo info echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo stack purge ' echo -e "${CGREEN}#############################################${CEND}" -stack_purge='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis ufw ngxblocker' +stack_purge='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis ufw ngxblocker cheat' for stack in $stack_purge; do echo -ne " purging $stack [..]\r" if { @@ -244,6 +245,3 @@ for stack in $stack_purge; do fi done -if [ -n "$1" ]; then - cat /var/log/wo/test.log | ccze -A -p syslog -fi diff --git a/wo/cli/main.py b/wo/cli/main.py index dad8f1c..fc80c05 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -75,60 +75,49 @@ class WOTestApp(WOApp): class Meta: argv = [] config_files = [] + exit_on_close = True # Define the applicaiton object outside of main, as some libraries might wish # to import it as a global (rather than passing it into another class/func) -app = WOApp() +# app = WOApp() def main(): - try: - global sys - # Default our exit status to 0 (non-error) - code = 0 + with WOApp() as app: + try: + global sys - # if not root...kick out - if not os.geteuid() == 0: - print("\nNon-privileged users cant use WordOps. " - "Switch to root or invoke sudo.\n") - app.close(1) + # if not root...kick out + if not os.geteuid() == 0: + print("\nNon-privileged users cant use WordOps. " + "Switch to root or invoke sudo.\n") + app.close(1) + app.run() + except AssertionError as e: + print("AssertionError => %s" % e.args[0]) + app.exit_code = 1 + except exc.WOError as e: + # Catch our application errors and exit 1 (error) + print(e) + app.exit_code = 1 + except FrameworkError as e: + # Catch framework errors and exit 1 (error) + print('FrameworkError > %s' % e) + app.exit_code = 1 + except CaughtSignal as e: + # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error) + print('CaughtSignal > %s' % e) + app.exit_code = 0 + finally: + # Print an exception (if it occurred) and --debug was passed + if app.debug: + import sys + import traceback - # Setup the application - app.setup() - - # Dump all arguments into wo log - app.log.debug(sys.argv) - - # Run the application - app.run() - except exc.WOError as e: - # Catch our application errors and exit 1 (error) - code = 1 - print(e) - except CaughtSignal as e: - # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error) - code = 0 - print(e) - except FrameworkError as e: - # Catch framework errors and exit 1 (error) - code = 1 - print(e) - except Exception as e: - code = 1 - print(e) - finally: - # Print an exception (if it occurred) and --debug was passed - if app.debug: - import sys - import traceback - - exc_type, exc_value, exc_traceback = sys.exc_info() - if exc_traceback is not None: - traceback.print_exc() - - # # Close the application - app.close(code) + exc_type, exc_value, exc_traceback = sys.exc_info() + if exc_traceback is not None: + traceback.print_exc() def get_test_app(**kw): diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index b866fc4..3385876 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -1,7 +1,7 @@ """Clean Plugin for WordOps.""" import os -import urllib.request +import requests from cement.core.controller import CementBaseController, expose @@ -74,8 +74,10 @@ class WOCleanController(CementBaseController): def clean_opcache(self): try: Log.info(self, "Cleaning opcache") - urllib.request.urlopen("https://127.0.0.1:22222/cache" - "/opcache/opgui.php?reset=1").read() + opgui = requests.get( + "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1") + if opgui.status_code != '200': + Log.warn(self, 'Cleaning opcache failed') except Exception as e: Log.debug(self, "{0}".format(e)) Log.debug(self, "Unable hit url, " diff --git a/wo/cli/plugins/debug.py b/wo/cli/plugins/debug.py index 1cb171e..ed38c5d 100644 --- a/wo/cli/plugins/debug.py +++ b/wo/cli/plugins/debug.py @@ -14,7 +14,7 @@ from wo.core.fileutils import WOFileUtils from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.services import WOService -from wo.core.shellexec import WOShellExec +from wo.core.shellexec import WOShellExec, CommandExecutionError from wo.core.variables import WOVar @@ -703,8 +703,7 @@ class WODebugController(CementBaseController): "-l | sed '/WordOps " "start MySQL slow " "log/,+2d'" - "| crontab -\"" - .format(cron_time)): + "| crontab -\""): Log.error(self, "failed to remove crontab entry") except CommandExecutionError as e: Log.debug(self, str(e)) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index cb505fe..602a6ce 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -78,21 +78,31 @@ class WOInfoController(CementBaseController): upload_max_filesize = config['PHP']['upload_max_filesize'] max_execution_time = config['PHP']['max_execution_time'] - config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - www_listen = config['www-php72']['listen'] - www_ping_path = config['www-php72']['ping.path'] - www_pm_status_path = config['www-php72']['pm.status_path'] - www_pm = config['www-php72']['pm'] - www_pm_max_requests = config['www-php72']['pm.max_requests'] - www_pm_max_children = config['www-php72']['pm.max_children'] - www_pm_start_servers = config['www-php72']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php72']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php72']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php72'] - ['request_terminate_timeout']) + if os.path.exists('/etc/php/7.2/fpm/pool.d/www.conf'): + config.read('/etc/php/7.2/fpm/pool.d/www.conf') + else: + Log.error(self, 'php-fpm pool config not found') + if config.has_section('www'): + wconfig = config['www'] + elif config.has_section('www-php72'): + wconfig = config['www-php72'] + else: + Log.error(self, 'Unable to parse configuration') + www_listen = wconfig['listen'] + www_ping_path = wconfig['ping.path'] + www_pm_status_path = wconfig['pm.status_path'] + www_pm = wconfig['pm'] + www_pm_max_requests = wconfig['pm.max_requests'] + www_pm_max_children = wconfig['pm.max_children'] + www_pm_start_servers = wconfig['pm.start_servers'] + www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] + www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] + www_request_terminate_time = (wconfig + ['request_terminate_timeout']) try: - www_xdebug = (config['www-php72']['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + www_xdebug = ( + wconfig['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -155,20 +165,29 @@ class WOInfoController(CementBaseController): upload_max_filesize = config['PHP']['upload_max_filesize'] max_execution_time = config['PHP']['max_execution_time'] - config.read('/etc/php/7.3/fpm/pool.d/www.conf') - www_listen = config['www-php73']['listen'] - www_ping_path = config['www-php73']['ping.path'] - www_pm_status_path = config['www-php73']['pm.status_path'] - www_pm = config['www-php73']['pm'] - www_pm_max_requests = config['www-php73']['pm.max_requests'] - www_pm_max_children = config['www-php73']['pm.max_children'] - www_pm_start_servers = config['www-php73']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php73']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php73']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php73'] - ['request_terminate_timeout']) + if os.path.exists('/etc/php/7.3/fpm/pool.d/www.conf'): + config.read('/etc/php/7.3/fpm/pool.d/www.conf') + else: + Log.error(self, 'php-fpm pool config not found') + if config.has_section('www'): + wconfig = config['www'] + elif config.has_section('www-php73'): + wconfig = config['www-php73'] + else: + Log.error(self, 'Unable to parse configuration') + www_listen = wconfig['listen'] + www_ping_path = wconfig['ping.path'] + www_pm_status_path = wconfig['pm.status_path'] + www_pm = wconfig['pm'] + www_pm_max_requests = wconfig['pm.max_requests'] + www_pm_max_children = wconfig['pm.max_children'] + www_pm_start_servers = wconfig['pm.start_servers'] + www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] + www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] + www_request_terminate_time = (wconfig + ['request_terminate_timeout']) try: - www_xdebug = (config['www-php73'] + www_xdebug = (wconfig ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: @@ -265,11 +284,11 @@ class WOInfoController(CementBaseController): self.app.pargs.php73 = True if self.app.pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-wo')): - self.info_nginx() - else: + if ((not WOAptGet.is_installed(self, 'nginx-custom')) and + (not os.path.exists('/usr/bin/nginx'))): Log.error(self, "Nginx is not installed") + else: + self.info_nginx() if self.app.pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index 68f9974..f108f3c 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -46,7 +46,7 @@ class WOLogShowController(CementBaseController): (['--php'], dict(help='Show PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='Show PHP5-fpm slow logs file', + dict(help='Show PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='Show MySQL logs file', action='store_true')), @@ -92,10 +92,10 @@ class WOLogShowController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -171,7 +171,7 @@ class WOLogResetController(CementBaseController): (['--php'], dict(help='Reset PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='Reset PHP5-fpm slow logs file', + dict(help='Reset PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='Reset MySQL logs file', action='store_true')), @@ -210,7 +210,7 @@ class WOLogResetController(CementBaseController): if ((not self.app.pargs.nginx) and (not self.app.pargs.fpm) and (not self.app.pargs.mysql) and (not self.app.pargs.access) and (not self.app.pargs.wp) and (self.app.pargs.site_name) and - (not self.app.pargs.slow-log-db)): + (not self.app.pargs.slow_log_db)): self.app.pargs.nginx = True self.app.pargs.wp = True self.app.pargs.access = True @@ -231,10 +231,10 @@ class WOLogResetController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -313,7 +313,7 @@ class WOLogGzipController(CementBaseController): (['--php'], dict(help='GZip PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='GZip PHP5-fpm slow logs file', + dict(help='GZip PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='GZip MySQL logs file', action='store_true')), @@ -359,10 +359,10 @@ class WOLogGzipController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -497,10 +497,10 @@ class WOLogMailController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index d293c3b..cb65342 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -78,7 +78,7 @@ class WOSiteController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exist") + Log.error(self, 'nginx configuration file does not exist') @expose(help="Disable site example.com") def disable(self): @@ -106,7 +106,7 @@ class WOSiteController(CementBaseController): if not os.path.isfile('/etc/nginx/sites-enabled/{0}' .format(wo_domain)): Log.debug(self, "Site {0} already disabled".format(wo_domain)) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE + "]") else: WOFileUtils.remove_symlink(self, '/etc/nginx/sites-enabled/{0}' @@ -176,7 +176,7 @@ class WOSiteController(CementBaseController): dbname=wo_db_name, dbuser=wo_db_user, php_version=php_version, dbpass=wo_db_pass, - ssl=ssl, sslprovider=sslprovider, sslexpiry=sslexpiry, + ssl=ssl, sslprovider=sslprovider, sslexpiry=sslexpiry, type=sitetype + " " + cachetype + " ({0})" .format("enabled" if siteinfo.is_enabled else "disabled")) @@ -227,8 +227,7 @@ class WOSiteController(CementBaseController): Log.info(self, Log.ENDC + text) f.close() else: - Log.error(self, "nginx configuration file does not exists" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exists") @expose(help="Change directory to site webroot") def cd(self): @@ -309,8 +308,7 @@ class WOSiteEditController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exists" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exists") class WOSiteCreateController(CementBaseController): @@ -453,7 +451,7 @@ class WOSiteCreateController(CementBaseController): if stype == 'proxy': data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=True, basic=False, php73=False, wp=False, + static=True, basic=False, php73=False, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -464,7 +462,7 @@ class WOSiteCreateController(CementBaseController): if pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=False, php73=True, wp=False, + static=False, basic=False, php73=True, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -472,7 +470,7 @@ class WOSiteCreateController(CementBaseController): if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=True, basic=False, php73=False, wp=False, + static=True, basic=False, php73=False, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -484,7 +482,7 @@ class WOSiteCreateController(CementBaseController): elif stype in ['mysql', 'wp', 'wpsubdir', 'wpsubdomain']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, @@ -510,8 +508,7 @@ class WOSiteCreateController(CementBaseController): elif data: data['php73'] = False - if ((not pargs.wpfc) and - (not pargs.wpsc) and + if ((not pargs.wpfc) and (not pargs.wpsc) and (not pargs.wprocket) and (not pargs.wpce) and (not pargs.wpredis)): @@ -640,10 +637,7 @@ class WOSiteCreateController(CementBaseController): # Setup WordPress if Wordpress site if data['wp']: - if pargs.vhostonly: - vhostonly = True - else: - vhostonly = False + vhostonly = bool(pargs.vhostonly) try: wo_wp_creds = setupwordpress(self, data, vhostonly) # Add database information for site into database @@ -725,7 +719,7 @@ class WOSiteCreateController(CementBaseController): Log.info(self, "Successfully created site" " http://{0}".format(wo_domain)) - except SiteError as e: + except SiteError: Log.error(self, "Check the log for details: " "`tail /var/log/wo/wordops.log` and please try again") @@ -905,8 +899,9 @@ class WOSiteUpdateController(CementBaseController): choices=('on', 'off'), const='on', nargs='?')), (['--ngxblocker'], - dict(help="enable HSTS for site secured with letsencrypt", + dict(help="enable Ultimate Nginx bad bot blocker", action='store' or 'store_const', + choices=('on', 'off'), const='on', nargs='?')), (['--proxy'], dict(help="update to proxy site", nargs='+')), @@ -1004,10 +999,7 @@ class WOSiteUpdateController(CementBaseController): check_ssl = check_site.is_ssl check_php_version = check_site.php_version - if check_php_version == "7.3": - old_php73 = True - else: - old_php73 = False + old_php73 = bool(check_php_version == "7.3") if (pargs.password and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or @@ -1069,23 +1061,40 @@ class WOSiteUpdateController(CementBaseController): else: Log.error(self, 'ngxblocker stack is not installed') elif pargs.ngxblocker == "off": - if os.path.isfile( - '/var/www/{0}/conf/nginx/ngxblocker.conf' - .format(wo_domain)): - WOFileUtils.mvfile(self, '/var/www/{0}/conf/' - 'nginx/ngxblocker.conf' - .format(wo_domain), - '/var/www/{0}/conf/' - 'nginx/ngxblocker.conf.disabled' - .format(wo_domain)) - else: - Log.error(self, "ngxblocker isn't enabled") + try: + setupngxblocker(self, wo_domain, False) + except SiteError as e: + Log.debug(self, str(e)) + Log.info(self, "\nngxblocker not enabled.") # Service Nginx Reload if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") return 0 + # + if (pargs.letsencrypt == 'renew' and + not (pargs.html or + pargs.php or pargs.php73 or pargs.mysql or + pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpce or + pargs.wpsubdir or pargs.wpsubdomain or + pargs.ngxblocker or pargs.hsts)): + + if WOAcme.cert_check(self, wo_domain): + if not pargs.force: + if (SSL.getexpirationdays(self, wo_domain) > 30): + Log.error( + self, "Your cert will expire in more " + "than 30 days ( " + + str(SSL.getexpirationdays(self, wo_domain)) + + " days).\nAdd \'--force\' to force to renew") + Log.wait(self, "Renewing SSL certificate") + if WOAcme.renew(self, wo_domain): + Log.valide(self, "Renewing SSL certificate") + else: + Log.error(self, "Certificate doesn't exist") + return 0 if ((stype == 'php' and oldsitetype not in ['html', 'proxy', 'php73']) or @@ -1114,7 +1123,7 @@ class WOSiteUpdateController(CementBaseController): if stype == 'php': data = dict( site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, currsitetype=oldsitetype, currcachetype=oldcachetype) @@ -1123,7 +1132,7 @@ class WOSiteUpdateController(CementBaseController): data = dict( site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', @@ -1150,7 +1159,7 @@ class WOSiteUpdateController(CementBaseController): stype = oldsitetype cache = oldcachetype if oldsitetype == 'html' or oldsitetype == 'proxy': - data['static'] = True + data['static'] = False data['wp'] = False data['multisite'] = False data['wpsubdir'] = False @@ -1246,10 +1255,7 @@ class WOSiteUpdateController(CementBaseController): if pargs.letsencrypt == 'on': data['letsencrypt'] = True letsencrypt = True - if (wo_domain_type == 'subdomain'): - acme_subdomain = True - else: - acme_subdomain = False + acme_subdomain = bool(wo_domain_type == 'subdomain') acme_wildcard = False elif pargs.letsencrypt == 'subdomain': data['letsencrypt'] = True @@ -1277,14 +1283,19 @@ class WOSiteUpdateController(CementBaseController): letsencrypt = False acme_subdomain = False acme_wildcard = False + else: + data['letsencrypt'] = False + letsencrypt = False + acme_subdomain = False + acme_wildcard = False if not (acme_subdomain is True): if letsencrypt is check_ssl: if letsencrypt is False: - Log.error(self, "SSl is not configured for given " + Log.error(self, "SSL is not configured for given " "site") elif letsencrypt is True: - Log.error(self, "SSl is already configured for given " + Log.error(self, "SSL is already configured for given " "site") pargs.letsencrypt = False @@ -1296,19 +1307,11 @@ class WOSiteUpdateController(CementBaseController): return 0 if data and (not pargs.php73): - if old_php73 is True: - data['php73'] = True - php73 = True - else: - data['php73'] = False - php73 = False + data['php73'] = bool(old_php73 is True) + php73 = bool(old_php73 is True) - if pargs.php73 == "on": - data['php73'] = True - php73 = True - else: - data['php73'] = False - php73 = False + data['php73'] = bool(pargs.php73 == "on") + php73 = bool(pargs.php73 == "on") if pargs.wpredis and data['currcachetype'] != 'wpredis': data['wpredis'] = True @@ -1330,16 +1333,10 @@ class WOSiteUpdateController(CementBaseController): return 1 if pargs.hsts: - if pargs.hsts == "on": - data['hsts'] = True - elif pargs.hsts == "off": - data['hsts'] = False + data['hsts'] = bool(pargs.hsts == "on") if pargs.ngxblocker: - if pargs.ngxblocker == 'on': - ngxblocker = True - elif pargs.ngxblocker == 'off': - ngxblocker = False + ngxblocker = bool(pargs.ngxblocker == 'on') if not data: Log.error(self, "Cannot update {0}, Invalid Options" @@ -1378,7 +1375,7 @@ class WOSiteUpdateController(CementBaseController): if 'proxy' in data.keys() and data['proxy']: updateSiteInfo(self, wo_domain, stype=stype, cache=cache, - ssl=True if check_site.is_ssl else False) + ssl=(bool(check_site.is_ssl))) Log.info(self, "Successfully updated site" " http://{0}".format(wo_domain)) return 0 @@ -1521,7 +1518,7 @@ class WOSiteUpdateController(CementBaseController): elif (pargs.letsencrypt == "clean" or pargs.letsencrypt == "purge"): - removeAcmeConf(self, wo_domain) + WOAcme.removeconf(self, wo_domain) # find all broken symlinks sympath = "/var/www" WOFileUtils.findBrokenSymlink(self, sympath) @@ -1927,7 +1924,7 @@ class WOSiteDeleteController(CementBaseController): dict(help="forcefully delete site and configuration", action='store_true')), (['--all'], - dict(help="delete all", action='store_true')), + dict(help="delete files & db", action='store_true')), (['--db'], dict(help="delete db only", action='store_true')), (['--files'], @@ -1938,7 +1935,7 @@ class WOSiteDeleteController(CementBaseController): @expose(hide=True) def default(self): pargs = self.app.pargs - if not pargs.site_name: + if not pargs.site_name and not pargs.all: try: while not pargs.site_name: pargs.site_name = (input('Enter site name : ') @@ -1964,6 +1961,9 @@ class WOSiteDeleteController(CementBaseController): (not pargs.all)): pargs.all = True + if pargs.force: + pargs.no_prompt = True + # Gather information from wo-db for wo_domain check_site = getSiteInfo(self, wo_domain) wo_site_type = check_site.site_type diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 4251c7a..a592e7b 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -366,45 +366,36 @@ def setupwordpress(self, data, vhostonly=False): except CommandExecutionError: raise SiteError("generate wp-config failed for wp multi site") - try: + # set all wp-config.php variables + wp_conf_variables = [ + ['WP_CACHE_KEY_SALT', '{0}:'.format(wo_domain_name)], + ['WP_MEMORY_LIMIT', '128M'], + ['WP_MAX_MEMORY_LIMIT', '256M'], + ['CONCATENATE_SCRIPTS', 'false'], + ['WP_POST_REVISIONS', '10'], + ['MEDIA_TRASH', 'true'], + ['EMPTY_TRASH_DAYS', '15'], + ['WP_AUTO_UPDATE_CORE', 'minor']] - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_CACHE_KEY_SALT " - "\'{0}:\'\"".format(wo_domain_name)) - - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_MEMORY_LIMIT " - "\'128M\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_MAX_MEMORY_LIMIT " - "\'256M\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set CONCATENATE_SCRIPTS " - "false --raw\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_POST_REVISIONS " - "\'10\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set MEDIA_TRASH " - "true --raw\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set EMPTY_TRASH_DAYS " - "\'15\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_AUTO_UPDATE_CORE " - "minor\"") - - except CommandExecutionError as e: - Log.debug(self, str(e)) - Log.error(self, "Unable to define extra variable in wp-config.php") + for wp_conf in wp_conf_variables: + wp_var = wp_conf[0] + wp_val = wp_conf[1] + if wp_val == 'true' or wp_val == 'false': + var_raw = True + else: + var_raw = False + try: + WOShellExec.cmd_exec( + self, "/bin/bash -c \"{0} --allow-root " + .format(WOVar.wo_wpcli_path) + + "config set {0} " + "\'{1}\' {wp_raw}\"" + .format(wp_var, wp_val, + wp_raw='--raw' + if var_raw is True else '')) + except CommandExecutionError as e: + Log.debug(self, str(e)) + Log.error(self, 'Unable to define wp-config.php variables') # WOFileUtils.mvfile(self, os.getcwd()+'/wp-config.php', # os.path.abspath(os.path.join(os.getcwd(), os.pardir))) @@ -412,14 +403,14 @@ def setupwordpress(self, data, vhostonly=False): try: Log.debug(self, "Moving file from {0} to {1}".format(os.getcwd( - )+'/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), - os.pardir)))) - shutil.move(os.getcwd()+'/wp-config.php', + ) + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), + os.pardir)))) + shutil.move(os.getcwd() + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), os.pardir))) except Exception as e: Log.debug(self, str(e)) Log.error(self, 'Unable to move file from {0} to {1}' - .format(os.getcwd()+'/wp-config.php', + .format(os.getcwd() + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), os.pardir))), False) raise SiteError("Unable to move wp-config.php") @@ -458,7 +449,8 @@ def setupwordpress(self, data, vhostonly=False): if not data['multisite']: Log.debug(self, "Creating tables for WordPress Single site") - Log.debug(self, "{0} --allow-root core install " + Log.debug( + self, "{0} --allow-root core install " .format(WOVar.wo_wpcli_path) + "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " .format(data['site_name'], wo_wp_user) + @@ -605,6 +597,11 @@ def setupwordpress(self, data, vhostonly=False): plugin_data = json.dumps(plugin_data_object) setupwp_plugin(self, 'cache-enabler', 'cache-enabler', plugin_data, data) + WOShellExec.cmd_exec( + self, "/bin/bash -c \"{0} --allow-root " + .format(WOVar.wo_wpcli_path) + + "config set WP_CACHE " + "true --raw\"") if vhostonly: try: @@ -1274,62 +1271,6 @@ def removeNginxConf(self, domain): .format(domain)) -def removeAcmeConf(self, domain): - sslconf = ("/var/www/{0}/conf/nginx/ssl.conf" - .format(domain)) - sslforce = ("/etc/nginx/conf.d/force-ssl-{0}.conf" - .format(domain)) - if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' - .format(domain)): - Log.info(self, "Removing Acme configuration") - Log.debug(self, "Removing Acme configuration") - try: - WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " - "--remove " - "-d {0} --ecc" - .format(domain)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Cert removal failed") - - WOFileUtils.rm(self, '{0}/{1}_ecc' - .format(WOVar.wo_ssl_archive, domain)) - WOFileUtils.rm(self, '{0}/{1}' - .format(WOVar.wo_ssl_live, domain)) - WOFileUtils.rm(self, '{0}'.format(sslconf)) - WOFileUtils.rm(self, '{0}.disabled'.format(sslconf)) - WOFileUtils.rm(self, '{0}'.format(sslforce)) - WOFileUtils.rm(self, '{0}.disabled' - .format(sslforce)) - WOFileUtils.rm(self, '/etc/letsencrypt/shared/{0}.conf' - .format(domain)) - - # find all broken symlinks - sympath = "/var/www" - WOFileUtils.findBrokenSymlink(self, sympath) - - else: - if os.path.islink("{0}".format(sslconf)): - WOFileUtils.remove_symlink(self, "{0}".format(sslconf)) - WOFileUtils.rm(self, '{0}'.format(sslforce)) - - if WOFileUtils.grepcheck(self, '/var/www/22222/conf/nginx/ssl.conf', - '{0}'.format(domain)): - Log.info(self, "Setting back default certificate for WordOps backend") - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "w") as ssl_conf_file: - ssl_conf_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") - WOGit.add(self, ["/etc/letsencrypt"], - msg="Deleted {0} " - .format(domain)) - WOService.restart_service(self, "nginx") - - def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', dbhost=''): """ @@ -1341,7 +1282,7 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', if os.path.isfile('/etc/nginx/sites-available/{0}' .format(domain)): removeNginxConf(self, domain) - removeAcmeConf(self, domain) + WOAcme.removeconf(self, domain) if webroot: deleteWebRoot(self, webroot) @@ -1610,19 +1551,31 @@ def setuprocketchat(self): def setupngxblocker(self, domain, block=True): - if os.path.isdir('/var/www/{0}/conf/nginx'.format(domain)): - if not os.path.isfile('/var/www/{0}/conf/nginx/ngxblocker.disabled' - .format(domain)): - ngxconf = open("/var/www/{0}/conf/nginx/ngxblocker.conf" - .format(domain), - encoding='utf-8', mode='w') - ngxconf.write("# Bad Bot Blocker\n" - "include /etc/nginx/bots.d/ddos.conf;\n" - "include /etc/nginx/bots.d/blockbots.conf;\n") - ngxconf.close() - else: + if block: + if os.path.isdir('/var/www/{0}/conf/nginx'.format(domain)): + if not os.path.isfile( + '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' + .format(domain)): + ngxconf = open( + "/var/www/{0}/conf/nginx/ngxblocker.conf" + .format(domain), + encoding='utf-8', mode='w') + ngxconf.write( + "# Bad Bot Blocker\n" + "include /etc/nginx/bots.d/ddos.conf;\n" + "include /etc/nginx/bots.d/blockbots.conf;\n") + ngxconf.close() + else: + WOFileUtils.mvfile( + self, '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' + .format(domain), '/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain)) + else: + if os.path.isfile('/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain)): WOFileUtils.mvfile( - self, '/var/www/{0}/conf/nginx/ngxblocker.disabled' - .format(domain), '/var/www/{0}/conf/nginx/ngxblocker' + self, '/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain), + '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' .format(domain)) return 0 diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 8dacd32..b21257e 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -5,7 +5,7 @@ import os from cement.core.controller import CementBaseController, expose from wo.cli.plugins.stack_migrate import WOStackMigrateController -from wo.cli.plugins.stack_pref import post_pref, pre_pref +from wo.cli.plugins.stack_pref import post_pref, pre_pref, pre_stack from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController from wo.core.aptget import WOAptGet @@ -85,6 +85,8 @@ class WOStackController(CementBaseController): (['--ngxblocker'], dict(help='Install Nginx Ultimate Bad Bot Blocker', action='store_true')), + (['--cheat'], + dict(help='Install cheat.sh', action='store_true')), (['--force'], dict(help='Force install/remove/purge without prompt', action='store_true')), @@ -116,6 +118,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -147,6 +150,7 @@ class WOStackController(CementBaseController): pargs.dashboard = True pargs.phpredisadmin = True pargs.extplorer = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -426,6 +430,7 @@ class WOStackController(CementBaseController): Log.debug(self, "eXtplorer is already installed") Log.info(self, "eXtplorer is already installed") + # ultimate ngx_blocker if pargs.ngxblocker: if not os.path.isdir('/etc/nginx/bots.d'): Log.debug(self, "Setting packages variable for ngxblocker") @@ -439,6 +444,21 @@ class WOStackController(CementBaseController): Log.debug(self, "ngxblocker is already installed") Log.info(self, "ngxblocker is already installed") + # cheat.sh + if pargs.cheat: + if ((not os.path.exists('/usr/local/bin/cht.sh')) and + (not os.path.exists('/usr/bin/cht.sh'))): + Log.debug(self, 'Setting packages variable for cheat.sh') + packages = packages + [[ + "https://raw.githubusercontent.com/chubin/cheat.sh" + "/master/share/cht.sh.txt", + "/usr/local/bin/cht.sh", + "cheat.sh"], + ["https://raw.githubusercontent.com/chubin/cheat.sh" + "/master/share/bash_completion.txt", + "/etc/bash_completion.d/cht.sh", + "bash_completion"]] + # UTILS if pargs.utils: Log.debug(self, "Setting packages variable for utils") @@ -485,6 +505,7 @@ class WOStackController(CementBaseController): Log.debug(self, "{0}".format(e)) if (apt_packages) or (packages): + pre_stack(self) if (apt_packages): Log.debug(self, "Calling pre_pref") pre_pref(self, apt_packages) @@ -518,6 +539,7 @@ class WOStackController(CementBaseController): Log.info(self, "Successfully installed packages") else: return self.msg + return 0 @expose(help="Remove packages") def remove(self): @@ -535,6 +557,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -564,6 +587,7 @@ class WOStackController(CementBaseController): pargs.utils = True pargs.netdata = True pargs.mysqltuner = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -673,6 +697,14 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing packages for MySQLTuner ") packages = packages + ['/usr/bin/mysqltuner'] + # cheat.sh + if pargs.cheat: + if os.path.isfile('/usr/local/bin/cht.sh'): + Log.debug(self, "Removing packages for cheat.sh ") + packages = packages + [ + '/usr/local/bin/cht.sh', '/usr/local/bin/cheat', + '/etc/bash_completion.d/cht.sh'] + # PHPREDISADMIN if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") @@ -793,6 +825,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -822,6 +855,7 @@ class WOStackController(CementBaseController): pargs.composer = True pargs.netdata = True pargs.mysqltuner = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -938,6 +972,14 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing packages for MySQLTuner ") packages = packages + ['/usr/bin/mysqltuner'] + # cheat.sh + if pargs.cheat: + if os.path.isfile('/usr/local/bin/cht.sh'): + Log.debug(self, "Removing packages for cheat.sh ") + packages = packages + [ + '/usr/local/bin/cht.sh', '/usr/local/bin/cheat', + '/etc/bash_completion.d/cht.sh'] + # PHPREDISADMIN if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 20bf585..205714e 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -1,7 +1,6 @@ import configparser import os -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose from wo.core.apt_repo import WORepo diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 0127746..50001fd 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1,4 +1,3 @@ -import codecs import configparser import os import random @@ -7,11 +6,8 @@ import string import psutil import requests -from wo.cli.plugins.site_functions import * -from wo.cli.plugins.stack_services import WOStackStatusController from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet -from wo.core.checkfqdn import check_fqdn_ip from wo.core.cron import WOCron from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils @@ -106,41 +102,68 @@ def pre_pref(self, apt_packages): # add nginx repository if set(WOVar.wo_nginx).issubset(set(apt_packages)): - Log.info(self, "Adding repository for NGINX, please wait...") if (WOVar.wo_distro == 'ubuntu'): - WORepo.add(self, ppa=WOVar.wo_nginx_repo) - Log.debug(self, 'Adding ppa for Nginx') + if not os.path.isfile( + 'wordops-ubuntu-nginx-wo-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, ppa=WOVar.wo_nginx_repo) + Log.debug(self, 'Adding ppa for Nginx') else: - WORepo.add(self, repo_url=WOVar.wo_nginx_repo) - Log.debug(self, 'Adding repository for Nginx') + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list/wo-repo.list', + 'download.opensuse.org'): + Log.info(self, "Adding repository for NGINX, please wait...") + Log.debug(self, 'Adding repository for Nginx') + WORepo.add(self, repo_url=WOVar.wo_nginx_repo) WORepo.add_key(self, WOVar.wo_nginx_key) # add php repository if (set(WOVar.wo_php73).issubset(set(apt_packages)) or set(WOVar.wo_php).issubset(set(apt_packages))): - Log.info(self, "Adding repository for PHP, please wait...") if (WOVar.wo_distro == 'ubuntu'): Log.debug(self, 'Adding ppa for PHP') - WORepo.add(self, ppa=WOVar.wo_php_repo) + if not os.path.isfile( + '/etc/apt/sources.list.d/ondrej-ubuntu-php-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for PHP, please wait...") + WORepo.add(self, ppa=WOVar.wo_php_repo) else: # Add repository for php if (WOVar.wo_platform_codename == 'buster'): php_pref = ("Package: *\nPin: origin " "packages.sury.org" "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'PHP.pref', 'w') as php_pref_file: + with open( + '/etc/apt/preferences.d/' + 'PHP.pref', mode='w', + encoding='utf-8') as php_pref_file: php_pref_file.write(php_pref) - Log.debug(self, 'Adding repo_url of php for debian') - WORepo.add(self, repo_url=WOVar.wo_php_repo) + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + 'packages.sury.org'): + Log.debug(self, 'Adding repo_url of php for debian') + Log.info(self, "Adding repository for PHP, please wait...") + WORepo.add(self, repo_url=WOVar.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') WORepo.add_key(self, WOVar.wo_php_key) # add redis repository if set(WOVar.wo_redis).issubset(set(apt_packages)): - Log.info(self, "Adding repository for Redis, please wait...") if WOVar.wo_distro == 'ubuntu': - Log.debug(self, 'Adding ppa for redis') - WORepo.add(self, ppa=WOVar.wo_redis_repo) + if not os.path.isfile( + '/etc/apt/sources.list.d/' + 'chris-lea-ubuntu-redis-server-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for Redis, please wait...") + Log.debug(self, 'Adding ppa for redis') + WORepo.add(self, ppa=WOVar.wo_redis_repo) + else: + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list/wo-repo.list', + 'download.opensuse.org'): + Log.info(self, "Adding repository for Redis, please wait...") + WORepo.add(self, repo_url=WOVar.wo_php_repo) + WORepo.add_key(self, WOVar.wo_nginx_key) def post_pref(self, apt_packages, packages, upgrade=False): @@ -489,6 +512,17 @@ def post_pref(self, apt_packages, packages, upgrade=False): "the cause of this issue", False) else: WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") + if not os.path.isdir('/etc/systemd/system/nginx.service.d'): + WOFileUtils.mkdir(self, + '/etc/systemd/system/nginx.service.d') + if not os.path.isdir( + '/etc/systemd/system/nginx.service.d/limits.conf'): + with open( + '/etc/systemd/system/nginx.service.d/limits.conf', + encoding='utf-8', mode='w') as ngx_limit: + ngx_limit.write('[Service]\nLimitNOFILE=500000') + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + WOService.restart_service(self, 'nginx') if set(WOVar.wo_php).issubset(set(apt_packages)): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") @@ -780,14 +814,14 @@ def post_pref(self, apt_packages, packages, upgrade=False): "/etc/mysql/my.cnf.default-pkg") wo_ram = psutil.virtual_memory().total / (1024 * 1024) # set InnoDB variable depending on the RAM available - wo_ram_innodb = int(wo_ram*0.3) - wo_ram_log_buffer = int(wo_ram_innodb*0.25) - wo_ram_log_size = int(wo_ram_log_buffer*0.5) + wo_ram_innodb = int(wo_ram * 0.3) + wo_ram_log_buffer = int(wo_ram_innodb * 0.25) + wo_ram_log_size = int(wo_ram_log_buffer * 0.5) if (wo_ram < 2000): wo_innodb_instance = int(1) tmp_table_size = int(32) elif (wo_ram > 2000) and (wo_ram < 64000): - wo_innodb_instance = int(wo_ram/1000) + wo_innodb_instance = int(wo_ram / 1000) tmp_table_size = int(128) elif (wo_ram > 64000): wo_innodb_instance = int(64) @@ -801,8 +835,19 @@ def post_pref(self, apt_packages, packages, upgrade=False): self, '/etc/mysql/my.cnf', 'my.mustache', data) # replacing default values Log.debug(self, "Tuning MySQL configuration") + if os.path.isdir('/etc/systemd/system/mariadb.service.d'): + if not os.path.isfile( + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf'): + WOFileUtils.textwrite( + self, + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf', + '[Service]\nLimitNOFILE=500000') + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') # set innodb_buffer_pool_instances depending # on the amount of RAM + WOService.stop_service(self, 'mysql') WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', '/var/lib/mysql/ib_logfile0.bak') @@ -970,24 +1015,26 @@ def post_pref(self, apt_packages, packages, upgrade=False): if wo_ram < 1024: Log.debug(self, "Setting maxmemory variable to " "{0} in redis.conf" - .format(int(wo_ram*1024*1024*0.1))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) + .format(int(wo_ram * 1024 * 1024 * 0.1))) + WOFileUtils.searchreplace( + self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram * 1024 * 1024 * 0.1))) else: Log.debug(self, "Setting maxmemory variable to {0} " "in redis.conf" - .format(int(wo_ram*1024*1024*0.2))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.2))) + .format(int(wo_ram * 1024 * 1024 * 0.2))) + WOFileUtils.searchreplace( + self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram * 1024 * 1024 * 0.2))) Log.debug( self, "Setting maxmemory-policy variable to " @@ -1147,6 +1194,21 @@ def post_pref(self, apt_packages, packages, upgrade=False): Log.debug(self, "CHMOD MySQLTuner in /usr/bin/mysqltuner") WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + # cheat.sh + if any('/usr/local/bin/cht.sh' == x[1] + for x in packages): + Log.debug(self, "CHMOD cht.sh in /usr/local/bin/cht.sh") + WOFileUtils.chmod(self, "/usr/local/bin/cht.sh", 0o775) + if WOFileUtils.grepcheck(self, '/etc/bash_completion.d/cht.sh', + 'cht_complete cht.sh'): + WOFileUtils.searchreplace( + self, '/etc/bash_completion.d/cht.sh', + '_cht_complete cht.sh', + '_cht_complete cheat') + if not os.path.islink('/usr/local/bin/cheat'): + WOFileUtils.create_symlink( + self, ['/usr/local/bin/cht.sh', '/usr/local/bin/cheat']) + # netdata install if any('/var/lib/wo/tmp/kickstart.sh' == x[1] for x in packages): @@ -1362,3 +1424,89 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOShellExec.cmd_exec(self, '/usr/local/sbin/install-ngxblocker -x') WOFileUtils.chmod( self, "/usr/local/sbin/update-ngxblocker", 0o700) + + +def pre_stack(self): + """Inital server configuration and tweak""" + # wo sysctl tweaks + # check system type + wo_arch = os.uname()[4] + if os.path.isfile('/proc/1/environ'): + # detect lxc containers + wo_lxc = WOFileUtils.grepcheck( + self, '/proc/1/environ', 'container=lxc') + # detect wsl + wo_wsl = WOFileUtils.grepcheck( + self, '/proc/1/environ', 'wsl') + else: + wo_wsl = True + wo_lxc = True + # remove old sysctl tweak + if os.path.isfile('/etc/sysctl.d/60-ubuntu-nginx-web-server.conf'): + WOFileUtils.rm(self, '/etc/sysctl.d/60-ubuntu-nginx-web-server.conf') + + if wo_arch == 'x86_64': + if (wo_lxc is not True) and (wo_wsl is not True): + data = dict() + WOTemplate.deploy( + self, '/etc/sysctl.d/60-wo-tweaks.conf', + 'sysctl.mustache', data, True) + # use tcp_bbr congestion algorithm only on new kernels + if (WOVar.wo_platform_codename == 'bionic' or + WOVar.wo_platform_codename == 'disco' or + WOVar.wo_platform_codename == 'buster'): + if WOShellExec.cmd_exec(self, 'modprobe tcp_bbr'): + with open("/etc/modules-load.d/bbr.conf", + encoding='utf-8', mode='w') as bbr_file: + bbr_file.write('tcp_bbr') + with open("/etc/sysctl.d/60-wo-tweaks.conf", + encoding='utf-8', mode='a') as sysctl_file: + sysctl_file.write( + '\nnet.ipv4.tcp_congestion_control = bbr' + '\nnet.ipv4.tcp_notsent_lowat = 16384') + else: + if WOShellExec.cmd_exec(self, 'modprobe tcp_htcp'): + with open("/etc/modules-load.d/htcp.conf", + encoding='utf-8', mode='w') as bbr_file: + bbr_file.write('tcp_htcp') + with open("/etc/sysctl.d/60-wo-tweaks.conf", + encoding='utf-8', mode='a') as sysctl_file: + sysctl_file.write( + '\nnet.ipv4.tcp_congestion_control = htcp') + # apply sysctl tweaks + WOShellExec.cmd_exec( + self, 'sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf') + # sysctl tweak service + data = dict() + if not os.path.isfile('/opt/wo-kernel.sh'): + WOTemplate.deploy(self, '/opt/wo-kernel.sh', + 'wo-kernel-script.mustache', data) + if not os.path.isfile('/lib/systemd/system/wo-kernel.service'): + WOTemplate.deploy( + self, '/lib/systemd/system/wo-kernel.service', + 'wo-kernel-service.mustache', data) + WOShellExec.cmd_exec(self, 'systemctl enable wo-kernel.service') + WOService.start_service(self, 'wo-kernel') + # open_files_limit tweak + if not WOFileUtils.grepcheck(self, '/etc/security/limits.conf', '500000'): + with open("/etc/security/limits.conf", + encoding='utf-8', mode='w') as limit_file: + limit_file.write( + '* hard nofile 500000\n' + '* soft nofile 500000\n' + 'root hard nofile 500000\n' + 'root soft nofile 500000\n') + # custom motd-news + data = dict() + # check if update-motd.d directory exist + if os.path.isdir('/etc/update-motd.d/'): + if not os.path.isfile('/etc/update-motd.d/98-wo-update'): + # render custom motd template + WOTemplate.deploy( + self, '/etc/update-motd.d/98-wo-update', + 'wo-update.mustache', data) + WOFileUtils.chmod( + self, "/etc/update-motd.d/98-wo-update", 0o755) + # restart motd-news service if available + if os.path.isfile('/lib/systemd/system/motd-news.service'): + WOService.restart_service(self, 'motd-news') diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index c9c4f43..6b40649 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -1,8 +1,6 @@ import os from cement.core.controller import CementBaseController, expose - -from wo.core.aptget import WOAptGet from wo.core.logging import Log from wo.core.services import WOService from wo.core.variables import WOVar @@ -19,6 +17,7 @@ class WOStackStatusController(CementBaseController): def start(self): """Start services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -34,23 +33,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -58,9 +57,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -69,28 +66,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -103,6 +100,7 @@ class WOStackStatusController(CementBaseController): def stop(self): """Stop services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -115,39 +113,32 @@ class WOStackStatusController(CementBaseController): pargs.php = True pargs.mysql = True - # nginx if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - # php7.2 if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - # php7.3 if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - # mysql if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -155,31 +146,29 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - # redis if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - # fail2ban if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -192,6 +181,7 @@ class WOStackStatusController(CementBaseController): def restart(self): """Restart services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -206,24 +196,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -231,10 +220,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if ((WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, - 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server'))): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -243,28 +229,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -277,6 +263,7 @@ class WOStackStatusController(CementBaseController): def status(self): """Status of services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -292,24 +279,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -317,9 +303,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -328,28 +312,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -362,6 +346,7 @@ class WOStackStatusController(CementBaseController): def reload(self): """Reload service""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -376,25 +361,23 @@ class WOStackStatusController(CementBaseController): pargs.fail2ban = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-mainline')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -402,9 +385,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -413,28 +394,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index e8cc4bf..64e75e1 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -3,7 +3,7 @@ import shutil from cement.core.controller import CementBaseController, expose -from wo.cli.plugins.stack_pref import post_pref, pre_pref +from wo.cli.plugins.stack_pref import post_pref, pre_pref, pre_stack from wo.core.aptget import WOAptGet from wo.core.download import WODownload from wo.core.extract import WOExtract @@ -46,6 +46,8 @@ class WOStackUpgradeController(CementBaseController): dict(help='Upgrade Composer', action='store_true')), (['--phpmyadmin'], dict(help='Upgrade phpMyAdmin', action='store_true')), + (['--ngxblocker'], + dict(help='Upgrade phpMyAdmin', action='store_true')), (['--no-prompt'], dict(help="Upgrade Packages without any prompt", action='store_true')), @@ -64,30 +66,35 @@ class WOStackUpgradeController(CementBaseController): if ((not pargs.web) and (not pargs.nginx) and (not pargs.php) and (not pargs.php73) and - (not pargs.mysql) and + (not pargs.mysql) and (not pargs.ngxblocker) and (not pargs.all) and (not pargs.wpcli) and (not pargs.netdata) and (not pargs.composer) and (not pargs.phpmyadmin) and (not pargs.dashboard) and (not pargs.redis)): pargs.web = True + pargs.admin = True if pargs.all: pargs.web = True - pargs.netdata = True - pargs.composer = True - pargs.dashboard = True - pargs.phpmyadmin = True + pargs.admin = True pargs.redis = True - pargs.wpcli = True pargs.php73 = True + pargs.ngxblocker = True if pargs.web: - if WOAptGet.is_installed(self, 'nginx-custom'): - pargs.nginx = True + pargs.nginx = True pargs.php = True pargs.mysql = True pargs.wpcli = True + if pargs.admin: + pargs.netdata = True + pargs.composer = True + pargs.dashboard = True + pargs.phpmyadmin = True + pargs.wpcli = True + + # nginx if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): apt_packages = apt_packages + WOVar.wo_nginx @@ -98,6 +105,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "Nginx Stable is not already installed") + # php 7.2 if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): apt_packages = apt_packages + WOVar.wo_php + \ @@ -105,6 +113,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") + # php 7.3 if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): apt_packages = apt_packages + WOVar.wo_php73 + \ @@ -112,71 +121,99 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.3 is not installed") + # mysql if pargs.mysql: if WOShellExec.cmd_exec(self, 'mysqladmin ping'): apt_packages = apt_packages + ['mariadb-server'] else: Log.info(self, "MariaDB is not installed") + # redis if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + ['redis-server'] else: Log.info(self, "Redis is not installed") + # wp-cli if pargs.wpcli: if os.path.isfile('/usr/local/bin/wp'): - packages = packages + [["https://github.com/wp-cli/wp-cli/" - "releases/download/v{0}/" - "wp-cli-{0}.phar" - "".format(WOVar.wo_wp_cli), - "/usr/local/bin/wp", - "WP-CLI"]] + packages = packages + [[ + "https://github.com/wp-cli/wp-cli/" + "releases/download/v{0}/" + "wp-cli-{0}.phar".format(WOVar.wo_wp_cli), + "/usr/local/bin/wp", + "WP-CLI"]] else: Log.info(self, "WPCLI is not installed with WordOps") + # netdata if pargs.netdata: - if (os.path.isdir('/opt/netdata') or - os.path.isdir('/etc/netdata')): - packages = packages + [['https://my-netdata.io/' - 'kickstart-static64.sh', - '/var/lib/wo/tmp/kickstart.sh', - 'Netdata']] + # detect static binaries install + if os.path.isdir('/opt/netdata'): + packages = packages + [[ + 'https://my-netdata.io/kickstart-static64.sh', + '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] + # detect install from source + elif os.path.isdir('/etc/netdata'): + packages = packages + [[ + 'https://my-netdata.io/kickstart.sh', + '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] + else: + Log.info(self, 'Netdata us not installed') + # wordops dashboard if pargs.dashboard: if (os.path.isfile('/var/www/22222/htdocs/index.php') or os.path.isfile('/var/www/22222/htdocs/index.html')): - packages = packages + \ - [["https://github.com/WordOps/wordops-dashboard/" - "releases/download/v{0}/wordops-dashboard.tar.gz" - .format(WOVar.wo_dashboard), - "/var/lib/wo/tmp/wo-dashboard.tar.gz", - "WordOps Dashboard"]] + packages = packages + [[ + "https://github.com/WordOps/wordops-dashboard/" + "releases/download/v{0}/wordops-dashboard.tar.gz" + .format(WOVar.wo_dashboard), + "/var/lib/wo/tmp/wo-dashboard.tar.gz", + "WordOps Dashboard"]] + else: + Log.info(self, 'WordOps dashboard is not installed') + # phpmyadmin if pargs.phpmyadmin: if os.path.isdir('/var/www/22222/htdocs/db/pma'): - packages = packages + \ - [["https://files.phpmyadmin.net" - "/phpMyAdmin/{0}/" - "phpMyAdmin-{0}-" - "all-languages" - ".tar.gz".format(WOVar.wo_phpmyadmin), - "/var/lib/wo/tmp/pma.tar.gz", - "PHPMyAdmin"]] + packages = packages + [[ + "https://files.phpmyadmin.net" + "/phpMyAdmin/{0}/phpMyAdmin-{0}-" + "all-languages.tar.gz" + .format(WOVar.wo_phpmyadmin), + "/var/lib/wo/tmp/pma.tar.gz", + "PHPMyAdmin"]] else: Log.info(self, "phpMyAdmin isn't installed") + # composer if pargs.composer: if os.path.isfile('/usr/local/bin/composer'): - packages = packages + [["https://getcomposer.org/installer", - "/var/lib/wo/tmp/composer-install", - "Composer"]] + packages = packages + [[ + "https://getcomposer.org/installer", + "/var/lib/wo/tmp/composer-install", + "Composer"]] else: Log.info(self, "Composer isn't installed") + # ngxblocker + if pargs.ngxblocker: + if os.path.exists('/usr/local/sbin/update-ngxblocker'): + packages = packages + [[ + 'https://raw.githubusercontent.com/mitchellkrogza/' + 'nginx-ultimate-bad-bot-blocker/master/update-ngxblocker', + '/usr/local/sbin/update-ngxblocker', + 'ngxblocker' + ]] + else: + Log.info(self, "ngxblocker is not installed") + if ((not (apt_packages)) and (not(packages))): self.app.args.print_help() else: + pre_stack(self) if (apt_packages): if (("php7.2-fpm" not in apt_packages) and ("php7.3-fpm" not in apt_packages) and @@ -196,15 +233,14 @@ class WOStackUpgradeController(CementBaseController): # apt-get update WOAptGet.update(self) Log.valide(self, "Updating APT packages") - Log.wait(self, "Upgrading APT Packages") # additional pre_pref - if ["nginx-custom"] in apt_packages: + if "nginx-custom" in apt_packages: pre_pref(self, WOVar.wo_nginx) - if ["php7.2-fpm"] in apt_packages: + if "php7.2-fpm" in apt_packages: WOAptGet.remove(self, ['php7.2-fpm'], auto=False, purge=True) - if ["php7.3-fpm"] in apt_packages: + if "php7.3-fpm" in apt_packages: WOAptGet.remove(self, ['php7.3-fpm'], auto=False, purge=True) # check if nginx upgrade is blocked @@ -213,9 +249,10 @@ class WOStackUpgradeController(CementBaseController): post_pref(self, WOVar.wo_nginx, [], True) # upgrade packages WOAptGet.install(self, apt_packages) - Log.valide(self, "Upgrading APT Packages") Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) + if "mariadb-server" in apt_packages: + WOShellExec.cmd_exec(self, 'mysql_upgrade') Log.valide(self, "Configuring APT Packages") # Post Actions after package updates @@ -226,6 +263,9 @@ class WOStackUpgradeController(CementBaseController): if pargs.netdata: WOFileUtils.rm(self, '/var/lib/wo/tmp/kickstart.sh') + if pargs.ngxblocker: + WOFileUtils.rm(self, '/usr/local/sbin/update-ngxblocker') + if pargs.dashboard: if os.path.isfile('/var/www/22222/htdocs/index.php'): WOFileUtils.rm(self, '/var/www/22222/htdocs/index.php') @@ -239,18 +279,38 @@ class WOStackUpgradeController(CementBaseController): if pargs.wpcli: WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + if pargs.ngxblocker: + WOFileUtils.chmod( + self, '/usr/local/sbin/update-ngxblocker', 0o700) + WOShellExec.cmd_exec( + self, '/usr/local/sbin/update-ngxblocker -nq' + ) + + # Netdata if pargs.netdata: Log.wait(self, "Upgrading Netdata") + # detect static binaries install if os.path.isdir('/opt/netdata'): - WOShellExec.cmd_exec( - self, "bash /opt/netdata/usr/" - "libexec/netdata/netdata-" - "updater.sh") + if os.path.exists( + '/opt/netdata/usr/libexec/' + 'netdata/netdata-updater.sh'): + WOShellExec.cmd_exec( + self, "bash /opt/netdata/usr/" + "libexec/netdata/netdata-" + "updater.sh") + else: + WOShellExec.cmd_exec( + self, "bash /var/lib/wo/tmp/kickstart.sh") + # detect install from source elif os.path.isdir('/etc/netdata'): - WOShellExec.cmd_exec( - self, "bash /usr/" - "libexec/netdata/netdata-" - "updater.sh") + if os.path.exists( + '/usr/libexec/netdata/netdata-updater.sh'): + WOShellExec.cmd_exec( + self, + 'bash /usr/libexec/netdata/netdata-updater.sh') + else: + WOShellExec.cmd_exec( + self, "bash /var/lib/wo/tmp/kickstart.sh") Log.valide(self, "Upgrading Netdata") if pargs.dashboard: @@ -265,10 +325,12 @@ class WOStackUpgradeController(CementBaseController): if pargs.composer: Log.wait(self, "Upgrading Composer") - WOShellExec.cmd_exec( - self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") + if WOShellExec.cmd_exec( + self, '/usr/bin/php -v'): + WOShellExec.cmd_exec( + self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") shutil.copyfile('/var/lib/wo/tmp/composer.phar', '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) diff --git a/wo/cli/plugins/sync.py b/wo/cli/plugins/sync.py index 662466d..872f962 100644 --- a/wo/cli/plugins/sync.py +++ b/wo/cli/plugins/sync.py @@ -52,18 +52,22 @@ class WOSyncController(CementBaseController): if configfiles: if WOFileUtils.isexist(self, configfiles[0]): - wo_db_name = (WOFileUtils.grep(self, configfiles[0], - 'DB_NAME').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_user = (WOFileUtils.grep(self, configfiles[0], - 'DB_USER').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_pass = (WOFileUtils.grep(self, configfiles[0], - 'DB_PASSWORD').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_host = (WOFileUtils.grep(self, configfiles[0], - 'DB_HOST').split(',')[1] - .split(')')[0].strip().replace('\'', '')) + wo_db_name = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_NAME').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_user = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_USER').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_pass = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_PASSWORD').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_host = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_HOST').split(',')[1] + .split(')')[0].strip().replace('\'', '')) # Check if database really exist try: diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 28f8287..f0fcad7 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -1,10 +1,11 @@ import os import time -from cement.core.controller import CementBaseController, expose +from cement.core.controller import CementBaseController, expose from wo.core.download import WODownload from wo.core.logging import Log +from wo.core.variables import WOVar def wo_update_hook(app): @@ -22,12 +23,17 @@ class WOUpdateController(CementBaseController): arguments = [ (['--force'], dict(help='Force WordOps update', action='store_true')), - (['--preserve'], - dict(help='Preserve current Nginx configuration', - action='store_true')), (['--beta'], - dict(help='Update WordOps to latest beta release', + dict(help='Update WordOps to latest mainline release ' + '(same than --mainline)', action='store_true')), + (['--mainline'], + dict(help='Update WordOps to latest mainline release', + action='store_true')), + (['--branch'], + dict(help="Update WordOps from a specific repository branch ", + action='store' or 'store_const', + const='develop', nargs='?')), (['--travis'], dict(help='Argument used only for WordOps development', action='store_true')), @@ -39,28 +45,42 @@ class WOUpdateController(CementBaseController): pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") - if pargs.beta: - wo_branch = "beta" - install_args = "" - else: - wo_branch = "master" - install_args = "" + install_args = "" + if pargs.mainline or pargs.beta: + wo_branch = "mainline" + install_args = install_args + "--mainline " + elif pargs.branch: + wo_branch = pargs.branch + install_args = install_args + "-b {0} ".format(wo_branch) if pargs.force: install_args = install_args + "--force " - if pargs.preserve: - install_args = install_args + "--preserve " + if pargs.travis: + install_args = install_args + "--travis " + wo_branch = "updating-configuration" + if ((not pargs.force) and (not pargs.travis) and + (not pargs.mainline) and (not pargs.beta) and + (not pargs.branch)): + wo_current = WOVar.wo_version + wo_latest = WODownload.latest_release(self, "WordOps/WordOps") + if wo_current == wo_latest: + Log.error( + self, "WordOps {0} is already installed" + .format(wo_latest)) + + if not os.path.isdir('/var/lib/wo/tmp'): + os.makedirs('/var/lib/wo/tmp') WODownload.download(self, [["https://raw.githubusercontent.com/" "WordOps/WordOps/{0}/install" .format(wo_branch), "/var/lib/wo/tmp/{0}".format(filename), "update script"]]) - if pargs.travis: + if os.path.isfile('install'): + Log.info(self, "updating WordOps from local install\n") try: Log.info(self, "updating WordOps, please wait...") - os.system("/bin/bash install --travis " - "-b $TRAVIS_BRANCH --force") + os.system("/bin/bash install --travis") except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") @@ -68,12 +88,13 @@ class WOUpdateController(CementBaseController): try: Log.info(self, "updating WordOps, please wait...") os.system("/bin/bash /var/lib/wo/tmp/{0} " - "-b {1} {2}".format(filename, - wo_branch, install_args)) + "{1}".format(filename, install_args)) except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") + os.remove("/var/lib/wo/tmp/{0}".format(filename)) + def load(app): # register the plugin class.. this only happens if the plugin is enabled diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index 0d9a88a..d0f1c17 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -11,7 +11,7 @@ location @empty_gif { empty_gif; } # Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest)$ { +location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest|cast)$ { more_set_headers 'Access-Control-Allow-Origin : *'; more_set_headers "Cache-Control : public, no-transform"; access_log off; diff --git a/wo/cli/templates/nextcloud.mustache b/wo/cli/templates/nextcloud.mustache index 48a7df1..8d2eae0 100644 --- a/wo/cli/templates/nextcloud.mustache +++ b/wo/cli/templates/nextcloud.mustache @@ -8,7 +8,7 @@ location = /robots.txt { access_log off; } location / { - rewrite ^ /index.php$request_uri; + rewrite ^ /index.php; } location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ { deny all; @@ -18,13 +18,12 @@ location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) { } location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) { fastcgi_split_path_info ^(.+?\.php)(\/.*|)$; + try_files $uri =404; include fastcgi_params; - fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; - fastcgi_param HTTPS on; -# Avoid sending the security headers twice + # Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; -# Enable pretty urls + # Enable pretty urls fastcgi_param front_controller_active true; fastcgi_pass {{upstream}}; fastcgi_intercept_errors on; diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index 5de705c..97fc476 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -55,12 +55,12 @@ http { ssl_session_cache shared:SSL:50m; ssl_session_tickets off; ssl_prefer_server_ciphers on; - ssl_early_data on; + ssl_early_data on; {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; ssl_protocols TLSv1.2 TLSv1.3;{{/tls13}} ssl_ecdh_curve X25519:P-521:P-384:P-256; - # Previous TLS v1.2 configuration - {{^tls13}}ssl_protocols TLSv1.2; + {{^tls13}}# Previous TLS v1.2 configuration + ssl_protocols TLSv1.2; ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES;{{/tls13}} # Common security headers diff --git a/wo/cli/templates/php-pool.mustache b/wo/cli/templates/php-pool.mustache index 654e944..cbdddb2 100644 --- a/wo/cli/templates/php-pool.mustache +++ b/wo/cli/templates/php-pool.mustache @@ -20,4 +20,4 @@ listen.backlog = 32768 catch_workers_output = yes -{{#openbasedir}}php_admin_value[open_basedir] = "/var/www/:/usr/share/php/:/tmp/:/var/run/nginx-cache/:/dev/urandom:/dev/shm"{{/openbasedir}} +{{#openbasedir}}php_admin_value[open_basedir] = "/var/www/:/usr/share/php/:/tmp/:/var/run/nginx-cache/:/dev/urandom:/dev/shm:/var/lib/php/sessions/"{{/openbasedir}} diff --git a/wo/cli/templates/wo-update.mustache b/wo/cli/templates/wo-update.mustache new file mode 100644 index 0000000..6ad2c01 --- /dev/null +++ b/wo/cli/templates/wo-update.mustache @@ -0,0 +1,37 @@ +#!/bin/sh +# script to update motd when a new WordOps release is available on Debian/Ubuntu +# the script is added in /etc/update-motd.d + +safe_print() { + cat "$1" | head -n 10 | tr -d '\000-\011\013\014\016-\037' | cut -c -80 +} + +# Ensure sane defaults +[ -n "$URL" ] || URL="https://api.github.com/repos/WordOps/WordOps/releases/latest" +[ -n "$WAIT" ] || WAIT=5 +[ -n "$CACHE" ] || CACHE="/var/cache/motd-wo" + +# Generate our temp files, clean up when done +NEWS=$(mktemp) || exit 1 +ERR=$(mktemp) || exit 1 +CLOUD=$(mktemp) || exit 1 +trap "rm -f $NEWS $ERR $CLOUD" HUP INT QUIT ILL TRAP BUS TERM + +if [ -n "$(command -v curl)" ]; then + LATEST_RELEASE=$(curl -m 5 --retry 3 -sL "$URL" | jq -r '.tag_name' 2>&1) +fi +if [ -n "$(command -v wo)" ]; then + CURRENT_RELEASE=$(wo -v 2>&1 | grep v | awk -F " " '{print $2}') +fi +if [ -n "$CURRENT_RELEASE" ] && [ -n "$LATEST_RELEASE" ]; then + if [ "$CURRENT_RELEASE" != "$LATEST_RELEASE" ]; then + # display message with motd-news on Ubuntu + echo '*** A new WordOps release is available ***' > "$NEWS" 2> "$ERR" + + echo + # At most, 10 lines of text, remove control characters, print at most 80 characters per line + safe_print "$NEWS" + # Try to update the cache + safe_print "$NEWS" 2> /dev/null > $CACHE || true + fi +fi diff --git a/wo/core/acme.py b/wo/core/acme.py index 3d5e04d..8196238 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -6,7 +6,7 @@ import requests from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log -from wo.core.shellexec import WOShellExec +from wo.core.shellexec import WOShellExec, CommandExecutionError from wo.core.variables import WOVar @@ -22,6 +22,7 @@ class WOAcme: self, "{0} ".format(WOAcme.wo_acme_exec) + "--list --listraw > /var/lib/wo/cert.csv"): Log.error(self, "Unable to export certs list") + WOFileUtils.chmod(self, '/var/lib/wo/cert.csv', 0o600) def setupletsencrypt(self, acme_domains, acmedata): """Issue SSL certificates with acme.sh""" @@ -38,6 +39,14 @@ class WOAcme: acme_mode = "-w /var/www/html" validation_mode = "Webroot challenge" Log.debug(self, "Validation : Webroot mode") + if not os.path.isdir('/var/www/html/.well-known/acme-challenge'): + WOFileUtils.mkdir( + self, '/var/www/html/.well-known/acme-challenge') + WOFileUtils.chown( + self, '/var/www/html/.well-known', 'www-data', 'www-data', + recursive=True) + WOFileUtils.chmod(self, '/var/www/html/.well-known', 0o750, + recursive=True) Log.info(self, "Validation mode : {0}".format(validation_mode)) Log.wait(self, "Issuing SSL cert with acme.sh") @@ -63,6 +72,7 @@ class WOAcme: return True def deploycert(self, wo_domain_name): + """Deploy Let's Encrypt certificates with acme.sh""" if not os.path.isfile('/etc/letsencrypt/renewal/{0}_ecc/fullchain.cer' .format(wo_domain_name)): Log.error(self, 'Certificate not found. Deployment canceled') @@ -122,6 +132,17 @@ class WOAcme: "ssl.conf") return 0 + def renew(self, domain): + """Renew letsencrypt certificate with acme.sh""" + try: + WOShellExec.cmd_exec( + self, "{0} ".format(WOAcme.wo_acme_exec) + + "--renew -d {0} --ecc --force".format(domain)) + except CommandExecutionError as e: + Log.debug(self, str(e)) + Log.error(self, 'Unable to renew certificate') + return True + def check_dns(self, acme_domains): """Check if a list of domains point to the server IP""" server_ip = requests.get('http://v4.wordops.eu/').text @@ -130,15 +151,16 @@ class WOAcme: .format(domain)).text if(not domain_ip == server_ip): Log.warn( - self, "{0} is not pointing to your server IP" - .format(domain)) + self, "{0}".format(domain) + + " point to the IP {0}".format(domain_ip) + + " but your server IP is {0}.".format(server_ip) + + "\nUse the flag --force to bypass this check.") Log.error( - self, "You have to add the " - "proper DNS record", False) + self, "You have to set the " + "proper DNS record for your domain", False) return False - else: - Log.debug(self, "DNS record are properly set") - return True + Log.debug(self, "DNS record are properly set") + return True def cert_check(self, wo_domain_name): """Check certificate existance with acme.sh and return Boolean""" @@ -153,9 +175,51 @@ class WOAcme: if wo_domain_name in row[0]: # check if cert expiration exist if not row[3] == '': - cert_exist = True - break - else: - cert_exist = False + return True certfile.close() - return cert_exist + return False + + def removeconf(self, domain): + sslconf = ("/var/www/{0}/conf/nginx/ssl.conf" + .format(domain)) + sslforce = ("/etc/nginx/conf.d/force-ssl-{0}.conf" + .format(domain)) + acmedir = [ + '{0}'.format(sslforce), '{0}'.format(sslconf), + '{0}/{1}_ecc'.format(WOVar.wo_ssl_archive, domain), + '{0}.disabled'.format(sslconf), '{0}.disabled' + .format(sslforce), '{0}/{1}' + .format(WOVar.wo_ssl_live, domain), + '/etc/letsencrypt/shared/{0}.conf'.format(domain)] + wo_domain = domain + if WOAcme.cert_check(self, wo_domain): + Log.info(self, "Removing Acme configuration") + Log.debug(self, "Removing Acme configuration") + try: + WOShellExec.cmd_exec( + self, "{0} ".format(WOAcme.wo_acme_exec) + + "--remove -d {0} --ecc".format(domain)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Cert removal failed") + # remove all files and directories + for dir in acmedir: + if os.path.exists('{0}'.format(dir)): + WOFileUtils.rm(self, '{0}'.format(dir)) + # find all broken symlinks + WOFileUtils.findBrokenSymlink(self, "/var/www") + else: + if os.path.islink("{0}".format(sslconf)): + WOFileUtils.remove_symlink(self, "{0}".format(sslconf)) + WOFileUtils.rm(self, '{0}'.format(sslforce)) + + if WOFileUtils.grepcheck(self, '/var/www/22222/conf/nginx/ssl.conf', + '{0}'.format(domain)): + Log.info( + self, "Setting back default certificate for WordOps backend") + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "w") as ssl_conf_file: + ssl_conf_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index 5192cdd..470ac4e 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -49,7 +49,7 @@ class WORepo(): Log.error(self, "Unable to add repo") if ppa is not None: if WOShellExec.cmd_exec( - self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" + self, "LC_ALL=C.UTF-8 add-apt-repository -y '{ppa_name}'" .format(ppa_name=ppa)): return True return False @@ -70,7 +70,7 @@ class WORepo(): WOVar().wo_repo_file) try: - repofile = open(repo_file_path, "w+") + repofile = open(repo_file_path, "w+", encoding='utf-8') repofile.write(repofile.read().replace(repo_url, "")) repofile.close() except IOError as e: @@ -96,19 +96,14 @@ class WORepo(): Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to import repo key") - def add_keys(self, keyids, keyserver=None): + def download_key(self, key_url): """ - This function adds imports repository keys from keyserver. - default keyserver is hkp://keyserver.ubuntu.com - user can provide other keyserver with keyserver="hkp://xyz" + This function download gpg keys and add import them with apt-key add" """ - all_keys = ' '.join(keyids) try: WOShellExec.cmd_exec( - self, "apt-key adv --keyserver {serv}" - .format(serv=(keyserver or - "hkp://keyserver.ubuntu.com")) + - " --recv-keys {keys}".format(keys=all_keys)) + self, "curl -sL {0} ".format(key_url) + + "| apt-key add -") except Exception as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to import repo keys") diff --git a/wo/core/checkfqdn.py b/wo/core/checkfqdn.py index 55f93d5..2ce5533 100644 --- a/wo/core/checkfqdn.py +++ b/wo/core/checkfqdn.py @@ -33,7 +33,4 @@ def check_fqdn_ip(self): y = requests.get('http://v4.wordops.eu/dns/{0}/'.format(wo_fqdn)) ip_fqdn = (y.text).strip() - if ip == ip_fqdn: - return True - else: - return False + return bool(ip == ip_fqdn) diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index fb4f477..e84a26b 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -2,7 +2,7 @@ import os -class WODomain(): +class WODomain: """WordOps domain validation utilities""" def validate(self, url): @@ -24,8 +24,6 @@ class WODomain(): return final_domain return domain_name - - def getlevel(self, domain): """ Returns the domain type : domain, subdomain and the root domain diff --git a/wo/core/download.py b/wo/core/download.py index 53abbfd..a67c486 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -1,7 +1,6 @@ """WordOps download core classes.""" import os -import urllib.error -import urllib.request +import requests from wo.core.logging import Log @@ -12,7 +11,7 @@ class WODownload(): pass def download(self, packages): - """Download packages, packges must be list in format of + """Download packages, packages must be list in format of [url, path, package name]""" for package in packages: url = package[0] @@ -23,25 +22,31 @@ class WODownload(): if not os.path.exists(directory): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') - req = urllib.request.Request( - url, headers={'User-Agent': 'Mozilla/5.0'}) - with urllib.request.urlopen(req) as response, open(filename, 'wb') as out_file: - out_file.write(response.read()) - Log.info(self, "{0}".format("[" + Log.ENDC + "Done" - + Log.OKBLUE + "]")) - except urllib.error.URLError as e: + with open(filename, "wb") as out_file: + req = requests.get(url, timeout=(5, 30)) + if req.encoding is None: + req.encoding = 'utf-8' + out_file.write(req.content) + Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + + Log.OKBLUE + "]")) + except requests.RequestException as e: Log.debug(self, "[{err}]".format(err=str(e.reason))) Log.error(self, "Unable to download file, {0}" .format(filename)) return False - except urllib.HTTPError.error as e: - Log.error(self, "Package download failed. {0}" - .format(pkg_name)) - Log.debug(self, "[{err}]".format(err=str(e.reason))) - return False - except urllib.ContentTooShortError.error as e: - Log.debug(self, "{0}{1}".format(e.errno, e.strerror)) - Log.error(self, "Package download failed. The amount of the" - " downloaded data is less than " - "the expected amount \{0} ".format(pkg_name)) - return False + return 0 + + def latest_release(self, repository): + """Get the latest release number of a GitHub repository.\n + repository format should be: \"user/repo\"""" + try: + req = requests.get( + 'https://api.github.com/repos/{0}/releases/latest' + .format(repository), + timeout=(5, 30)) + github_json = req.json() + except requests.RequestException as e: + Log.debug(self, str(e)) + Log.error(self, "Unable to query GitHub API") + + return github_json["tag_name"] diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index 6a9b505..7b70f5c 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -280,17 +280,19 @@ class WOFileUtils(): """ Searches for string in file and returns True or False. """ - try: - Log.debug(self, "Finding string {0} to file {1}" - .format(sstr, fnm)) - for line in open(fnm, encoding='utf-8'): - if sstr in line: - return True - return False - except OSError as e: - Log.debug(self, "{0}".format(e.strerror)) - Log.error(self, "Unable to Search string {0} in {1}" - .format(sstr, fnm)) + if os.path.isfile('{0}'.format(fnm)): + try: + Log.debug(self, "Finding string {0} to file {1}" + .format(sstr, fnm)) + for line in open(fnm, encoding='utf-8'): + if sstr in line: + return True + return False + except OSError as e: + Log.debug(self, "{0}".format(e.strerror)) + Log.error(self, "Unable to Search string {0} in {1}" + .format(sstr, fnm)) + return False def rm(self, path): """ @@ -341,3 +343,29 @@ class WOFileUtils(): # If it's not a symlink we're not interested. continue return True + + def textwrite(self, path, content): + """ + Write content into a file + """ + Log.debug(self, "Writing content in {0}".format(path)) + try: + with open("{0}".format(path), + encoding='utf-8', mode='w') as final_file: + final_file.write('{0}'.format(content)) + except IOError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to write content in {0}".format(path)) + + def textappend(self, path, content): + """ + Append content to a file + """ + Log.debug(self, "Writing content in {0}".format(path)) + try: + with open("{0}".format(path), + encoding='utf-8', mode='a') as final_file: + final_file.write('{0}'.format(content)) + except IOError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to write content in {0}".format(path)) diff --git a/wo/core/git.py b/wo/core/git.py index a157282..5bf903f 100644 --- a/wo/core/git.py +++ b/wo/core/git.py @@ -21,7 +21,7 @@ class WOGit: git = git.bake("--git-dir={0}/.git".format(path), "--work-tree={0}".format(path)) if os.path.isdir(path): - if not os.path.isdir(path+"/.git"): + if not os.path.isdir(path + "/.git"): try: Log.debug(self, "WOGit: git init at {0}" .format(path)) @@ -67,11 +67,13 @@ class WOGit: git = git.bake("--git-dir={0}/.git".format(path), "--work-tree={0}".format(path)) if os.path.isdir(path): - if not os.path.isdir(path+"/.git"): - Log.error(self, "Unable to find a git repository at {0}" + if not os.path.isdir(path + "/.git"): + Log.error( + self, "Unable to find a git repository at {0}" .format(path)) try: - Log.debug(self, "WOGit: git stash --include-untracked at {0}" + Log.debug( + self, "WOGit: git stash --include-untracked at {0}" .format(path)) git.stash("push", "--include-untracked", "-m {0}" .format(msg)) diff --git a/wo/core/mysql.py b/wo/core/mysql.py index cd9de53..907a608 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -70,7 +70,7 @@ class WOMysql(): # Get login details from /etc/mysql/conf.d/my.cnf # & Execute MySQL query connection = WOMysql.connect(self) - log and Log.debug(self, "Exceuting MySQL Statement : {0}" + log and Log.debug(self, "Executing MySQL Statement : {0}" .format(statement)) try: cursor = connection.cursor() @@ -93,12 +93,12 @@ class WOMysql(): import subprocess try: Log.info(self, "Backing up database at location: " - "/var/wo-mysqlbackup") + "/var/lib/wo-backup/mysql") # Setup Nginx common directory - if not os.path.exists('/var/wo-mysqlbackup'): + if not os.path.exists('/var/lib/wo-backup/mysql'): Log.debug(self, 'Creating directory' - '/var/wo-mysqlbackup') - os.makedirs('/var/wo-mysqlbackup') + '/var/lib/wo-backup/mysql') + os.makedirs('/var/lib/wo-backup/mysql') db = subprocess.check_output(["/usr/bin/mysql " "-Bse \'show databases\'"], @@ -114,7 +114,7 @@ class WOMysql(): stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen("/usr/bin/pigz -c > " - "/var/wo-mysqlbackup/{0}{1}.sql.gz" + "/var/lib/wo-backup/mysql/{0}{1}.sql.gz" .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True) diff --git a/wo/core/services.py b/wo/core/services.py index 7b4d748..1c59e63 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -43,7 +43,8 @@ class WOService(): return True else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + + "Failed" + Log.OKBLUE + "]") return False except OSError as e: Log.debug(self, "{0}".format(e)) @@ -65,7 +66,7 @@ class WOService(): return True else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE + "]") return False except OSError as e: Log.debug(self, "{0}".format(e)) diff --git a/wo/core/shellexec.py b/wo/core/shellexec.py index 61b12ea..3fb638c 100644 --- a/wo/core/shellexec.py +++ b/wo/core/shellexec.py @@ -27,14 +27,9 @@ class WOShellExec(): cmd_stderr_bytes.decode('utf-8', "replace")) - if proc.returncode == 0: - Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" - .format(cmd_stdout, cmd_stderr)) - return True - else: - Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" - .format(cmd_stdout, cmd_stderr)) - return False + Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" + .format(cmd_stdout, cmd_stderr)) + return bool(proc.returncode == 0) except OSError as e: Log.debug(self, str(e)) raise CommandExecutionError diff --git a/wo/core/variables.py b/wo/core/variables.py index 8654783..d0b45ff 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -4,6 +4,7 @@ import os from datetime import datetime from re import match from socket import getfqdn +from shutil import copy2 from distro import linux_distribution from sh import git @@ -13,7 +14,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.9.4" + wo_version = "3.10.0" # WordOps packages versions wo_wp_cli = "2.3.0" wo_adminer = "4.7.3" @@ -28,16 +29,18 @@ class WOVar(): wo_date = datetime.now().strftime('%d%b%Y-%H-%M-%S') # WordOps core variables + # linux distribution wo_distro = linux_distribution( full_distribution_name=False)[0].lower() wo_platform_version = linux_distribution( full_distribution_name=False)[1].lower() + # distro codename (bionic, xenial, stretch ...) wo_platform_codename = linux_distribution( full_distribution_name=False)[2].lower() # Get timezone of system if os.path.isfile('/etc/timezone'): - with open("/etc/timezone", "r") as tzfile: + with open("/etc/timezone", mode='r', encoding='utf-8') as tzfile: wo_timezone = tzfile.read().replace('\n', '') if wo_timezone == "Etc/UTC": wo_timezone = "UTC" @@ -59,9 +62,9 @@ class WOVar(): # PHP user wo_php_user = 'www-data' - # Get git user name and EMail + # WordOps git configuration management config = configparser.ConfigParser() - config.read(os.path.expanduser("~")+'/.gitconfig') + config.read(os.path.expanduser("~") + '/.gitconfig') try: wo_user = config['user']['name'] wo_email = config['user']['email'] @@ -86,13 +89,16 @@ class WOVar(): git.config("--global", "user.name", "{0}".format(wo_user)) git.config("--global", "user.email", "{0}".format(wo_email)) + if not os.path.isfile('/root/.gitconfig'): + copy2(os.path.expanduser("~") + '/.gitconfig', '/root/.gitconfig') + # MySQL hostname wo_mysql_host = "" config = configparser.RawConfigParser() if os.path.exists('/etc/mysql/conf.d/my.cnf'): cnfpath = "/etc/mysql/conf.d/my.cnf" else: - cnfpath = os.path.expanduser("~")+"/.my.cnf" + cnfpath = os.path.expanduser("~") + "/.my.cnf" if [cnfpath] == config.read(cnfpath): try: wo_mysql_host = config.get('client', 'host') diff --git a/wo/utils/test.py b/wo/utils/test.py index 920fd62..d737fdd 100644 --- a/wo/utils/test.py +++ b/wo/utils/test.py @@ -1,5 +1,5 @@ """Testing utilities for WordOps""" -from cement.utils.test import * +from cement.utils.test import CementTestCase from wo.cli.main import WOTestApp