diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 53ae6b1..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: [push] - -jobs: - build: - - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v1 - - name: Prepare VM - run: | - unset LANG - sudo apt update -qq - sudo LC_ALL=C.UTF-8 add-apt-repository ppa:ondrej/php -y - sudo apt-get -qq purge graphviz* redis* - sudo apt-get install -qq git python3-setuptools python3-dev python3-apt ccze tree - sudo apt-get -qq autoremove --purge - sudo mkdir -p /etc/mysql/conf.d - echo -e '[client]\nuser = root\npassword = root\n' | sudo tee /etc/mysql/conf.d/my.cnf - - name: Install WordOps - run: sudo timeout 1800 bash install --travis - - name: Run tests - run: sudo timeout 1800 bash tests/travis.sh --actions diff --git a/.travis.yml b/.travis.yml index 64c9c5f..3c37fe2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: xenial +dist: bionic language: bash @@ -23,7 +23,7 @@ before_script: - sudo bash -c 'echo example.com > /etc/hostname' - unset LANG - sudo apt-get -qq purge mysql* graphviz* redis* - - sudo apt-get install -qq git python3-setuptools python3-dev python3-apt ccze tree + - sudo apt-get install -qq git python3-setuptools python3-dev python3-apt ccze tree python3-wheel python3-pip - sudo apt-get -qq autoremove --purge @@ -37,6 +37,7 @@ script: - lsb_release -a - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" + - sudo python3 setup.py sdist bdist_wheel >> /var/log/wo/install.log 2>&1 - sudo time bash install --travis -b "$TRAVIS_BRANCH" - sudo time bash tests/travis.sh - sudo wo update --travis \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 852f9c7..7e09887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Changed + +- WordOps is now installed from PyPI using pip [WordOps](https://pypi.org/project/wordops/) +- New Nginx package built with OpenSSL_1.1.1d and the latest ngx_brotli module + +#### Fixed + +- `wo stack upgrade` when using nginx-ee +- `wo secure --auth` +- `wo secure --sshport` not working with default ssh config +- Issues after APT repositories informations changed + ### v3.9.9.2 - 2019-10-04 #### Added diff --git a/install b/install index 4378362..24df83f 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.9.9.1 - 2019-09-27 +# Version 3.9.9.3 - 2019-10-08 # ------------------------------------------------------------------------- # CONTENTS @@ -62,9 +62,6 @@ while [ "$#" -gt 0 ]; do wo_branch="$2" shift ;; - -p | --preserve) - wo_preserve_config="y" - ;; --force) wo_force_install="y" ;; @@ -78,10 +75,6 @@ while [ "$#" -gt 0 ]; do --purge | --uninstall) wo_purge="y" ;; - -v | --version) - wo_version="$2" - shift - ;; *) # positional args ;; esac @@ -97,14 +90,15 @@ if [[ $EUID -ne 0 ]]; then exit 100 fi -### -# 1- Update the apt sewers with fresh info -### export DEBIAN_FRONTEND=noninteractive unset LANG export LANG='en_US.UTF-8' export LC_ALL='C.UTF-8' +### +# 1- Main functions +### + # check if a command exist command_exists() { command -v "$@" > /dev/null 2>&1 @@ -121,14 +115,9 @@ _run() { } -check_path() { - if [ -d "$1" ]; then - return 0 - else - return 1 - fi -} - +### +# 1- Update the apt sewers with fresh info +### if [ -z "$wo_travis" ]; then if command_exists curl; then apt-get update -qq @@ -167,7 +156,6 @@ readonly wo_tmp_dir=/var/lib/wo/tmp readonly wo_install_log=/var/log/wo/install.log readonly wo_linux_distro=$(lsb_release -is) readonly wo_distro_version=$(lsb_release -sc) -readonly wo_distro_id=$(lsb_release -rs) readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' readonly TIME=$(date +"$TIME_FORMAT") readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" @@ -227,18 +215,32 @@ wo_dist_upgrade() { apt-get dist-upgrade --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --option=Dpkg::options::=--force-unsafe-io --assume-yes --quiet } +# install apt-mirror-updater and find the fastest mirror available +wo_apt_mirror() { + + apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ + python3-pip python3-wheel python3-apt python3-setuptools > /dev/null 2>&1 + python3 -m pip install pip --upgrade + if { + python3 -m pip install apt-mirror-updater --upgrade + }; then + /usr/local/bin/apt-mirror-updater -a + fi + +} + wo_install_dep() { if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \ - gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + build-essential curl gzip python3-dev sqlite3 git tar software-properties-common pigz \ + gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 add-apt-repository ppa:wordops/nginx-wo -yn else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip dirmngr sudo python3 python3-apt python3-setuptools python3-requests python3-dev ca-certificates sqlite3 git tar \ + build-essential curl gzip dirmngr sudo python3-dev ca-certificates sqlite3 git tar \ software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } @@ -318,16 +320,6 @@ wo_sync_db() { wo_site_current_type=$(grep "common/" "/etc/nginx/sites-available/$site" | awk -F "/" '{print $2}') - if echo "$wo_site_current_type" | grep -q "php"; then - if echo "$wo_site_current_type" | grep -q "php7"; then - wo_php_version="7.0" - else - wo_php_version="5.6" - fi - else - wo_php_version="" - fi - if echo "$wo_site_current_type" | grep -q "redis"; then wo_site_current_cache="wpredis" elif echo "$wo_site_current_type" | grep -q wpsc; then @@ -458,19 +450,6 @@ wo_install_acme_sh() { fi } -# Download WordOps -wo_download() { - rm -f /etc/bash_completion.d/wo_auto.rc - rm -rf /var/lib/wo/tmp/WordOps-* - if [ -z "$wo_version" ]; then - curl -sL https://github.com/WordOps/WordOps/archive/${wo_branch}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_branch" /var/lib/wo/tmp/WordOps-install - else - curl -sL https://github.com/WordOps/WordOps/archive/v${wo_version}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_version" /var/lib/wo/tmp/WordOps-install - fi -} - wo_git_config() { if [ "$wo_force_install" = "y" ]; then @@ -486,22 +465,18 @@ wo_git_config() { # WordOps install wo_install() { - cd /var/lib/wo/tmp/WordOps-install || exit 1 - python3 setup.py install + python3 -m pip install wordops --upgrade + cp -rf /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ } -# Clone Github repository if it doesn't exist -wo_install_travis() { +# WordOps install +wo_travis_install() { - if [ "$wo_force_install" = "y" ]; then - [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } - fi - - if [ -f "$HOME/.gitconfig" ]; then - # install and redirect log to not print python package install - python3 setup.py install - fi + python3 -m pip install dist/*.whl --upgrade + cp -rf /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ } @@ -610,32 +585,20 @@ wo_update_latest() { chmod 600 /root/.my.cnf fi fi - - # Fix Redis-server security issue - # http://redis.io/topics/security - if [ -f /etc/redis/redis.conf ]; then - CHECK_REDIS_BIND=$(grep -0 -v "#" /etc/redis/redis.conf | grep 'bind' >> /dev/null 2>&1) - - if [ -z "$CHECK_REDIS_BIND" ]; then - echo 'bind 127.0.0.1 ::1' >> /etc/redis/redis.conf - - service redis-server restart - - fi - fi - } wo_backup_ee() { local BACKUP_EE="" [ -d /etc/nginx ] && { BACKUP_EE="$BACKUP_EE /etc/nginx"; } [ -d /etc/letsencrypt ] && { BACKUP_EE="$BACKUP_EE /etc/letsencrypt"; } - /bin/tar -I pigz -cf "$EE_BACKUP_FILE" /usr/local/bin/ee /usr/lib/ee/templates /usr/local/lib/python3.*/dist-packages/ee-*.egg /etc/ee /var/lib/ee "$BACKUP_EE" + /bin/tar -I pigz -cf "$EE_BACKUP_FILE" /usr/local/bin/ee /usr/lib/ee/templates /etc/ee /var/lib/ee "$BACKUP_EE" return 0 } wo_backup_wo() { - /bin/tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx /etc/wo /var/lib/wo "$WO_LE" + if [ -d /etc/nginx ] && [ -d /etc/wo ] && [ -d /var/lib/wo ]; then + /bin/tar -I pigz -cf "$WO_BACKUP_FILE" /etc/nginx /etc/wo /var/lib/wo + fi return 0 } @@ -686,7 +649,7 @@ wo_systemd_tweak() { } wo_domain_suffix() { - curl -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat + curl -m 10 --retry 3 -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat } wo_mariadb_tweak() { @@ -715,6 +678,7 @@ wo_clean() { } wo_uninstall() { + python3 -m pip uninstall wordops -y rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates } @@ -820,6 +784,7 @@ else _run wo_remove_ee_cron "Removing EasyEngine cronjob" _run wo_sync_db "Syncing WO database" fi + _run wo_apt_mirror "Finding the fastest APT mirror" _run wo_install_dep "Installing wo dependencies" _run wo_timesync #if [ "$ufw" = "y" ]; then @@ -827,12 +792,11 @@ else #fi # skip steps if travis if [ -z "$wo_travis" ]; then - _run wo_download "Downloading WordOps" _run wo_dist_upgrade wo_git_config _run wo_install "Installing WordOps" else - _run wo_install_travis "Installing WordOps" + _run wo_travis_install "Installing WordOps" fi if [ -x /usr/local/bin/ee ]; then _run wo_upgrade_nginx "Upgrading Nginx" diff --git a/setup.cfg b/setup.cfg index 0e6f9dd..4ee5196 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,4 +9,7 @@ cover-inclusive=1 cover-erase=1 cover-html=1 cover-html-dir=coverage_report/ -where=tests/ \ No newline at end of file +where=tests/ + +[metadata] +license-file = LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 160ee85..a0f765e 100644 --- a/setup.py +++ b/setup.py @@ -10,24 +10,26 @@ with open("README.md", "r") as fh: conf = [] templates = [] +short_description = """An essential toolset that eases WordPress + site and server administration""" + for name in glob.glob('config/plugins.d/*.conf'): conf.insert(1, name) for name in glob.glob('wo/cli/templates/*.mustache'): templates.insert(1, name) -if os.geteuid() == 0: - if not os.path.exists('/var/log/wo/'): - os.makedirs('/var/log/wo/') +if not os.path.exists('/var/log/wo/'): + os.makedirs('/var/log/wo/') - if not os.path.exists('/var/lib/wo/tmp/'): - os.makedirs('/var/lib/wo/tmp/') +if not os.path.exists('/var/lib/wo/tmp/'): + os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.9.9.2', - description='WordPress & server administration toolset', + version='3.9.9.3', + description=short_description, long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -61,7 +63,6 @@ setup(name='wordops', 'SQLAlchemy >= 1.3.8', 'requests >= 2.22.0', 'distro >= 1.4.0', - 'apt-mirror-updater >= 6.1', ], extras_require={ # Optional 'testing': ['nose', 'coverage'], diff --git a/wo/cli/plugins/secure.py b/wo/cli/plugins/secure.py index 889f765..719e0b5 100644 --- a/wo/cli/plugins/secure.py +++ b/wo/cli/plugins/secure.py @@ -3,7 +3,6 @@ import os from cement.core import handler, hook from cement.core.controller import CementBaseController, expose - from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log @@ -70,7 +69,7 @@ class WOSecureController(CementBaseController): WOGit.add(self, ["/etc/nginx"], msg="Add Nginx to into Git") pargs = self.app.pargs - passwd = RANDOM.gen(self, length='24') + passwd = RANDOM.long(self) if not pargs.user_input: username = input("Provide HTTP authentication user " "name [{0}] :".format(WOVar.wo_user)) @@ -109,11 +108,12 @@ class WOSecureController(CementBaseController): Log.info(self, "Please enter a valid port number ") pargs.user_input = input("WordOps " "admin port [22222]:") - if not pargs.user_input: + else: port = input("WordOps admin port [22222]:") if port == "": port = 22222 - while (not port.isdigit()) and (port != "") and (not port < 65536): + while ((not port.isdigit()) and (not port != "") and + (not port < 65536)): Log.info(self, "Please Enter valid port number :") port = input("WordOps admin port [22222]:") pargs.user_input = port @@ -220,9 +220,14 @@ class WOSecureController(CementBaseController): Log.info(self, "Please Enter valid port number :") port = input("Server SSH port [22]:") pargs.user_input = port - WOShellExec.cmd_exec(self, "sed -i \"s/Port.*/Port " - "{port}/\" /etc/ssh/sshd_config" - .format(port=pargs.user_input)) + if WOFileUtils.grepcheck(self, '/etc/ssh/sshd_config', '#Port'): + WOShellExec.cmd_exec(self, "sed -i \"s/#Port.*/Port " + "{port}/\" /etc/ssh/sshd_config" + .format(port=pargs.user_input)) + else: + WOShellExec.cmd_exec(self, "sed -i \"s/Port.*/Port " + "{port}/\" /etc/ssh/sshd_config" + .format(port=pargs.user_input)) # allow new ssh port if ufw is enabled if os.path.isfile('/etc/ufw/ufw.conf'): # add rule for proftpd with UFW diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 136aeaf..e4415b5 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -94,7 +94,7 @@ class WOStackUpgradeController(CementBaseController): if WOAptGet.is_installed(self, 'nginx-custom'): apt_packages = apt_packages + WOVar.wo_nginx else: - if os.path.isfile(self, '/usr/sbin/nginx'): + if os.path.isfile('/usr/sbin/nginx'): Log.info(self, "Updating Nginx templates") post_pref(self, WOVar.wo_nginx, []) else: @@ -186,15 +186,14 @@ class WOStackUpgradeController(CementBaseController): self.app.args.print_help() else: if (apt_packages): - if not (set(["php7.2-fpm"]).issubset(set(apt_packages)) and - set(["php7.3-fpm"]).issubset(set(apt_packages)) and - set(["nginx-custom", - "nginx-wo"]).issubset(set(apt_packages)) and - set(['mariadb-server']).issubset(set(apt_packages))): + if (("php7.2-fpm" not in apt_packages) and + ("php7.3-fpm" not in apt_packages) and + ("nginx-custom" not in apt_packages) and + ("mariadb-server" not in apt_packages)): pass else: Log.info( - self, "Your site may be down for few seconds if " + self, "Your sites may be down for few seconds if " "you are upgrading Nginx, PHP-FPM, MariaDB or Redis") # Check prompt if ((not pargs.no_prompt) and (not pargs.force)): @@ -219,7 +218,6 @@ class WOStackUpgradeController(CementBaseController): # check if nginx upgrade is blocked if os.path.isfile( '/etc/apt/preferences.d/nginx-block'): - apt_packages.remove(WOVar.wo_nginx) post_pref(self, WOVar.wo_nginx, [], True) # upgrade packages WOAptGet.install(self, apt_packages) diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index f7ba7b6..5192cdd 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -48,9 +48,11 @@ class WORepo(): Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to add repo") if ppa is not None: - WOShellExec.cmd_exec( - self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" - .format(ppa_name=ppa)) + if WOShellExec.cmd_exec( + self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" + .format(ppa_name=ppa)): + return True + return False def remove(self, ppa=None, repo_url=None): """ diff --git a/wo/core/aptget.py b/wo/core/aptget.py index d3c2a81..4e6fa40 100644 --- a/wo/core/aptget.py +++ b/wo/core/aptget.py @@ -18,11 +18,11 @@ class WOAptGet(): """ try: with open('/var/log/wo/wordops.log', 'a') as f: - proc = subprocess.Popen('apt-mirror-updater -u', - shell=True, - stdin=None, stdout=f, - stderr=subprocess.PIPE, - executable="/bin/bash") + proc = subprocess.Popen( + 'DEBIAN_FRONTEND=noninteractive apt-get update ' + '--allow-releaseinfo-change', + shell=True, stdin=None, stdout=f, + stderr=subprocess.PIPE, executable="/bin/bash") proc.wait() output, error_output = proc.communicate() @@ -83,18 +83,16 @@ class WOAptGet(): """ try: with open('/var/log/wo/wordops.log', 'a') as f: - proc = subprocess.Popen("DEBIAN_FRONTEND=noninteractive " - "apt-get dist-upgrade " - "--option=Dpkg::options::=" - "--force-confdef " - "--option=Dpkg::options::=" - "--force-unsafe-io " - "--option=Dpkg::options::=" - "--force-confold " - "--assume-yes --quiet ", - shell=True, stdin=None, - stdout=f, stderr=f, - executable="/bin/bash") + proc = subprocess.Popen( + "DEBIAN_FRONTEND=noninteractive " + "apt-get dist-upgrade " + "--option=Dpkg::options::=--force-confdef " + "--option=Dpkg::options::=--force-unsafe-io " + "--option=Dpkg::options::=--force-confold " + "--assume-yes --quiet ", + shell=True, stdin=None, + stdout=f, stderr=f, + executable="/bin/bash") proc.wait() if proc.returncode == 0: @@ -228,16 +226,18 @@ class WOAptGet(): WORepo.add(self, repo_url=repo_url) if repo_key is not None: WORepo.add_key(self, repo_key) - proc = subprocess.Popen("apt-get update && " - "DEBIAN_FRONTEND=noninteractive " - "apt-get install -o " - "Dpkg::Options::=\"--force-confdef\"" - " -o " - "Dpkg::Options::=\"--force-confold\"" - " -y --download-only {0}" - .format(packages), shell=True, - stdin=None, stdout=f, stderr=f, - executable="/bin/bash") + proc = subprocess.Popen( + "DEBIAN_FRONTEND=noninteractive apt-get update " + "--allow-releaseinfo-change && " + "DEBIAN_FRONTEND=noninteractive " + "apt-get install -o " + "Dpkg::Options::=\"--force-confdef\"" + " -o " + "Dpkg::Options::=\"--force-confold\"" + " -y --download-only {0}" + .format(packages), shell=True, + stdin=None, stdout=f, stderr=f, + executable="/bin/bash") proc.wait() if proc.returncode == 0: diff --git a/wo/core/random.py b/wo/core/random.py index f6ff3b0..a0bc7f6 100644 --- a/wo/core/random.py +++ b/wo/core/random.py @@ -5,8 +5,14 @@ import string class RANDOM: """Random strings generator""" - def gen(self, length='24'): + def long(self): + long_random = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(24)]) + return long_random + + def short(self): short_random = ''.join([random.choice (string.ascii_letters + string.digits) - for n in range(length)]) + for n in range(24)]) return short_random diff --git a/wo/core/variables.py b/wo/core/variables.py index 67c86dd..5e452b7 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -13,7 +13,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.9.2" + wo_version = "3.9.9.3" # WordOps packages versions wo_wp_cli = "2.3.0" wo_adminer = "4.7.3"