diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3c96045..d6ac914 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: build: strategy: matrix: - platform: [ubuntu-20.04, ubuntu-18.04, ubuntu-16.04] + platform: [ubuntu-20.04, ubuntu-18.04] runs-on: ${{ matrix.platform }} @@ -30,8 +30,9 @@ jobs: echo "$PWD" sudo apt update -qq > /dev/null 2>&1 sudo LC_ALL=C.UTF-8 add-apt-repository ppa:ondrej/php -y > /dev/null 2>&1 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 188C9FB063F0247A sudo rm -rf /etc/mysql /var/lib/mysql - sudo apt-get purge --option=Dpkg::options::=--force-all --assume-yes graphviz* redis* php* mysql* > /dev/null 2>&1 + sudo apt-get purge --option=Dpkg::options::=--force-all --assume-yes graphviz* redis* php* mysql* nginx* > /dev/null 2>&1 sudo apt-get install -qq git ccze tree > /dev/null 2>&1 sudo apt-get -qq autoremove --purge > /dev/null 2>&1 sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > $HOME/.gitconfig' diff --git a/config/wo.conf b/config/wo.conf index 2a9fa94..37376f7 100644 --- a/config/wo.conf +++ b/config/wo.conf @@ -79,7 +79,12 @@ keylength = "ec-384" [php] ### Default PHP version -version = 7.3 +version = 7.4 + +[mariadb] + +### Default MariaDB release +release = 10.5 [update] diff --git a/install b/install index d696f5f..1c52d8d 100755 --- a/install +++ b/install @@ -4,12 +4,12 @@ # ------------------------------------------------------------------------- # Website: https://wordops.net # GitHub: https://github.com/WordOps/WordOps -# Copyright (c) 2019-2020 - WordOps +# Copyright (c) 2019-2021 - WordOps # This script is licensed under M.I.T # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo -E bash wo # ------------------------------------------------------------------------- -# Version 3.13.2 - 2020-10-27 +# Version 3.13.3 - 2021-05-20 # ------------------------------------------------------------------------- # CONTENTS @@ -210,18 +210,18 @@ wo_install_dep() { apt-get --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip python3-pip python3-apt python3-venv gcc python3-dev sqlite3 git tar software-properties-common pigz \ gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp zstd idn \ - python3-distutils-extra libapt-pkg-dev >/dev/null 2>&1 - curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - + python3-distutils-extra libapt-pkg-dev bash-completion >/dev/null 2>&1 + curl -sL https://build.opensuse.org/projects/home:virtubox/public_key | apt-key add - else # install dependencies apt-get --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip dirmngr sudo python3-pip python3-apt python3-venv gcc python3-dev ca-certificates sqlite3 git tar \ software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp zstd idn \ - python3-distutils-extra libapt-pkg-dev >/dev/null 2>&1 + python3-distutils-extra libapt-pkg-dev bash-completion >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key - curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_10/Release.key | apt-key add - + curl -sL https://build.opensuse.org/projects/home:virtubox/public_key | apt-key add - fi locale-gen en # enable unattended upgades @@ -513,7 +513,7 @@ wo_travis_install() { elif [ "$wo_distro_codename" = "bionic" ]; then python3 -m pip install git+https://salsa.debian.org/apt-team/python-apt.git@1.6.y#egg=python-apt else - python3 -m pip install git+https://salsa.debian.org/apt-team/python-apt.git@1.1.y-xenial#egg=python-apt + python3 -m pip install git+https://salsa.debian.org/apt-team/python-apt.git@1.1.0_beta1ubuntu0.16.04.9#egg=python-apt fi fi fi @@ -553,9 +553,9 @@ wo_upgrade_nginx() { # add new Nginx repository if [ "$wo_distro_version" == "jessie" ]; then # import the respository key for updates - curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_8.0/Release.key | apt-key add - + curl -sL https://build.opensuse.org/projects/home:virtubox/public_key | apt-key add - else - curl -sL https://download.opensuse.org/repositories/home:virtubox:WordOps/Debian_9.0/Release.key | apt-key add - + curl -sL https://build.opensuse.org/projects/home:virtubox/public_key | apt-key add - fi # install new nginx package @@ -713,7 +713,7 @@ wo_woconf() { echo -e '\n[letsencrypt]\n\nkeylength = "ec-384"' >>/etc/wo/wo.conf fi if ! grep -q "php" /etc/wo/wo.conf; then - echo -e '\n[php]\n\nversion = 7.3' >>/etc/wo/wo.conf + echo -e '\n[php]\n\nversion = 7.4' >>/etc/wo/wo.conf fi fi } diff --git a/requirements.txt b/requirements.txt index 9a33f07..0799eb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ cement==2.10.12 pystache>=0.5.4 pynginxconfig>=0.3.4 -PyMySQL>=0.10.0 -psutil>=5.7.2 -sh>=1.12.14 -SQLAlchemy>=1.3.18 +PyMySQL>=0.10.1 +psutil>=5.7.3 +sh>=1.14.1 +SQLAlchemy>=1.3.20 requests>=2.24.0 -distro>=1.4.0 +distro>=1.5.0 argcomplete>=1.12.0 -colorlog>=4.2.1 \ No newline at end of file +colorlog>=4.6.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 1de363f..340893f 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if os.geteuid() == 0: os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.13.2', + version='3.13.3', description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', @@ -64,14 +64,14 @@ setup(name='wordops', 'cement == 2.10.12', 'pystache >= 0.5.4', 'pynginxconfig >= 0.3.4', - 'PyMySQL >= 0.10.0', - 'psutil >= 5.7.2', - 'sh >= 1.12.14', - 'SQLAlchemy >= 1.3.18', + 'PyMySQL >= 0.10.1', + 'psutil >= 5.7.3', + 'sh >= 1.14.1', + 'SQLAlchemy >= 1.3.20', 'requests >= 2.24.0', - 'distro >= 1.4.0', + 'distro >= 1.5.0', 'argcomplete >= 1.12.0', - 'colorlog >= 4.2.1', + 'colorlog >= 4.6.2', ], extras_require={ # Optional 'testing': ['nose', 'coverage'], diff --git a/tests/travis.sh b/tests/travis.sh index 04e8ac8..7ee3096 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -32,9 +32,9 @@ echo -e "${CGREEN}#############################################${CEND}" echo -e ' stack install ' echo -e "${CGREEN}#############################################${CEND}" if [ "$wo_distro" != "xenial" ]; then -stack_list='nginx php php73 php74 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis sendmail phpredisadmin mysqltuner utils ufw ngxblocker cheat nanorc' +stack_list='nginx php php73 php74 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis sendmail phpredisadmin mysqltuner utils ufw cheat nanorc' else -stack_list='nginx php php73 php74 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw ngxblocker cheat nanorc' +stack_list='nginx php php73 php74 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw cheat nanorc' fi for stack in $stack_list; do echo -ne " Installing $stack [..]\r" diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 516c9ed..d824641 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -810,11 +810,11 @@ def sitebackup(self, data): if data['wo_db_name']: Log.info(self, 'Backing up database \t\t', end='') try: - if not WOShellExec.cmd_exec(self, "mysqldump --single-transaction " - "{0} | zstd -T0 -c" - "> {1}/{0}.zst" - .format(data['wo_db_name'], - backup_path)): + if not WOShellExec.cmd_exec( + self, "mysqldump --single-transaction --hex-blob " + "{0} | zstd -c > {1}/{0}.zst" + .format(data['wo_db_name'], + backup_path)): Log.info(self, "[" + Log.ENDC + Log.FAIL + "Fail" + Log.OKBLUE + "]") raise SiteError("mysqldump failed to backup database") diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index fd82f11..0053128 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -146,8 +146,18 @@ class WOStackController(CementBaseController): pargs.proftpd = True if pargs.web: + if self.app.config.has_section('php'): + config_php_ver = self.app.config.get( + 'php', 'version') + if config_php_ver == '7.2': + pargs.php72 = True + elif config_php_ver == '7.3': + pargs.php73 = True + elif config_php_ver == '7.4': + pargs.php74 = True + else: + pargs.php74 = True pargs.nginx = True - pargs.php73 = True pargs.mysql = True pargs.wpcli = True pargs.sendmail = True @@ -476,7 +486,7 @@ class WOStackController(CementBaseController): if not (WOAptGet.is_installed(self, 'php7.2-fpm') or WOAptGet.is_installed(self, 'php7.3-fpm') or WOAptGet.is_installed(self, 'php7.4-fpm')): - pargs.php = True + pargs.php74 = True Log.debug(self, "Setting packages variable for utils") packages = packages + [[ "https://raw.githubusercontent.com" diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index ca619a9..7ad72ed 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -51,6 +51,7 @@ def pre_pref(self, apt_packages): [client] user = root password = {chars} + socket = /run/mysqld/mysqld.sock """.format(chars=chars) config = configparser.ConfigParser() config.read_string(mysql_config) @@ -924,7 +925,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): # create mysql config if it doesn't exist if "mariadb-server" in apt_packages: WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") - if not os.path.isfile("/etc/mysql/my.cnf"): + if not os.path.exists("/etc/mysql/my.cnf"): config = ("[mysqld]\nwait_timeout = 30\n" "interactive_timeout=60\nperformance_schema = 0" "\nquery_cache_type = 1") @@ -939,16 +940,11 @@ def post_pref(self, apt_packages, packages, upgrade=False): config = configparser.ConfigParser() config.read('/etc/mysql/conf.d/my.cnf.tmp') chars = config['client']['password'] - WOShellExec.cmd_exec( - self, - 'mysql -e "ALTER USER root@localhost ' - 'IDENTIFIED VIA mysql_native_password;"') WOShellExec.cmd_exec( self, 'mysql -e "SET PASSWORD = ' - 'PASSWORD(\'{0}\');"'.format(chars)) - WOShellExec.cmd_exec( - self, 'mysql -e "flush privileges;"') + 'PASSWORD(\'{0}\'); flush privileges;"' + .format(chars)) WOFileUtils.mvfile( self, '/etc/mysql/conf.d/my.cnf.tmp', '/etc/mysql/conf.d/my.cnf') @@ -956,6 +952,30 @@ def post_pref(self, apt_packages, packages, upgrade=False): Log.error(self, "Unable to set MySQL password") WOGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git") + elif os.path.exists('/etc/mysql/conf.d/my.cnf'): + if ((WOAptGet.is_installed( + self, 'mariadb-server-10.5')) and + not (WOFileUtils.grepcheck( + self, '/etc/mysql/conf.d/my.cnf', 'socket'))): + try: + config = configparser.ConfigParser() + config.read('/etc/mysql/conf.d/my.cnf') + chars = config['client']['password'] + WOShellExec.cmd_exec( + self, + 'mysql -e "ALTER USER root@localhost ' + 'IDENTIFIED VIA unix_socket OR ' + 'mysql_native_password; ' + 'SET PASSWORD = PASSWORD(\'{0}\'); ' + 'flush privileges;"'.format(chars)) + WOFileUtils.textappend( + self, '/etc/mysql/conf.d/my.cnf', + 'socket = /run/mysqld/mysqld.sock') + except CommandExecutionError: + Log.error(self, "Unable to set MySQL password") + WOGit.add(self, ["/etc/mysql"], + msg="Adding MySQL into Git") + Log.wait(self, "Tuning MariaDB configuration") if not os.path.isfile("/etc/mysql/my.cnf.default-pkg"): WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", @@ -1660,7 +1680,7 @@ def pre_stack(self): 'sysctl.mustache', data, True) # use tcp_bbr congestion algorithm only on new kernels if (WOVar.wo_platform_codename == 'bionic' or - WOVar.wo_platform_codename == 'disco' or + WOVar.wo_platform_codename == 'focal' or WOVar.wo_platform_codename == 'buster'): try: WOShellExec.cmd_exec( diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 01b3188..a775565 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -78,8 +78,8 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if os.path.exists('/etc/systemd/system/mysql.service'): - services = services + ['mysql'] + if os.path.exists('/lib/systemd/system/mariadb.service'): + services = services + ['mariadb'] else: Log.info(self, "MySQL is not installed") else: @@ -176,8 +176,8 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if os.path.exists('/etc/systemd/system/mysql.service'): - services = services + ['mysql'] + if os.path.exists('/lib/systemd/system/mariadb.service'): + services = services + ['mariadb'] else: Log.info(self, "MySQL is not installed") else: @@ -275,8 +275,8 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if os.path.exists('/etc/systemd/system/mysql.service'): - services = services + ['mysql'] + if os.path.exists('/lib/systemd/system/mariadb.service'): + services = services + ['mariadb'] else: Log.info(self, "MySQL is not installed") else: @@ -377,8 +377,8 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if os.path.exists('/etc/systemd/system/mysql.service'): - services = services + ['mysql'] + if os.path.exists('/lib/systemd/system/mariadb.service'): + services = services + ['mariadb'] else: Log.info(self, "MySQL is not installed") else: @@ -487,7 +487,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if os.path.exists('/etc/systemd/system/mysql.service'): + if os.path.exists('/lib/systemd/system/mariadb.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache index 47568ad..15300b9 100644 --- a/wo/cli/templates/my.mustache +++ b/wo/cli/templates/my.mustache @@ -64,10 +64,10 @@ max_heap_table_size = {{tmp_table_size}}M # This replaces the startup script and checks MyISAM tables if needed # the first time they are touched. On error, make copy and try a repair. myisam_recover_options = BACKUP -key_buffer_size = 128M +key_buffer_size = 16M open-files-limit = 500000 table_open_cache = 16000 -myisam_sort_buffer_size = 512M +myisam_sort_buffer_size = 128M concurrent_insert = 2 read_buffer_size = 2M read_rnd_buffer_size = 1M diff --git a/wo/core/mysql.py b/wo/core/mysql.py index cdab230..1ad8d67 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -110,11 +110,11 @@ class WOMysql(): Log.info(self, "Backing up {0} database".format(dbs)) p1 = subprocess.Popen( "/usr/bin/mysqldump {0} --max_allowed_packet=1024M " - "--single-transaction ".format(dbs), + "--single-transaction --hex-blob".format(dbs), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen( - "/usr/bin/zstd -T0 -c > " + "/usr/bin/zstd -c > " "/var/lib/wo-backup/mysql/{0}{1}.sql.zst" .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True) @@ -130,12 +130,12 @@ class WOMysql(): Log.info(self, "Backing up all databases") p1 = subprocess.Popen( "/usr/bin/mysqldump --all-databases " - "--max_allowed_packet=1024M " + "--max_allowed_packet=1024M --hex-blob " "--single-transaction --events", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen( - "/usr/bin/zstd -T0 -c > " + "/usr/bin/zstd -c > " "/var/lib/wo-backup/mysql/fulldump-{0}.sql.zst" .format(WOVar.wo_date), stdin=p1.stdout, shell=True) diff --git a/wo/core/variables.py b/wo/core/variables.py index b99a83c..502fc5e 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -14,7 +14,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.13.2" + wo_version = "3.13.3" # WordOps packages versions wo_wp_cli = "2.4.0" wo_adminer = "4.7.5"