Fix utf8 encoding when reading files

CHANGELOG.md

@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - netdata configuration
 - extplorer installation
 - add LANG='en_US.UTF-8' in install script
+- Read public_suffix list with utf8 encoding. Issue [#128](https://github.com/WordOps/WordOps/issues/128)
 
 ### v3.9.8.7 - 2019-08-31
 

install

@@ -103,7 +103,9 @@ fi
 # 1- Update the apt sewers with fresh info
 ###
 export DEBIAN_FRONTEND=noninteractive
+unset LANG
 export LANG='en_US.UTF-8'
+export LC_ALL='C.UTF-8'
 
 [ -z "$wo_travis" ] && {
     apt-get update -qq

wo/cli/plugins/site.py

@@ -429,7 +429,6 @@ class WOSiteCreateController(CementBaseController):
 
         pargs.site_name = pargs.site_name.strip()
         (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name)
-        (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain)
         if not wo_domain.strip():
             Log.error("Invalid domain name, "
                       "Provide valid domain name")
@@ -566,8 +565,10 @@ class WOSiteCreateController(CementBaseController):
 
             if data['php73']:
                 php_version = "7.3"
+                php73 = 1
             else:
                 php_version = "7.2"
+                php73 = 0
 
             addNewSite(self, wo_domain, stype, cache, wo_site_webroot,
                        php_version=php_version)
@@ -742,6 +743,7 @@ class WOSiteCreateController(CementBaseController):
                       "`tail /var/log/wo/wordops.log` and please try again")
 
         if pargs.letsencrypt:
+            (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain)
             data['letsencrypt'] = True
             letsencrypt = True
             if data['letsencrypt'] is True:
@@ -956,7 +958,6 @@ class WOSiteUpdateController(CementBaseController):
         pargs.site_name = pargs.site_name.strip()
         (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name)
         wo_site_webroot = WOVariables.wo_webroot + wo_domain
-        (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain)
         check_site = getSiteInfo(self, wo_domain)
 
         if check_site is None:
@@ -1149,6 +1150,53 @@ class WOSiteUpdateController(CementBaseController):
                              "site")
                 pargs.php73 = False
 
+        if pargs.letsencrypt:
+            (wo_domain_type, wo_root_domain) = GetDomainlevel(wo_domain)
+            if pargs.letsencrypt == 'on':
+                data['letsencrypt'] = True
+                letsencrypt = True
+                if ((wo_domain_type == 'subdomain') and
+                        (not pargs.letsencrypt == 'wildcard')):
+                    wo_subdomain = True
+                else:
+                    wo_subdomain = False
+                wo_wildcard = False
+            elif pargs.letsencrypt == 'subdomain':
+                data['letsencrypt'] = True
+                letsencrypt = True
+                wo_subdomain = True
+                wo_wildcard = False
+            elif pargs.letsencrypt == 'wildcard':
+                data['letsencrypt'] = True
+                letsencrypt = True
+                wo_wildcard = True
+                wo_subdomain = False
+            elif pargs.letsencrypt == 'off':
+                data['letsencrypt'] = False
+                letsencrypt = False
+                wo_subdomain = False
+                wo_wildcard = False
+            elif pargs.letsencrypt == 'clean':
+                data['letsencrypt'] = False
+                letsencrypt = False
+                wo_subdomain = False
+                wo_wildcard = False
+            elif pargs.letsencrypt == 'purge':
+                data['letsencrypt'] = False
+                letsencrypt = False
+                wo_subdomain = False
+                wo_wildcard = False
+
+            if not wo_subdomain:
+                if letsencrypt is check_ssl:
+                    if letsencrypt is False:
+                        Log.error(self, "SSl is not configured for given "
+                                  "site")
+                    elif letsencrypt is True:
+                        Log.error(self, "SSl is already configured for given "
+                                  "site")
+                    pargs.letsencrypt = False
+
         # --letsencrypt=renew code goes here
         if pargs.letsencrypt == "renew" and not pargs.all:
             expiry_days = SSL.getExpirationDays(self, wo_domain)
@@ -1236,52 +1284,6 @@ class WOSiteUpdateController(CementBaseController):
                               "site", False)
                     return 0
 
-        if pargs.letsencrypt:
-            if pargs.letsencrypt == 'on':
-                data['letsencrypt'] = True
-                letsencrypt = True
-                if ((wo_domain_type == 'subdomain') and
-                        (not pargs.letsencrypt == 'wildcard')):
-                    wo_subdomain = True
-                else:
-                    wo_subdomain = False
-                wo_wildcard = False
-            elif pargs.letsencrypt == 'subdomain':
-                data['letsencrypt'] = True
-                letsencrypt = True
-                wo_subdomain = True
-                wo_wildcard = False
-            elif pargs.letsencrypt == 'wildcard':
-                data['letsencrypt'] = True
-                letsencrypt = True
-                wo_wildcard = True
-                wo_subdomain = False
-            elif pargs.letsencrypt == 'off':
-                data['letsencrypt'] = False
-                letsencrypt = False
-                wo_subdomain = False
-                wo_wildcard = False
-            elif pargs.letsencrypt == 'clean':
-                data['letsencrypt'] = False
-                letsencrypt = False
-                wo_subdomain = False
-                wo_wildcard = False
-            elif pargs.letsencrypt == 'purge':
-                data['letsencrypt'] = False
-                letsencrypt = False
-                wo_subdomain = False
-                wo_wildcard = False
-
-            if not wo_subdomain:
-                if letsencrypt is check_ssl:
-                    if letsencrypt is False:
-                        Log.error(self, "SSl is not configured for given "
-                                  "site")
-                    elif letsencrypt is True:
-                        Log.error(self, "SSl is already configured for given "
-                                  "site")
-                    pargs.letsencrypt = False
-
         if data and (not pargs.php73):
             if old_php73 is True:
                 data['php73'] = True
@@ -1468,6 +1470,9 @@ class WOSiteUpdateController(CementBaseController):
                 elif (pargs.letsencrypt == "clean" or
                       pargs.letsencrypt == "purge"):
                     removeAcmeConf(self, wo_domain)
+                    # find all broken symlinks
+                    sympath = "/var/www"
+                    WOFileUtils.findBrokenSymlink(self, sympath)
                 if not WOService.reload_service(self, 'nginx'):
                     Log.error(self, "service nginx reload failed. "
                               "check issues with `nginx -t` command")
@@ -1872,7 +1877,6 @@ class WOSiteDeleteController(CementBaseController):
 
         pargs.site_name = pargs.site_name.strip()
         (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name)
-        wo_domain_type, wo_root_domain = GetDomainlevel(wo_domain)
         wo_db_name = ''
         wo_prompt = ''
         wo_nginx_prompt = ''

wo/cli/plugins/site_functions.py

@@ -1335,10 +1335,8 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='',
         if os.path.isfile('/etc/nginx/sites-available/{0}'
                           .format(domain)):
             removeNginxConf(self, domain)
-        if (os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc'
+        if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc'
-                          .format(domain)) or
+                         .format(domain)):
-                os.path.isdir('/etc/letsencrypt/live/{0}'
-                              .format(domain))):
             removeAcmeConf(self, domain)
 
     if webroot:
@@ -1486,7 +1484,7 @@ def checkWildcardExist(self, wo_domain_name):
     # define new csv dialect
     csv.register_dialect('acmeconf', delimiter='|')
     # open file
-    certfile = open('/var/lib/wo/cert.csv', mode='rt', encoding='utf-8')
+    certfile = open('/var/lib/wo/cert.csv', mode='r', encoding='utf-8')
     reader = csv.reader(certfile, 'acmeconf')
     wo_wildcard_domain = ("*.{0}".format(wo_domain_name))
     for row in reader:

wo/cli/plugins/stack.py

@@ -191,8 +191,6 @@ class WOStackController(CementBaseController):
             # Redis
             if pargs.redis:
                 pargs.php = True
-                WOTemplate.addAptPackage(
-                    self, 'redis-server', WOVariables.wo_redis)
                 if not WOAptGet.is_installed(self, 'redis-server'):
                     apt_packages = apt_packages + WOVariables.wo_redis
 
@@ -248,8 +246,8 @@ class WOStackController(CementBaseController):
             # WP-CLI
             if pargs.wpcli:
                 Log.debug(self, "Setting packages variable for WP-CLI")
-                if (not os.path.isfile("/usr/local/bin/wp") and not
+                if ((not os.path.isfile("/usr/local/bin/wp")) and
-                        os.path.isfile("/usr/bin/wp")):
+                        (not os.path.isfile("/usr/bin/wp"))):
                     packages = packages + [["https://github.com/wp-cli/wp-cli/"
                                             "releases/download/v{0}/"
                                             "wp-cli-{0}.phar"

wo/core/apt_repo.py

@@ -47,7 +47,7 @@ class WORepo():
                 Log.debug(self, "{0}".format(e))
                 Log.error(self, "Unable to add repo")
         if ppa is not None:
-            WOShellExec.cmd_exec(self, "add-apt-repository -yu '{ppa_name}'"
+            WOShellExec.cmd_exec(self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'"
                                  .format(ppa_name=ppa))
 
     def remove(self, ppa=None, repo_url=None):

wo/core/domainvalidate.py

@@ -29,14 +29,14 @@ def GetDomainlevel(domain):
     """
         This function returns the domain type : domain, subdomain,
     """
-    domain_name = domain.split('.')
+    domain_name = domain.split('.').lower()
     if domain_name[0] == 'www':
         domain_name = domain_name[1:]
     domain_type = ''
     if os.path.isfile("/var/lib/wo/public_suffix_list.dat"):
         # Read mode opens a file for reading only.
         Suffix_file = open(
-            "/var/lib/wo/public_suffix_list.dat", mode='rt', encoding='utf-8')
+            "/var/lib/wo/public_suffix_list.dat", encoding='utf-8', )
         # Read all the lines into a list.
         for domain_suffix in Suffix_file:
             if (str(domain_suffix).strip()) == ('.'.join(domain_name[1:])):

wo/core/fileutils.py

@@ -109,7 +109,7 @@ class WOFileUtils():
             rstr: replace string
         """
         try:
-            Log.debug(self, "Doning search and replace, File:{0},"
+            Log.debug(self, "Doing search and replace, File:{0},"
                       "Source string:{1}, Dest String:{2}"
                       .format(fnm, sstr, rstr))
             for line in fileinput.input(fnm, inplace=True):