Combined whitelist updated

Change function visibility to allow usage as library

scan.php

@@ -104,7 +104,7 @@ class MalwareScanner
     }
 
     //Handles pattern loading and saving to the class object
-    private function initializePatterns()
+    public function initializePatterns()
     {
         $dir = dirname(__FILE__);
         //Loads either the primary scanning patterns or the base64 patterns depending on -b/--base64 flag
@@ -176,7 +176,7 @@ class MalwareScanner
     }
 
     //Loads the whitelist file
-    private function loadWhitelist()
+    public function loadWhitelist()
     {
         if (!is_file(__DIR__ . '/whitelist.txt')) {
             return;
@@ -511,7 +511,7 @@ class MalwareScanner
         }
 
         if ($this->outputFormat) {
-            $map = [
+            $map = array(
                 '%S' => $state,
                 '%T' => $ctime,
                 '%M' => $hash,
@@ -519,9 +519,9 @@ class MalwareScanner
                 '%P' => $pattern,
                 '%C' => $comment,
                 '%L' => $lineNumber,
-            ];
+            );
         } else {
-            $map = [
+            $map = array(
                 '%S' => $state_color . '# ' . $state . $this->ANSI_OFF,
                 '%T' => $this->ANSI_BLUE . $ctime . $this->ANSI_OFF,
                 '%M' => $this->ANSI_BLUE . $hash . $this->ANSI_OFF,
@@ -529,7 +529,7 @@ class MalwareScanner
                 '%P' => $state_color . '#' . $pattern . $this->ANSI_OFF,
                 '%C' => $this->ANSI_BLUE . $comment . $this->ANSI_OFF,
                 '%L' => $lineNumber,
-            ];
+            );
         }
 
         if ($this->outputFormat) {
@@ -769,7 +769,7 @@ class MalwareScanner
         }
 
         $content = gzdecode(file_get_contents($file));
-        $this->combined_whitelist = [];
+        $this->combined_whitelist = array();
         $this->combined_whitelist_count = 0;
         foreach (explode("\n", $content) as $line) { // faster than strtok, but needs more memory
             if ($line) {

tools/bigdata/generate.php

@@ -15,8 +15,8 @@ function fetch($url, $file = false)
 
     $headers = array(
         // drupal suxx
-        'Cookie: pxvid=44e1b040-4dde-11e8-b1dc-f15e898556c7; _ga=GA1.2.2042202377.1525247839; _gat=1; _gid=GA1.2.1601332121.1550831838; _px2=eyJ1IjoiZDM3OTk1MDAtMzY4ZC0xMWU5LWI3MDItYTdlMDI1ZWZhZmI2IiwidiI6IjQ0ZTFiMDQwLTRkZGUtMTFlOC1iMWRjLWYxNWU4OTg1NTZjNyIsInQiOjE1NTA4MzIxMzc5MjcsImgiOiJjMjBhNTQzNGIxYWQwNWFiOWUzNTI2OWRjNTM1MjgzNjkxNzg5OTIxNGM4YmIzZDBkZTg5ZTIxMzY0NTc5Zjk3In0=; has_js=1; _pxvid=44e1b040-4dde-11e8-b1dc-f15e898556c7',
+        'Cookie: _px2=eyJ1IjoiZDZhNGM3MjAtYjZmNC0xMWVhLWI2MzMtNzk5YzRmZjM4ZmJkIiwidiI6IjQ0ZTFiMDQwLTRkZGUtMTFlOC1iMWRjLWYxNWU4OTg1NTZjNyIsInQiOjE1OTMwOTc2Mjg2NzAsImgiOiIzNzk5N2RkYTU3ZTI1NGY0ZDM5MmRiMWExNWZhZjhjNTZkMmM5NTZkZDJiZWVkZGVlZDc1MThiNTE5MTFjYzgwIn0=; _ga=GA1.2.2042202377.1525247839; _gat=1; _gid=GA1.2.1034461360.1593095881; has_js=1; _pxff_fp=1; _pxff_rf=1; pxvid=44e1b040-4dde-11e8-b1dc-f15e898556c7',
-        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.3 Safari/605.1.15',
+        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
     );
     curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
 
@@ -161,7 +161,7 @@ function fetch_typo3($fp)
                     continue;
                 }
                 $file = 'type3-' . $release->version . '.tar.gz';
-                fetch_archive($file, $release->url->tar, $release->checksums->tar->sha1, 'sha1');
+                fetch_archive($file, 'https://get.typo3.org' . $release->url->tar, $release->checksums->tar->sha1, 'sha1');
                 hash_archive($fp, $file);
             }
         }
@@ -185,38 +185,40 @@ function fetch_pagekit($fp)
 }
 
 // Ignored releases are: alpha, beta, rc, dev
-function fetch_drupal($fp, $versions)
+function fetch_drupal($fp)
 {
-    foreach ($versions as $version => $id) {
+    echo 'Fetching Drupal ' . PHP_EOL;
-        echo 'Fetching Drupal ' . $version . PHP_EOL;
 
-        $page = 0;
+    $page = 0;
-        $pages = false;
+    $pages = false;
-        do {
+    do {
-            $data = fetch('https://www.drupal.org/project/drupal/releases?api_version%5B%5D=' . $id . '&page=' .$page);
+        $data = fetch('https://www.drupal.org/project/drupal/releases?page=' . $page);
 
-            // pagination init
+        // pagination init
-            if ($pages === false && preg_match('/&amp;page=(\d+)">last »<\/a>/', $data, $m)) {
+        if ($pages === false && preg_match('/\?page=(\d+)">last »<\/a>/', $data, $m)) {
-                $pages = $m[1];
+            $pages = $m[1];
+        }
+
+        preg_match_all(
+            '/<a href="(\/project\/drupal\/releases\/(\d\.\d\.\d))">drupal/i',
+            $data,
+            $m
+        );
+        foreach ($m[1] as $k => $ver_uri) {
+            $ver_data = fetch('https://www.drupal.org' . $ver_uri);
+            if (!preg_match('/<span class="field-content hash">([a-z0-9]+)<\/span>/i', $ver_data, $ver_m)) {
+                die('Missing hash info: ' . $m[2][$k]);
             }
+            $file = 'drupal-' . $m[2][$k] . '.tar.gz';
+            fetch_archive($file, 'https://ftp.drupal.org/files/projects/' . $file, $ver_m[1], 'md5');
+            hash_archive($fp, $file);
+        }
 
-            preg_match_all(
+        if ($pages === false) {
-                '/data-th="Download">(.*?)<a href="(https:\/\/ftp\.drupal\.org\/files\/projects\/(drupal\-([0-9.]+)\.tar\.gz)).*?md5 hash">\s*([a-z0-9]{32})\s*<\/td>/is',
+            break;
-                $data,
+        }
-                $m
+        $page++;
-            );
+    } while ($page <= $pages);
-            foreach ($m[3] as $k => $file) {
-                fetch_archive($file, $m[2][$k], $m[5][$k], 'md5');
-                hash_archive($fp, $file);
-            }
-
-            if ($pages === false) {
-                break;
-            }
-
-            $page++;
-        }while($page <= $pages);
-    }
 }
 
 function fetch_joomla($fp, $versions)
@@ -272,24 +274,7 @@ fetch_jquery($fp);
 fetch_wordpress($fp);
 fetch_typo3($fp);
 fetch_pagekit($fp);
-fetch_drupal(
+fetch_drupal($fp);
-    $fp,
-    [
-        '9.x' => 39794,
-        '8.x' => 7234,
-        '7.x' => 103,
-        '6.x' => 87,
-        '5.x' => 78,
-        '4.7.x' => 79,
-        '4.6.x' => 80,
-        '4.5.x' => 81,
-        '4.4.x' => 82,
-        '4.3.x' => 83,
-        '4.2.x' => 84,
-        '4.1.x' => 85,
-        '4.0.x' => 86
-    ]
-);
 fetch_joomla($fp, ['3.0' => 3, '2.5' => 25, '1.5' => 15, '1.0' => 10]);
 
 fclose($fp);

whitelist.txt

@@ -275,3 +275,6 @@ fc8f1e9f0ff666af7beb3f61b055c0e8  /core/model/smarty/sysplugins/smarty_internal_
 bb127b5ce56b45e8b4b91de2e60dd9eb  /assets/components/googleanalytics/js/mgr/libs/highcharts.js
 7d7958bb0a9438a8966807f9202d0bce  /assets/components/tinymce/jscripts/tiny_mce/plugins/spellchecker/classes/PSpellShell.php
 3ee0a4d8a06cedc0a56f29e8f351ef72  /pclzip-2-8-2/pclzip.lib.php
+abfd2987afd1f66e3eed50bebbeb6750  /sucuri-scanner-1.8.24/src/base.lib.php
+78477b67cb223e4504689fef33119884  /sucuri-scanner-1.8.24/src/sitecheck.lib.php
+e48460f6ef0c911dc5ad558c57bfd52f  /sucuri-scanner-1.8.24/src/integrity.lib.php