Allowing multiple use of -d option and braces in path syntax, closes #56

Change function visibility to allow usage as library

README.md

@@ -35,6 +35,7 @@ Usage: php scan.php -d <directory>
     -o                   --output-format      Custom defined output format
     -j                   --wordpress-version  Version of wordpress to get md5 signatures
                          --combined-whitelist Combined whitelist
+                         --disable-stats      Disable statistics output
 ```
 
 Ignore argument could be used multiple times and accept glob style matching ex.: "`cache*`", "`??-cache.php`" or "`/cache`" etc.

scan.php

@@ -40,6 +40,7 @@ class MalwareScanner
     private $flagLineNumber = false;
     private $flagScanEverything = false;
     private $flagCombinedWhitelist = false;
+    private $flagDisableStats = false;
     private $outputFormat = '';
     private $whitelist = array();
     private $ignore = array();
@@ -68,16 +69,31 @@ class MalwareScanner
         if ($cli === true) {
             //Read Run Options
             $this->parseArgs();
-            $this->dir = realpath($this->dir);
+
+            $dirs = array();
+            if (is_array($this->dir)) {
+                // allow multiple directory aka. array
+                foreach ($this->dir as $path) {
+                    $dirs[] = realpath($path);
+                }
+            } elseif ($bpos = strpos($this->dir, '{')) {
+                // Check path has a "brace", expand it to subdirectories
+                foreach (glob($this->dir, GLOB_BRACE) as $path) {
+                    $dirs[] = realpath($path);
+                }
+            } else {
+                // only one directory specified
+                $dirs = array (realpath($this->dir));
+            }
 
             //Make sure a directory was specified.
-            if ($this->dir === '') {
+            if (empty($dirs)) {
                 $this->error('No directory specified or directory doesn\'t exist');
                 exit(-1);
             }
 
             //Initiate Scan
-            if (!$this->run($this->dir)) {
+            if (!$this->run($dirs)) {
                 exit(-1);
             }
         }
@@ -103,7 +119,7 @@ class MalwareScanner
     }
 
     //Handles pattern loading and saving to the class object
-    private function initializePatterns()
+    public function initializePatterns()
     {
         $dir = dirname(__FILE__);
         //Loads either the primary scanning patterns or the base64 patterns depending on -b/--base64 flag
@@ -175,7 +191,7 @@ class MalwareScanner
     }
 
     //Loads the whitelist file
-    private function loadWhitelist()
+    public function loadWhitelist()
     {
         if (!is_file(__DIR__ . '/whitelist.txt')) {
             return;
@@ -230,7 +246,8 @@ class MalwareScanner
                 'output-format:',
                 'wordpress-version:',
                 'scan-everything',
-                'combined-whitelist'
+                'combined-whitelist',
+                'disable-stats'
             )
         );
 
@@ -313,6 +330,9 @@ class MalwareScanner
         if (isset($options['combined-whitelist'])) {
             $this->setFlagCombinedWhitelist(true);
         }
+        if (isset($options['disable-stats'])) {
+            $this->setFlagDisableStats(true);
+        }
     }
 
     public function setExtensions(array $a)
@@ -401,6 +421,11 @@ class MalwareScanner
         $this->flagCombinedWhitelist = $b;
     }
 
+    public function setFlagDisableStats($b)
+    {
+        $this->flagDisableStats = $b;
+    }
+
     // @see http://stackoverflow.com/a/13914119
     private function pathMatches($path, $pattern, $ignoreCase = false)
     {
@@ -501,7 +526,7 @@ class MalwareScanner
         }
 
         if ($this->outputFormat) {
-            $map = [
+            $map = array(
                 '%S' => $state,
                 '%T' => $ctime,
                 '%M' => $hash,
@@ -509,9 +534,9 @@ class MalwareScanner
                 '%P' => $pattern,
                 '%C' => $comment,
                 '%L' => $lineNumber,
-            ];
+            );
         } else {
-            $map = [
+            $map = array(
                 '%S' => $state_color . '# ' . $state . $this->ANSI_OFF,
                 '%T' => $this->ANSI_BLUE . $ctime . $this->ANSI_OFF,
                 '%M' => $this->ANSI_BLUE . $hash . $this->ANSI_OFF,
@@ -519,7 +544,7 @@ class MalwareScanner
                 '%P' => $state_color . '#' . $pattern . $this->ANSI_OFF,
                 '%C' => $this->ANSI_BLUE . $comment . $this->ANSI_OFF,
                 '%L' => $lineNumber,
-            ];
+            );
         }
 
         if ($this->outputFormat) {
@@ -582,18 +607,11 @@ class MalwareScanner
      * - Fetch and load combined whitelist
      * - Calls the process and report functions.
      *
-     * @param $dir
+     * @param string|array $dir A directory path or a list of paths in array
      * @return bool
      */
     public function run($dir)
     {
-        // Make sure the input is a valid directory path.
-        $dir = rtrim($dir, '/');
-        if (!is_dir($dir)) {
-            $this->error('Specified path is not a directory: ' . $dir);
-            return false;
-        }
-
         $this->initializePatterns();
 
         $this->loadWhitelist();
@@ -603,8 +621,24 @@ class MalwareScanner
         }
 
         $start = time();
-        $this->process($dir . '/');
+
-        $this->report($start, $dir . '/');
+        if (!is_array($dir)) {
+            $dir = array ($dir);
+        }
+
+        foreach ($dir as $path) {
+            // Make sure the input is a valid directory path.
+            $path = rtrim($path, '/');
+            if (!is_dir($path)) {
+                $this->error('Specified path is not a directory: ' . $path);
+                return false;
+            }
+            $this->process($path . '/');
+        }
+
+        if (!$this->flagDisableStats) {
+            $this->report($start, implode(', ', $dir));
+        }
         return true;
     }
 
@@ -757,7 +791,7 @@ class MalwareScanner
         }
 
         $content = gzdecode(file_get_contents($file));
-        $this->combined_whitelist = [];
+        $this->combined_whitelist = array();
         $this->combined_whitelist_count = 0;
         foreach (explode("\n", $content) as $line) { // faster than strtok, but needs more memory
             if ($line) {
@@ -795,6 +829,7 @@ class MalwareScanner
         echo '    -o                   --output-format      Custom defined output format' . PHP_EOL;
         echo '    -j                   --wordpress-version  Version of wordpress to get md5 signatures' . PHP_EOL;
         echo '                         --combined-whitelist Combined whitelist' . PHP_EOL;
+        echo '                         --disable-stats      Disable statistics output' . PHP_EOL;
 
     }
 

tools/bigdata/generate.php

@@ -15,7 +15,8 @@ function fetch($url, $file = false)
 
     $headers = array(
         // drupal suxx
-        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.3 Safari/605.1.15',
+        'Cookie: _px2=eyJ1IjoiZDZhNGM3MjAtYjZmNC0xMWVhLWI2MzMtNzk5YzRmZjM4ZmJkIiwidiI6IjQ0ZTFiMDQwLTRkZGUtMTFlOC1iMWRjLWYxNWU4OTg1NTZjNyIsInQiOjE1OTMwOTc2Mjg2NzAsImgiOiIzNzk5N2RkYTU3ZTI1NGY0ZDM5MmRiMWExNWZhZjhjNTZkMmM5NTZkZDJiZWVkZGVlZDc1MThiNTE5MTFjYzgwIn0=; _ga=GA1.2.2042202377.1525247839; _gat=1; _gid=GA1.2.1034461360.1593095881; has_js=1; _pxff_fp=1; _pxff_rf=1; pxvid=44e1b040-4dde-11e8-b1dc-f15e898556c7',
+        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
     );
     curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
 
@@ -89,7 +90,7 @@ function fetch_jquery($fp)
     foreach ($m[1] as $k => $file) {
         if (!is_cached($file)) {
             echo 'Downloading: ' . 'https://code.jquery.com/' . $file . PHP_EOL;
-            $data = fetch('https://code.jquery.com/' . $file);
+            $data = fetch('https://code.jquery.com/' . $file) . PHP_EOL;
             if (base64_encode(hash('sha256', $data, true)) != $m[2][$k]) {
                 die('Hash mismatch' . PHP_EOL);
             }
@@ -160,7 +161,7 @@ function fetch_typo3($fp)
                     continue;
                 }
                 $file = 'type3-' . $release->version . '.tar.gz';
-                fetch_archive($file, $release->url->tar, $release->checksums->tar->sha1, 'sha1');
+                fetch_archive($file, 'https://get.typo3.org' . $release->url->tar, $release->checksums->tar->sha1, 'sha1');
                 hash_archive($fp, $file);
             }
         }
@@ -184,38 +185,40 @@ function fetch_pagekit($fp)
 }
 
 // Ignored releases are: alpha, beta, rc, dev
-function fetch_drupal($fp, $versions)
+function fetch_drupal($fp)
 {
-    foreach ($versions as $version => $id) {
+    echo 'Fetching Drupal ' . PHP_EOL;
-        echo 'Fetching Drupal ' . $version . PHP_EOL;
 
-        $page = 0;
+    $page = 0;
-        $pages = false;
+    $pages = false;
-        do {
+    do {
-            $data = fetch('https://www.drupal.org/project/drupal/releases?api_version%5B%5D=' . $id . '&page=' .$page);
+        $data = fetch('https://www.drupal.org/project/drupal/releases?page=' . $page);
 
-            // pagination init
+        // pagination init
-            if ($pages === false && preg_match('/&amp;page=(\d+)">last »<\/a>/', $data, $m)) {
+        if ($pages === false && preg_match('/\?page=(\d+)">last »<\/a>/', $data, $m)) {
-                $pages = $m[1];
+            $pages = $m[1];
+        }
+
+        preg_match_all(
+            '/<a href="(\/project\/drupal\/releases\/(\d\.\d\.\d))">drupal/i',
+            $data,
+            $m
+        );
+        foreach ($m[1] as $k => $ver_uri) {
+            $ver_data = fetch('https://www.drupal.org' . $ver_uri);
+            if (!preg_match('/<span class="field-content hash">([a-z0-9]+)<\/span>/i', $ver_data, $ver_m)) {
+                die('Missing hash info: ' . $m[2][$k]);
             }
+            $file = 'drupal-' . $m[2][$k] . '.tar.gz';
+            fetch_archive($file, 'https://ftp.drupal.org/files/projects/' . $file, $ver_m[1], 'md5');
+            hash_archive($fp, $file);
+        }
 
-            preg_match_all(
+        if ($pages === false) {
-                '/data-th="Download">(.*?)<a href="(https:\/\/ftp\.drupal\.org\/files\/projects\/(drupal\-([0-9.]+)\.tar\.gz)).*?md5 hash">\s*([a-z0-9]{32})\s*<\/td>/is',
+            break;
-                $data,
+        }
-                $m
+        $page++;
-            );
+    } while ($page <= $pages);
-            foreach ($m[3] as $k => $file) {
-                fetch_archive($file, $m[2][$k], $m[5][$k], 'md5');
-                hash_archive($fp, $file);
-            }
-
-            if ($pages === false) {
-                break;
-            }
-
-            $page++;
-        }while($page <= $pages);
-    }
 }
 
 function fetch_joomla($fp, $versions)
@@ -271,24 +274,7 @@ fetch_jquery($fp);
 fetch_wordpress($fp);
 fetch_typo3($fp);
 fetch_pagekit($fp);
-fetch_drupal(
+fetch_drupal($fp);
-    $fp,
-    [
-        '9.x' => 39794,
-        '8.x' => 7234,
-        '7.x' => 103,
-        '6.x' => 87,
-        '5.x' => 78,
-        '4.7.x' => 79,
-        '4.6.x' => 80,
-        '4.5.x' => 81,
-        '4.4.x' => 82,
-        '4.3.x' => 83,
-        '4.2.x' => 84,
-        '4.1.x' => 85,
-        '4.0.x' => 86
-    ]
-);
 fetch_joomla($fp, ['3.0' => 3, '2.5' => 25, '1.5' => 15, '1.0' => 10]);
 
 fclose($fp);

whitelist.txt

@@ -275,3 +275,6 @@ fc8f1e9f0ff666af7beb3f61b055c0e8  /core/model/smarty/sysplugins/smarty_internal_
 bb127b5ce56b45e8b4b91de2e60dd9eb  /assets/components/googleanalytics/js/mgr/libs/highcharts.js
 7d7958bb0a9438a8966807f9202d0bce  /assets/components/tinymce/jscripts/tiny_mce/plugins/spellchecker/classes/PSpellShell.php
 3ee0a4d8a06cedc0a56f29e8f351ef72  /pclzip-2-8-2/pclzip.lib.php
+abfd2987afd1f66e3eed50bebbeb6750  /sucuri-scanner-1.8.24/src/base.lib.php
+78477b67cb223e4504689fef33119884  /sucuri-scanner-1.8.24/src/sitecheck.lib.php
+e48460f6ef0c911dc5ad558c57bfd52f  /sucuri-scanner-1.8.24/src/integrity.lib.php