README update with new -r flag

Feature flagHideErr #66

README.md

@@ -26,6 +26,7 @@ Usage: php scan.php -d <directory>
     -x                   --extra-check        Adds GoogleBot and htaccess to Scan List
     -l                   --follow-symlink     Follow symlinked directories
     -k                   --hide-ok            Hide results with 'OK' status
+    -r                   --hide-err           Hide results with 'ER' status
     -w                   --hide-whitelist     Hide results with 'WL' status
     -n                   --no-color           Disable color mode
     -s                   --no-stop            Continue scanning file after first hit
@@ -35,6 +36,7 @@ Usage: php scan.php -d <directory>
     -o                   --output-format      Custom defined output format
     -j                   --wordpress-version  Version of wordpress to get md5 signatures
                          --combined-whitelist Combined whitelist
+                         --disable-stats      Disable statistics output
 ```
 
 Ignore argument could be used multiple times and accept glob style matching ex.: "`cache*`", "`??-cache.php`" or "`/cache`" etc.
@@ -112,6 +114,22 @@ It is guaranteed that IF 'base64_decode' was present in the plain text code, the
 The presence of 'YmFzZTY0X2RlY29kZ' in a block of code may be because 'ase64_decod' was in the original code.  
 ote the missing edge characters which is due to bit misalignment and character bleed.
 
+Using as library
+----------------
+
+The scan.php perform a check, that it's called by commandline or not, so to use as library use different directory than scan.php it self.
+ 
+```php
+<?php
+
+require_once '../scan.php';
+
+$scan = new MalwareScanner();
+$scan->setFlagHideWhitelist(true);
+$scan->setFlagHideOk(true);
+$scan->run('../samples/test');
+```
+
 Resources
 ---------
 

definitions/patterns_iraw.txt

@@ -1,7 +1,7 @@
-#This file contains raw strings that will be matched case-insensitive.
+# This file contains raw strings that will be matched case-insensitive.
-#Comments and whitespace are possible, but comments must have '#' at the first character of the line.
+# Comments and whitespace are possible, but comments must have '#' at the first character of the line.
 
-#List of security service providers that phishers often block.
+# List of security service providers that phishers often block.
 abovenet
 avira
 bitdefender
@@ -17,3 +17,6 @@ phishtank
 sophos
 surfright
 symantec
+
+# SEO poison, pharmacy redirect
+dealonline.su

definitions/patterns_raw.txt

@@ -201,6 +201,7 @@ eval(base64_decode(
 $data = base64_decode("
 edoced_46esab
 base=base64_encode
+'b'.'ase6'.'4_e'.'ncode'
 cr"."eat"."e_fun"."cti"."on
 gz'.'inf'.'late
 # fopo.com.ar - free online php obfuscator. It conveniently leaves comments in the code.
@@ -260,7 +261,18 @@ itsoknoproblembro
 tmhapbzcerff
 IndoXploit
 FaisaL Ahmed aka rEd X
+smisbot
+smotherbot
+Indonesian Hacker Rulez
 
+# WP-VCD Malware https://www.getastra.com/blog/911/how-to-fix-wp-vcd-backdoor-hack-in-wordpress-functions-php/
+wp-vcd
+class.theme-modules.php
+wp-tmp.php
+tmpcontentx
+function wp_temp_setupx
+derna.top/code.php
+stripos($tmpcontent, $wp_auth_key)
 
 #Miscellaneous
 uname -a
@@ -362,4 +374,8 @@ ZeroByte
 100, 111, 99, 117, 109, 101, 110, 116, 46, 99, 114, 101, 97, 116, 101, 69, 108, 101, 109, 101, 110, 116, 40, 39, 115, 99, 114, 105, 112, 116, 39, 41, 59
 
 # JS escaped: String.fromCharCode(
-83, 116, 114, 105, 110, 103, 46, 102, 114, 111, 109, 67, 104, 97, 114, 67, 111, 100, 101, 40
+83, 116, 114, 105, 110, 103, 46, 102, 114, 111, 109, 67, 104, 97, 114, 67, 111, 100, 101, 40
+
+# SEO poisoning control site call
+"http://$xxx
+?useragent=$botbotbot

definitions/patterns_re.txt

@@ -4,10 +4,13 @@ eval\/\*[a-z0-9]+\*\/
 #
 eval\([a-z0-9]{4,}\(\$[a-z0-9]{4,}, \$[0-9a-z]{4,}\)\);
 
-#
+# chr(101).chr(118).chr(97)
 (chr\(\d+\^\d+\)\.){4,}
 
-#
+# $_uU(101).$_uU(118).$_uU(97)
+(\$\_[a-z0-9]{2,}\(\d+\)\.){4,}
+
+# $uUx[101].$uUx[118].$uUx[97]
 (\$[a-z0-9]{3,}\[\d+\]\.){4,}
 
 #
@@ -37,6 +40,9 @@ Googlebot['"]{0,1}\s*\)\){echo\s+file_get_contents
 #execute base64 code
 eVaL\(\s*trim\(\s*baSe64_deCoDe\(
 
+# execute escaped code
+exec\("(\\[0-9a-fx]{2,3}){3,}
+
 #
 if\s*\(\s*mail\s*\(\s*\$mails\[\$i\]\s*,\s*\$tema\s*,\s*base64_encode\s*\(\s*\$text
 
@@ -107,4 +113,7 @@ function\s+_[0-9]{8,}\(
 @include ".*?(\\x[0-9a-f]{2,}.*?){2,}.*?";
 
 # create_function is dangerous as like eval() see http://php.net/manual/en/function.create-function.php
-create_function\s*\(\s*['"]{2}
+create_function\s*\(\s*['"]{2}
+
+# control concated from cookie at the call
+(\$[a-z]{2,}=urldecode\(\$_COOKIE\['[a-z]{2,}'\]\);){3,}

scan.php

@@ -2,13 +2,13 @@
 
 /*
  * Copyright (c) 2016 Gabor Gyorvari
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *  http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,6 +31,7 @@ class MalwareScanner
     private $flagChecksum = false;
     private $flagComments = false;
     private $flagHideOk = false;
+    private $flagHideErr = false;
     private $flagHideWhitelist = false;
     private $flagNoStop = false;
     private $flagPattern = false;
@@ -40,6 +41,7 @@ class MalwareScanner
     private $flagLineNumber = false;
     private $flagScanEverything = false;
     private $flagCombinedWhitelist = false;
+    private $flagDisableStats = false;
     private $outputFormat = '';
     private $whitelist = array();
     private $ignore = array();
@@ -68,16 +70,31 @@ class MalwareScanner
         if ($cli === true) {
             //Read Run Options
             $this->parseArgs();
-            $this->dir = realpath($this->dir);
+
+            $dirs = array();
+            if (is_array($this->dir)) {
+                // allow multiple directory aka. array
+                foreach ($this->dir as $path) {
+                    $dirs[] = realpath($path);
+                }
+            } elseif ($bpos = strpos($this->dir, '{')) {
+                // Check path has a "brace", expand it to subdirectories
+                foreach (glob($this->dir, GLOB_BRACE) as $path) {
+                    $dirs[] = realpath($path);
+                }
+            } else {
+                // only one directory specified
+                $dirs = array (realpath($this->dir));
+            }
 
             //Make sure a directory was specified.
-            if ($this->dir === '') {
+            if (empty($dirs)) {
                 $this->error('No directory specified or directory doesn\'t exist');
                 exit(-1);
             }
 
             //Initiate Scan
-            if (!$this->run($this->dir)) {
+            if (!$this->run($dirs)) {
                 exit(-1);
             }
         }
@@ -103,7 +120,7 @@ class MalwareScanner
     }
 
     //Handles pattern loading and saving to the class object
-    private function initializePatterns()
+    public function initializePatterns()
     {
         $dir = dirname(__FILE__);
         //Loads either the primary scanning patterns or the base64 patterns depending on -b/--base64 flag
@@ -175,7 +192,7 @@ class MalwareScanner
     }
 
     //Loads the whitelist file
-    private function loadWhitelist()
+    public function loadWhitelist()
     {
         if (!is_file(__DIR__ . '/whitelist.txt')) {
             return;
@@ -187,7 +204,7 @@ class MalwareScanner
         }
     }
 
-    private function addWordpressChecksums($wp_version)
+    public function addWordpressChecksums($wp_version)
     {
         $apiurl = 'https://api.wordpress.org/core/checksums/1.0/?version=' . $wp_version;
         $json = json_decode(file_get_contents($apiurl));
@@ -230,7 +247,8 @@ class MalwareScanner
                 'output-format:',
                 'wordpress-version:',
                 'scan-everything',
-                'combined-whitelist'
+                'combined-whitelist',
+                'disable-stats'
             )
         );
 
@@ -281,6 +299,9 @@ class MalwareScanner
         if (isset($options['hide-ok']) || isset($options['k'])) {
             $this->setFlagHideOk(true);
         }
+        if (isset($options['hide-err']) || isset($options['r'])) {
+            $this->setFlagHideErr(true);
+        }
         if (isset($options['hide-whitelist']) || isset($options['w'])) {
             $this->setFlagHideWhitelist(true);
         }
@@ -313,6 +334,9 @@ class MalwareScanner
         if (isset($options['combined-whitelist'])) {
             $this->setFlagCombinedWhitelist(true);
         }
+        if (isset($options['disable-stats'])) {
+            $this->setFlagDisableStats(true);
+        }
     }
 
     public function setExtensions(array $a)
@@ -376,6 +400,11 @@ class MalwareScanner
         $this->flagHideOk = $b;
     }
 
+    public function setFlagHideErr($b)
+    {
+        $this->flagHideErr = $b;
+    }
+
     public function setFlagHideWhitelist($b)
     {
         $this->flagHideWhitelist = $b;
@@ -401,6 +430,11 @@ class MalwareScanner
         $this->flagCombinedWhitelist = $b;
     }
 
+    public function setFlagDisableStats($b)
+    {
+        $this->flagDisableStats = $b;
+    }
+
     // @see http://stackoverflow.com/a/13914119
     private function pathMatches($path, $pattern, $ignoreCase = false)
     {
@@ -465,6 +499,9 @@ class MalwareScanner
             $state = 'WL';
             $state_color = $this->ANSI_YELLOW;
         } else {
+            if ($this->flagHideErr) {
+                return;
+            }
             $state = 'ER';
             $state_color = $this->ANSI_RED;
         }
@@ -501,7 +538,7 @@ class MalwareScanner
         }
 
         if ($this->outputFormat) {
-            $map = [
+            $map = array(
                 '%S' => $state,
                 '%T' => $ctime,
                 '%M' => $hash,
@@ -509,9 +546,9 @@ class MalwareScanner
                 '%P' => $pattern,
                 '%C' => $comment,
                 '%L' => $lineNumber,
-            ];
+            );
         } else {
-            $map = [
+            $map = array(
                 '%S' => $state_color . '# ' . $state . $this->ANSI_OFF,
                 '%T' => $this->ANSI_BLUE . $ctime . $this->ANSI_OFF,
                 '%M' => $this->ANSI_BLUE . $hash . $this->ANSI_OFF,
@@ -519,7 +556,7 @@ class MalwareScanner
                 '%P' => $state_color . '#' . $pattern . $this->ANSI_OFF,
                 '%C' => $this->ANSI_BLUE . $comment . $this->ANSI_OFF,
                 '%L' => $lineNumber,
-            ];
+            );
         }
 
         if ($this->outputFormat) {
@@ -582,18 +619,11 @@ class MalwareScanner
      * - Fetch and load combined whitelist
      * - Calls the process and report functions.
      *
-     * @param $dir
+     * @param string|array $dir A directory path or a list of paths in array
      * @return bool
      */
     public function run($dir)
     {
-        // Make sure the input is a valid directory path.
-        $dir = rtrim($dir, '/');
-        if (!is_dir($dir)) {
-            $this->error('Specified path is not a directory: ' . $dir);
-            return false;
-        }
-
         $this->initializePatterns();
 
         $this->loadWhitelist();
@@ -603,8 +633,24 @@ class MalwareScanner
         }
 
         $start = time();
-        $this->process($dir . '/');
+
-        $this->report($start, $dir . '/');
+        if (!is_array($dir)) {
+            $dir = array ($dir);
+        }
+
+        foreach ($dir as $path) {
+            // Make sure the input is a valid directory path.
+            $path = rtrim($path, '/');
+            if (!is_dir($path)) {
+                $this->error('Specified path is not a directory: ' . $path);
+                return false;
+            }
+            $this->process($path . '/');
+        }
+
+        if (!$this->flagDisableStats) {
+            $this->report($start, implode(', ', $dir));
+        }
         return true;
     }
 
@@ -757,7 +803,7 @@ class MalwareScanner
         }
 
         $content = gzdecode(file_get_contents($file));
-        $this->combined_whitelist = [];
+        $this->combined_whitelist = array();
         $this->combined_whitelist_count = 0;
         foreach (explode("\n", $content) as $line) { // faster than strtok, but needs more memory
             if ($line) {
@@ -786,6 +832,7 @@ class MalwareScanner
         echo '    -x                   --extra-check        Adds GoogleBot and htaccess to Scan List' . PHP_EOL;
         echo '    -l                   --follow-symlink     Follow symlinked directories' . PHP_EOL;
         echo '    -k                   --hide-ok            Hide results with \'OK\' status' . PHP_EOL;
+        echo '    -r                   --hide-err           Hide results with \'ER\' status' . PHP_EOL;
         echo '    -w                   --hide-whitelist     Hide results with \'WL\' status' . PHP_EOL;
         echo '    -n                   --no-color           Disable color mode' . PHP_EOL;
         echo '    -s                   --no-stop            Continue scanning file after first hit' . PHP_EOL;
@@ -795,6 +842,7 @@ class MalwareScanner
         echo '    -o                   --output-format      Custom defined output format' . PHP_EOL;
         echo '    -j                   --wordpress-version  Version of wordpress to get md5 signatures' . PHP_EOL;
         echo '                         --combined-whitelist Combined whitelist' . PHP_EOL;
+        echo '                         --disable-stats      Disable statistics output' . PHP_EOL;
 
     }
 

tools/bigdata/generate.php

@@ -15,7 +15,8 @@ function fetch($url, $file = false)
 
     $headers = array(
         // drupal suxx
-        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.3 Safari/605.1.15',
+        'Cookie: _px2=eyJ1IjoiZDZhNGM3MjAtYjZmNC0xMWVhLWI2MzMtNzk5YzRmZjM4ZmJkIiwidiI6IjQ0ZTFiMDQwLTRkZGUtMTFlOC1iMWRjLWYxNWU4OTg1NTZjNyIsInQiOjE1OTMwOTc2Mjg2NzAsImgiOiIzNzk5N2RkYTU3ZTI1NGY0ZDM5MmRiMWExNWZhZjhjNTZkMmM5NTZkZDJiZWVkZGVlZDc1MThiNTE5MTFjYzgwIn0=; _ga=GA1.2.2042202377.1525247839; _gat=1; _gid=GA1.2.1034461360.1593095881; has_js=1; _pxff_fp=1; _pxff_rf=1; pxvid=44e1b040-4dde-11e8-b1dc-f15e898556c7',
+        'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
     );
     curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
 
@@ -89,7 +90,7 @@ function fetch_jquery($fp)
     foreach ($m[1] as $k => $file) {
         if (!is_cached($file)) {
             echo 'Downloading: ' . 'https://code.jquery.com/' . $file . PHP_EOL;
-            $data = fetch('https://code.jquery.com/' . $file);
+            $data = fetch('https://code.jquery.com/' . $file) . PHP_EOL;
             if (base64_encode(hash('sha256', $data, true)) != $m[2][$k]) {
                 die('Hash mismatch' . PHP_EOL);
             }
@@ -160,7 +161,7 @@ function fetch_typo3($fp)
                     continue;
                 }
                 $file = 'type3-' . $release->version . '.tar.gz';
-                fetch_archive($file, $release->url->tar, $release->checksums->tar->sha1, 'sha1');
+                fetch_archive($file, 'https://get.typo3.org' . $release->url->tar, $release->checksums->tar->sha1, 'sha1');
                 hash_archive($fp, $file);
             }
         }
@@ -184,38 +185,40 @@ function fetch_pagekit($fp)
 }
 
 // Ignored releases are: alpha, beta, rc, dev
-function fetch_drupal($fp, $versions)
+function fetch_drupal($fp)
 {
-    foreach ($versions as $version => $id) {
+    echo 'Fetching Drupal ' . PHP_EOL;
-        echo 'Fetching Drupal ' . $version . PHP_EOL;
 
-        $page = 0;
+    $page = 0;
-        $pages = false;
+    $pages = false;
-        do {
+    do {
-            $data = fetch('https://www.drupal.org/project/drupal/releases?api_version%5B%5D=' . $id . '&page=' .$page);
+        $data = fetch('https://www.drupal.org/project/drupal/releases?page=' . $page);
 
-            // pagination init
+        // pagination init
-            if ($pages === false && preg_match('/&amp;page=(\d+)">last »<\/a>/', $data, $m)) {
+        if ($pages === false && preg_match('/\?page=(\d+)">last »<\/a>/', $data, $m)) {
-                $pages = $m[1];
+            $pages = $m[1];
+        }
+
+        preg_match_all(
+            '/<a href="(\/project\/drupal\/releases\/(\d\.\d\.\d))">drupal/i',
+            $data,
+            $m
+        );
+        foreach ($m[1] as $k => $ver_uri) {
+            $ver_data = fetch('https://www.drupal.org' . $ver_uri);
+            if (!preg_match('/<span class="field-content hash">([a-z0-9]+)<\/span>/i', $ver_data, $ver_m)) {
+                die('Missing hash info: ' . $m[2][$k]);
             }
+            $file = 'drupal-' . $m[2][$k] . '.tar.gz';
+            fetch_archive($file, 'https://ftp.drupal.org/files/projects/' . $file, $ver_m[1], 'md5');
+            hash_archive($fp, $file);
+        }
 
-            preg_match_all(
+        if ($pages === false) {
-                '/data-th="Download">(.*?)<a href="(https:\/\/ftp\.drupal\.org\/files\/projects\/(drupal\-([0-9.]+)\.tar\.gz)).*?md5 hash">\s*([a-z0-9]{32})\s*<\/td>/is',
+            break;
-                $data,
+        }
-                $m
+        $page++;
-            );
+    } while ($page <= $pages);
-            foreach ($m[3] as $k => $file) {
-                fetch_archive($file, $m[2][$k], $m[5][$k], 'md5');
-                hash_archive($fp, $file);
-            }
-
-            if ($pages === false) {
-                break;
-            }
-
-            $page++;
-        }while($page <= $pages);
-    }
 }
 
 function fetch_joomla($fp, $versions)
@@ -271,24 +274,7 @@ fetch_jquery($fp);
 fetch_wordpress($fp);
 fetch_typo3($fp);
 fetch_pagekit($fp);
-fetch_drupal(
+fetch_drupal($fp);
-    $fp,
-    [
-        '9.x' => 39794,
-        '8.x' => 7234,
-        '7.x' => 103,
-        '6.x' => 87,
-        '5.x' => 78,
-        '4.7.x' => 79,
-        '4.6.x' => 80,
-        '4.5.x' => 81,
-        '4.4.x' => 82,
-        '4.3.x' => 83,
-        '4.2.x' => 84,
-        '4.1.x' => 85,
-        '4.0.x' => 86
-    ]
-);
 fetch_joomla($fp, ['3.0' => 3, '2.5' => 25, '1.5' => 15, '1.0' => 10]);
 
 fclose($fp);

whitelist.txt

@@ -275,3 +275,6 @@ fc8f1e9f0ff666af7beb3f61b055c0e8  /core/model/smarty/sysplugins/smarty_internal_
 bb127b5ce56b45e8b4b91de2e60dd9eb  /assets/components/googleanalytics/js/mgr/libs/highcharts.js
 7d7958bb0a9438a8966807f9202d0bce  /assets/components/tinymce/jscripts/tiny_mce/plugins/spellchecker/classes/PSpellShell.php
 3ee0a4d8a06cedc0a56f29e8f351ef72  /pclzip-2-8-2/pclzip.lib.php
+abfd2987afd1f66e3eed50bebbeb6750  /sucuri-scanner-1.8.24/src/base.lib.php
+78477b67cb223e4504689fef33119884  /sucuri-scanner-1.8.24/src/sitecheck.lib.php
+e48460f6ef0c911dc5ad558c57bfd52f  /sucuri-scanner-1.8.24/src/integrity.lib.php