jQuery hashes added to pre generated list

tools/bigdata/generate.php

@@ -1,41 +1,92 @@
 <?php
+$cache_dir = __DIR__ . '/cache';
 
-if (!is_readable(__DIR__ . '/cache')) {
-    if (!mkdir(__DIR__ . '/cache')) {
-        die('Unablle to create cache directory');
+function is_cached($file)
+{
+    global $cache_dir;
+
+    return is_readable($cache_dir . '/' . $file);
+}
+
+function set_cache($file, $data)
+{
+    global $cache_dir;
+
+    file_put_contents($cache_dir . '/' . $file, $data);
+}
+
+function get_cache($file)
+{
+    global $cache_dir;
+
+    return file_get_contents($cache_dir . '/' . $file);
+}
+
+function fetch_kubik_rubik($fp)
+{
+    echo 'Fetching checksums.kubik-rubik.de' . PHP_EOL;
+    $data = file_get_contents('https://checksums.kubik-rubik.de');
+
+    // excluded: contao
+    foreach (['drupal', 'joomla', 'pagekit', 'typo3', 'wordpress'] as $type) {
+        preg_match_all('/<a class="btn btn-success" href="(' . $type . '\/.*?)">\s+JSON.*?<\/a>/is', $data, $m);
+        foreach ($m[1] as $url) {
+            $file = str_replace(['/', '.'], '_', $url); // fix file name
+            if (!is_cached($file . '.json')) {
+                echo 'Downloading: ' . 'https://checksums.kubik-rubik.de/' . $url . PHP_EOL;
+                $json = file_get_contents('https://checksums.kubik-rubik.de/' . $url);
+                set_cache($file . '.json', $json);
+            } else {
+                $json = get_cache($file . '.json');
+            }
+
+            // decode json, write hash values to one file
+            echo 'Parsing json file: ' . $file . PHP_EOL;
+            foreach (json_decode($json) as $k => $hash) {
+                fputs($fp, $hash . "\n");
+            }
+        }
     }
 }
 
-echo 'Fetching checksums.kubik-rubik.de' . PHP_EOL;
-$data = file_get_contents('https://checksums.kubik-rubik.de');
+function fetch_jquery($fp)
+{
+    echo 'Fetching jQuery' . PHP_EOL;
+    $data = file_get_contents('https://code.jquery.com/jquery/');
+
+    preg_match_all('/<a class=\'open\-sri\-modal\' href=\'\/(jquery-.*?\.js)/', $data, $m);
+    foreach ($m[1] as $file) {
+        if (!is_cached($file)) {
+            echo 'Downloading: ' . 'https://code.jquery.com/' . $file . PHP_EOL;
+            $data = file_get_contents('https://code.jquery.com/' . $file);
+            set_cache($file, $data);
+        } else {
+            $data = get_cache($file);
+        }
+
+        fputs($fp, md5($data) . "\n");
+    }
+}
+
+if (!is_readable($cache_dir)) {
+    if (!mkdir($cache_dir)) {
+        die('Unable to create cache directory');
+    }
+}
 
 $fp = fopen('all.txt', 'w');
 
-// excluded: contao
-foreach (['drupal', 'joomla', 'pagekit', 'typo3', 'wordpress'] as $type) {
-    preg_match_all('/<a class="btn btn-success" href="(' . $type . '\/.*?)">\s+JSON.*?<\/a>/is', $data, $m);
-    foreach ($m[1] as $url) {
-        $file = str_replace(['/', '.'], '_', $url); // fix file name
-        if (!is_readable(__DIR__ . '/cache/' . $file . '.json')) {
-            echo 'Downloading: ' . 'https://checksums.kubik-rubik.de/' . $url . PHP_EOL;
-            $json = file_get_contents('https://checksums.kubik-rubik.de/' . $url);
-            file_put_contents(__DIR__ . '/cache/' . $file . '.json', $json);
-        } else {
-            $json = file_get_contents(__DIR__ . '/cache/' . $file . '.json');
-        }
-
-        // decode json, write hash values to one file
-        echo 'Parsing json file: ' . $file . PHP_EOL;
-        foreach (json_decode($json) as $k => $hash) {
-            fputs($fp, $hash . "\n");
-        }
-    }
-}
+fetch_kubik_rubik($fp);
+fetch_jquery($fp);
 
 fclose($fp);
 
-echo 'Create unique database' . PHP_EOL;
+echo 'Creating unique database' . PHP_EOL;
 exec('sort -u -o unique.txt all.txt');
 
 echo 'Compressing all.txt' . PHP_EOL;
-exec('gzip < unique.txt > compressed.dat'); // gzencode
+exec('gzip < unique.txt > compressed.dat'); // gzencode
+
+$hash = hash_file('sha256', 'compressed.dat');
+file_put_contents('compressed.sha256', $hash);
+echo 'SHA256 is ' . $hash . PHP_EOL;

tools/jquery.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-
-import urllib2
-import re
-import hashlib
-
-def fetch(url):
-        response = urllib2.urlopen(url)
-        return response.read()
-
-def main():
-        html = fetch('https://code.jquery.com/jquery/')
-        regex = re.compile(r"<a class='open\-sri\-modal' href='(/jquery-.*?\.js)'")
-        m1 = regex.search(html)
-        for m in regex.findall(html):
-        	js = fetch("https://code.jquery.com" + m)
-        	hash = hashlib.md5()
-        	hash.update(js)
-        	print hash.hexdigest() + " " + m
-
-if __name__ == '__main__':
-        main()