\s+JSON.*?<\/a>/is', $data, $m); foreach ($m[1] as $url) { $file = str_replace(['/', '.'], '_', $url); // fix file name if (!is_cached($file . '.json')) { echo 'Downloading: ' . 'https://checksums.kubik-rubik.de/' . $url . PHP_EOL; $json = file_get_contents('https://checksums.kubik-rubik.de/' . $url); set_cache($file . '.json', $json); } else { $json = get_cache($file . '.json'); } // decode json, write hash values to one file echo 'Parsing json file: ' . $file . PHP_EOL; foreach (json_decode($json) as $k => $hash) { fputs($fp, $hash . "\n"); } } } } function fetch_jquery($fp) { echo 'Fetching jQuery' . PHP_EOL; $data = file_get_contents('https://code.jquery.com/jquery/'); preg_match_all( '/ $file) { if (!is_cached($file)) { echo 'Downloading: ' . 'https://code.jquery.com/' . $file . PHP_EOL; $data = file_get_contents('https://code.jquery.com/' . $file); if (base64_encode(hash('sha256', $data, true)) != $m[2][$k]) { die('Hash mismatch' . PHP_EOL); } set_cache($file, $data, 'sha256', bin2hex(base64_decode($m[2][$k]))); } else { $data = get_cache($file); } fputs($fp, md5($data) . "\n"); } } if (!is_readable($cache_dir)) { if (!mkdir($cache_dir)) { die('Unable to create cache directory'); } } $fp = fopen('all.txt', 'w'); // fetch_kubik_rubik($fp); fetch_jquery($fp); fclose($fp); echo 'Creating unique database' . PHP_EOL; exec('sort -u -o unique.txt all.txt'); echo 'Compressing all.txt' . PHP_EOL; exec('gzip < unique.txt > compressed.dat'); // gzencode $hash = hash_file('sha256', 'compressed.dat'); file_put_contents('compressed.sha256', $hash); echo 'SHA256 is ' . $hash . PHP_EOL;