\s+JSON.*?<\/a>/is', $data, $m); foreach ($m[1] as $url) { $file = str_replace(['/', '.'], '_', $url); // fix file name if (!is_readable(__DIR__ . '/cache/' . $file . '.json')) { echo 'Downloading: ' . 'https://checksums.kubik-rubik.de/' . $url . PHP_EOL; $json = file_get_contents('https://checksums.kubik-rubik.de/' . $url); file_put_contents(__DIR__ . '/cache/' . $file . '.json', $json); } else { $json = file_get_contents(__DIR__ . '/cache/' . $file . '.json'); } // decode json, write hash values to one file echo 'Parsing json file: ' . $file . PHP_EOL; foreach (json_decode($json) as $k => $hash) { fputs($fp, $hash . "\n"); } } } fclose($fp); echo 'Create unique database' . PHP_EOL; exec('sort -u -o unique.txt all.txt'); echo 'Compressing all.txt' . PHP_EOL; exec('gzip < unique.txt > compressed.dat'); // gzencode