diff --git a/.gitignore b/.gitignore index 50f8ba2..d6cd563 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,16 @@ +# .DS_Store +.DS_Store + # disk images *.dmg *.sparseimage +# .pyc and .pyo files +*.pyc +*.pyo + # our content directory content/ +# the outputted list file +softwareupdate.plist diff --git a/README.md b/README.md index 14dc272..5c80bff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,48 @@ -macadmin-scripts +### macadmin-scripts Some scripts that might be of use to macOS admins. Might be related to Munki; -might not. \ No newline at end of file +might not. + +#### createbootvolfromautonbi.py + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` + +#### installinstallmacos.py + +This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. + +It does this by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. + +Since it is using Apple's installer, any install check or volume check scripts are run. This means that you can only use this tool to create a diskimage containing the versions of macOS that will run on the exact machine you are running the script on. + +For example, to create a diskimage containing the version 10.13.6 that runs on 2018 MacBook Pros, you must run this script on a 2018 MacBook Pro, and choose the proper version. + +Typically "forked" OS build numbers are 4 digits, so when this document was last updated, build 17G2208 was the correct build for 2018 MacBook Pros; 17G65 was the correct build for all other Macs that support High Sierra. + +If you attempt to install an incompatible version of macOS, you'll see an error similar to the following: + +``` +Making empty sparseimage... +installer: Error - ERROR_B14B14D9B7 +Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/awldiototubemmsbocipx0ic9lj2kcu0pt/091-95774.English.dist', '-target', '/private/tmp/dmg.Hf0PHy']' returned non-zero exit status 1 +Product installation failed. +``` + +Use a compatible Mac or select a different build compatible with your current hardware and try again. + +Run `./installinstallmacos.py --help` to see the available options. + +#### make_firmwareupdater_pkg.sh + +This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. + +Later High Sierra installer changes have broken this script; since installing High Sierra via imaging is not recommended or supported by Apple and several other alternatives are now available, I don't plan on attempting to fix or upgrade this tool. diff --git a/installinstallmacos.py b/installinstallmacos.py index c0500de..ec76d92 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -25,11 +25,15 @@ empty disk image''' import argparse +import gzip import os import plistlib import subprocess +import re import sys +import time import urlparse +import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError @@ -37,7 +41,57 @@ from xml.parsers.expat import ExpatError DEFAULT_SUCATALOG = ( 'https://swscan.apple.com/content/catalogs/others/' 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' - '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog') + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') + +SEED_CATALOGS_PLIST = ( + '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' + 'Resources/SeedCatalogs.plist' +) + + +def get_board_id(): + '''Gets the local system board ID''' + ioreg_cmd = ['ioreg', '-p', 'IODeviceTree', '-r', '-n', '/', '-d', '1'] + try: + ioreg_output = subprocess.check_output(ioreg_cmd).splitlines() + for line in ioreg_output: + if 'board-id' in line: + board_id = line.split(" ")[-1] + board_id = board_id[board_id.find('<"')+2:board_id.find('">')] + return board_id + except subprocess.CalledProcessError, err: + raise ReplicationError(err) + + +def get_hw_model(): + '''Gets the local system ModelIdentifier''' + sysctl_cmd = ['/usr/sbin/sysctl', 'hw.model'] + try: + sysctl_output = subprocess.check_output(sysctl_cmd) + hw_model = sysctl_output.split(" ")[-1].split("\n")[0] + except subprocess.CalledProcessError, err: + raise ReplicationError(err) + return hw_model + + +def get_seeding_program(sucatalog_url): + '''Returns a seeding program name based on the sucatalog_url''' + try: + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + for key, value in seed_catalogs.items(): + if sucatalog_url == value: + return key + except (OSError, ExpatError, AttributeError, KeyError): + return None + + +def get_seed_catalog(): + '''Returns the developer seed sucatalog''' + try: + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) + except (OSError, ExpatError, AttributeError, KeyError): + return DEFAULT_SUCATALOG def make_sparse_image(volume_name, output_path): @@ -153,7 +207,7 @@ def replicate_url(full_url, root_dir='/tmp', if not ignore_cache and os.path.exists(local_file_path): curl_cmd.extend(['-z', local_file_path]) curl_cmd.append(full_url) - print "Downloading %s..." % full_url + # print "Downloading %s..." % full_url try: subprocess.check_call(curl_cmd) except subprocess.CalledProcessError, err: @@ -241,6 +295,30 @@ def parse_dist(filename): return dist_info +def get_board_ids(filename): + '''Parses a softwareupdate dist file, returning a list of supported + Board IDs''' + supported_board_ids = "" + with open(filename) as search: + for line in search: + line = line.rstrip() # remove '\n' at end of line + if 'boardIds' in line: + supported_board_ids = line.split(" ")[-1][:-1] + return supported_board_ids + + +def get_unsupported_models(filename): + '''Parses a softwareupdate dist file, returning a list of non-supported + ModelIdentifiers''' + unsupported_models = "" + with open(filename) as search: + for line in search: + line = line.rstrip() # remove '\n' at end of line + if 'nonSupportedModels' in line: + unsupported_models = line.split(" ")[-1][:-1] + return unsupported_models + + def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): '''Downloads and returns a parsed softwareupdate catalog''' try: @@ -249,13 +327,24 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): except ReplicationError, err: print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) exit(-1) - try: - catalog = plistlib.readPlist(localcatalogpath) - return catalog - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) - exit(-1) + if os.path.splitext(localcatalogpath)[1] == '.gz': + with gzip.open(localcatalogpath) as f: + content = f.read() + try: + catalog = plistlib.readPlistFromString(content) + return catalog + except ExpatError, err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) + else: + try: + catalog = plistlib.readPlist(localcatalogpath) + return catalog + except (OSError, IOError, ExpatError), err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) def find_mac_os_installers(catalog): @@ -285,7 +374,7 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): filename = get_server_metadata(catalog, product_key, workdir) product_info[product_key] = parse_server_metadata(filename) product = catalog['Products'][product_key] - product_info[product_key]['PostDate'] = str(product['PostDate']) + product_info[product_key]['PostDate'] = product['PostDate'] distributions = product['Distributions'] dist_url = distributions.get('English') or distributions.get('en') try: @@ -295,6 +384,10 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): print >> sys.stderr, 'Could not replicate %s: %s' % (dist_url, err) dist_info = parse_dist(dist_path) product_info[product_key]['DistributionPath'] = dist_path + unsupported_models = get_unsupported_models(dist_path) + product_info[product_key]['UnsupportedModels'] = unsupported_models + board_ids = get_board_ids(dist_path) + product_info[product_key]['BoardIDs'] = board_ids product_info[product_key].update(dist_info) return product_info @@ -326,16 +419,30 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): % (package['MetadataURL'], err)) exit(-1) + +def find_installer_app(mountpoint): + '''Returns the path to the Install macOS app on the mountpoint''' + applications_dir = os.path.join(mountpoint, 'Applications') + for item in os.listdir(applications_dir): + if item.endswith('.app'): + return os.path.join(applications_dir, item) + return None + + def main(): '''Do the main thing here''' + print + print "installinstallmacos.py - get macOS installers from Apple's software catalog" + print + if os.getuid() != 0: sys.exit('This command requires root (to install packages), so please ' 'run again with sudo or as root.') parser = argparse.ArgumentParser() parser.add_argument('--catalogurl', metavar='sucatalog_url', - default=DEFAULT_SUCATALOG, + default=get_seed_catalog(), help='Software Update catalog URL.') parser.add_argument('--workdir', metavar='path_to_working_dir', default='.', @@ -344,7 +451,13 @@ def main(): 'directory.') parser.add_argument('--compress', action='store_true', help='Output a read-only compressed disk image with ' - 'the Install macOS app at the root.') + 'the Install macOS app at the root. This is now the ' + 'default. Use --raw to get a read-write sparse image ' + 'with the app in the Applications directory.') + parser.add_argument('--raw', action='store_true', + help='Output a read-write sparse image ' + 'with the app in the Applications directory. Requires ' + 'less available disk space and is faster.') parser.add_argument('--ignore-cache', action='store_true', help='Ignore any previously cached files.') parser.add_argument('--build', metavar='build_version', @@ -354,8 +467,19 @@ def main(): parser.add_argument('--list', action='store_true', help='Output the available updates to a plist ' 'and quit.') + parser.add_argument('--validate', action='store_true', + help='Validate builds for board ID and hardware model ' + 'and only show appropriate builds.') args = parser.parse_args() + # show this Mac's hardware model + hw_model = get_hw_model() + print "This Mac's ModelIdentifier: %s" % hw_model + # show this Mac's board-id + board_id = get_board_id() + print "This Mac's Board ID: %s" % board_id + print + # download sucatalog and look for products that are for macOS installers catalog = download_and_parse_sucatalog( args.catalogurl, args.workdir, ignore_cache=args.ignore_cache) @@ -372,14 +496,22 @@ def main(): pl['result'] = [] # display a menu of choices (some seed catalogs have multiple installers) - print '%2s %12s %10s %8s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Title') + print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', + 'Build', 'Post Date', 'Title') for index, product_id in enumerate(product_info): - print '%2s %12s %10s %8s %s' % (index+1, - product_id, - product_info[product_id]['version'], - product_info[product_id]['BUILD'], - product_info[product_id]['title']) + if args.validate: + if board_id not in product_info[product_id]['BoardIDs']: + continue + if hw_model in product_info[product_id]['UnsupportedModels']: + continue + print '%2s %12s %10s %8s %11s %s' % ( + index + 1, + product_id, + product_info[product_id]['version'], + product_info[product_id]['BUILD'], + product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), + product_info[product_id]['title'] + ) pl_index = {'index': index+1, 'product_id': product_id, @@ -448,22 +580,25 @@ def main(): print >> sys.stderr, 'Product installation failed.' unmountdmg(mountpoint) exit(-1) + # add the seeding program xattr to the app if applicable + seeding_program = get_seeding_program(args.catalogurl) + if seeding_program: + installer_app = find_installer_app(mountpoint) + if installer_app: + xattr.setxattr(installer_app, 'SeedProgram', seeding_program) print 'Product downloaded and installed to %s' % sparse_diskimage_path - if not args.compress: + if args.raw: unmountdmg(mountpoint) else: - # if --compress option given, create a r/o compressed diskimage + # if --raw option not given, create a r/o compressed diskimage # containing the Install macOS app compressed_diskimagepath = os.path.join( args.workdir, volname + '.dmg') if os.path.exists(compressed_diskimagepath): os.unlink(compressed_diskimagepath) - applications_dir = os.path.join(mountpoint, 'Applications') - for item in os.listdir(applications_dir): - if item.endswith('.app'): - app_path = os.path.join(applications_dir, item) - make_compressed_dmg(app_path, compressed_diskimagepath) - break + app_path = find_installer_app(mountpoint) + if app_path: + make_compressed_dmg(app_path, compressed_diskimagepath) # unmount sparseimage unmountdmg(mountpoint) # delete sparseimage since we don't need it any longer