From aca77ebbf20573e4a970b914559dec55b492b597 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 7 Aug 2018 13:50:38 -0700 Subject: [PATCH 1/6] Get default sucatalog URL from Seeding.framework if possible; handle .gz sucatalogs --- installinstallmacos.py | 43 +++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 7c5f217..6017cd3 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -25,6 +25,7 @@ empty disk image''' import argparse +import gzip import os import plistlib import subprocess @@ -37,7 +38,20 @@ from xml.parsers.expat import ExpatError DEFAULT_SUCATALOG = ( 'https://swscan.apple.com/content/catalogs/others/' 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' - '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog') + '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') + + +def get_seed_catalog(): + '''Returns the developer seed sucatalog''' + seed_catalogs_plist = ( + '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' + 'Resources/SeedCatalogs.plist' + ) + try: + seed_catalogs = plistlib.readPlist(seed_catalogs_plist) + return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) + except (OSError, ExpatError, AttributeError, KeyError): + return DEFAULT_SUCATALOG def make_sparse_image(volume_name, output_path): @@ -249,13 +263,24 @@ def download_and_parse_sucatalog(sucatalog, workdir, ignore_cache=False): except ReplicationError, err: print >> sys.stderr, 'Could not replicate %s: %s' % (sucatalog, err) exit(-1) - try: - catalog = plistlib.readPlist(localcatalogpath) - return catalog - except (OSError, IOError, ExpatError), err: - print >> sys.stderr, ( - 'Error reading %s: %s' % (localcatalogpath, err)) - exit(-1) + if os.path.splitext(localcatalogpath)[1] == '.gz': + with gzip.open(localcatalogpath) as f: + content = f.read() + try: + catalog = plistlib.readPlistFromString(content) + return catalog + except ExpatError, err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) + else: + try: + catalog = plistlib.readPlist(localcatalogpath) + return catalog + except (OSError, IOError, ExpatError), err: + print >> sys.stderr, ( + 'Error reading %s: %s' % (localcatalogpath, err)) + exit(-1) def find_mac_os_installers(catalog): @@ -335,7 +360,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--catalogurl', metavar='sucatalog_url', - default=DEFAULT_SUCATALOG, + default=get_seed_catalog(), help='Software Update catalog URL.') parser.add_argument('--workdir', metavar='path_to_working_dir', default='.', From 314eca5f23599be39e5f5bad8795d595eff6eab2 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Tue, 14 Aug 2018 15:26:17 -0700 Subject: [PATCH 2/6] Add SeedProgram xattr to Install macOS app; making a compressed read-only dmg is now the default --- installinstallmacos.py | 59 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 6017cd3..863159c 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -31,6 +31,7 @@ import plistlib import subprocess import sys import urlparse +import xattr from xml.dom import minidom from xml.parsers.expat import ExpatError @@ -40,15 +41,27 @@ DEFAULT_SUCATALOG = ( 'index-10.13seed-10.13-10.12-10.11-10.10-10.9' '-mountainlion-lion-snowleopard-leopard.merged-1.sucatalog.gz') +SEED_CATALOGS_PLIST = ( + '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' + 'Resources/SeedCatalogs.plist' +) + + +def get_seeding_program(sucatalog_url): + '''Returns a seeding program name based on the sucatalog_url''' + try: + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) + for key, value in seed_catalogs.items(): + if sucatalog_url == value: + return key + except (OSError, ExpatError, AttributeError, KeyError): + return None + def get_seed_catalog(): '''Returns the developer seed sucatalog''' - seed_catalogs_plist = ( - '/System/Library/PrivateFrameworks/Seeding.framework/Versions/Current/' - 'Resources/SeedCatalogs.plist' - ) try: - seed_catalogs = plistlib.readPlist(seed_catalogs_plist) + seed_catalogs = plistlib.readPlist(SEED_CATALOGS_PLIST) return seed_catalogs.get('DeveloperSeed', DEFAULT_SUCATALOG) except (OSError, ExpatError, AttributeError, KeyError): return DEFAULT_SUCATALOG @@ -352,6 +365,15 @@ def replicate_product(catalog, product_id, workdir, ignore_cache=False): exit(-1) +def find_installer_app(mountpoint): + '''Returns the path to the Install macOS app on the mountpoint''' + applications_dir = os.path.join(mountpoint, 'Applications') + for item in os.listdir(applications_dir): + if item.endswith('.app'): + return os.path.join(applications_dir, item) + return None + + def main(): '''Do the main thing here''' if os.getuid() != 0: @@ -369,7 +391,13 @@ def main(): 'directory.') parser.add_argument('--compress', action='store_true', help='Output a read-only compressed disk image with ' - 'the Install macOS app at the root.') + 'the Install macOS app at the root. This is now the ' + 'default. Use --raw to get a read-write sparse image ' + 'with the app in the Applications directory.') + parser.add_argument('--raw', action='store_true', + help='Output a read-write sparse image ' + 'with the app in the Applications directory. Requires ' + 'less available disk space and is faster.') parser.add_argument('--ignore-cache', action='store_true', help='Ignore any previously cached files.') args = parser.parse_args() @@ -431,22 +459,25 @@ def main(): print >> sys.stderr, 'Product installation failed.' unmountdmg(mountpoint) exit(-1) + # add the seeding program xattr to the app if applicable + seeding_program = get_seeding_program(args.catalogurl) + if seeding_program: + installer_app = find_installer_app(mountpoint) + if installer_app: + xattr.setxattr(installer_app, 'SeedProgram', seeding_program) print 'Product downloaded and installed to %s' % sparse_diskimage_path - if not args.compress: + if args.raw: unmountdmg(mountpoint) else: - # if --compress option given, create a r/o compressed diskimage + # if --raw option not given, create a r/o compressed diskimage # containing the Install macOS app compressed_diskimagepath = os.path.join( args.workdir, volname + '.dmg') if os.path.exists(compressed_diskimagepath): os.unlink(compressed_diskimagepath) - applications_dir = os.path.join(mountpoint, 'Applications') - for item in os.listdir(applications_dir): - if item.endswith('.app'): - app_path = os.path.join(applications_dir, item) - make_compressed_dmg(app_path, compressed_diskimagepath) - break + app_path = find_installer_app(mountpoint) + if app_path: + make_compressed_dmg(app_path, compressed_diskimagepath) # unmount sparseimage unmountdmg(mountpoint) # delete sparseimage since we don't need it any longer From 6c10f488331c4b12c266a89b69f2e5e1fb3b8335 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Wed, 15 Aug 2018 15:44:05 -0700 Subject: [PATCH 3/6] Add more items to .gitignore; update README for more info on expected installinstallmacos.py behavior --- .gitignore | 7 +++++++ README.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 50f8ba2..b6f4ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,14 @@ +# .DS_Store +.DS_Store + # disk images *.dmg *.sparseimage +# .pyc and .pyo files +*.pyc +*.pyo + # our content directory content/ diff --git a/README.md b/README.md index 14dc272..67e7754 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,42 @@ -macadmin-scripts +### macadmin-scripts Some scripts that might be of use to macOS admins. Might be related to Munki; -might not. \ No newline at end of file +might not. + +#### installinstallmacos.py + +This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. + +It does this by downloading the packages from Apple's softwareupdate servers and then installing them into a new empty disk image. + +Since it is using Apple's installer, any install check or volume check scripts are run. This means that you can only use this tool to create a diskimage containing the versions of macOS that will run on the exact machine you are running the script on. + +For example, to create a diskimage containing the version 10.13.6 that runs on 2018 MacBook Pros, you must run this script on a 2018 MacBook Pro, and choose the proper version. + +Typically "forked" OS build numbers are 4 digits, so when this document was last updated, build 17G2208 was the correct build for 2018 MacBook Pros; 17G65 was the correct build for all other Macs that support High Sierra. + +If you attempt to install an incompatible version of macOS, you'll see an error similar to the following: + +``` +Making empty sparseimage... +installer: Error - ERROR_B14B14D9B7 +Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/awldiototubemmsbocipx0ic9lj2kcu0pt/091-95774.English.dist', '-target', '/private/tmp/dmg.Hf0PHy']' returned non-zero exit status 1 +Product installation failed. +``` + +Use a compatible Mac or a build compatible with your current hardware and try again. + +Run `./installinstallmacos.py --help` to see the available options. + +#### createbootvolfromautonbi.py + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` From d5a6247deea93fc0ef9c5ff50ab76e7b1653ecc0 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 16 Aug 2018 18:28:14 -0700 Subject: [PATCH 4/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67e7754..a5e27b9 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Command '['/usr/sbin/installer', '-pkg', './content/downloads/07/20/091-95774/aw Product installation failed. ``` -Use a compatible Mac or a build compatible with your current hardware and try again. +Use a compatible Mac or select a diffrent build compatible with your current hardware and try again. Run `./installinstallmacos.py --help` to see the available options. From f9985c90adb0aa396d98bbe39192aeca2866f190 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Fri, 17 Aug 2018 08:09:32 -0700 Subject: [PATCH 5/6] Reorganize README.md; add info for make_firmwareupdater_pkg.sh --- README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a5e27b9..1473e9d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,19 @@ Some scripts that might be of use to macOS admins. Might be related to Munki; might not. +#### createbootvolfromautonbi.py + +A tool to make bootable disk volumes from the output of autonbi. Especially +useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, +which allows Imagr to run scripts that affect SIP state, set UAKEL options, and +run the `startosinstall` component, all of which might otherwise require network +booting from a NetInstall-style nbi. + +This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. + +This command converts the output of Imagr's `make nbi` into a bootable external USB disk: +`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` + #### installinstallmacos.py This script can create disk images containing macOS Installer applications available via Apple's softwareupdate catalogs. @@ -28,15 +41,9 @@ Use a compatible Mac or select a diffrent build compatible with your current har Run `./installinstallmacos.py --help` to see the available options. -#### createbootvolfromautonbi.py +#### make_firmwareupdater_pkg.sh -A tool to make bootable disk volumes from the output of autonbi. Especially -useful to make bootable disks containing Imagr and the 'SIP-ignoring' kernel, -which allows Imagr to run scripts that affect SIP state, set UAKEL options, and -run the `startosinstall` component, all of which might otherwise require network -booting from a NetInstall-style nbi. +This script was used to extract the firmware updaters from early High Sierra installers and make a standalone installer package that could be used to upgrade Mac firmware before installing High Sierra via imaging. -This provides a way to create a bootable external disk that acts like the Netboot environment used by/needed by Imagr. +Later High Sierra installer changes have broken this script; since installing High Sierra via imaging is not recommended or supported by Apple and several other alternatives are now available, I don't plan on attempting to fix or upgrade this tool. -This command converts the output of Imagr's `make nbi` into a bootable external USB disk: -`sudo ./createbootvolfromautonbi.py --nbi ~/Desktop/10.13.6_Imagr.nbi --volume /Volumes/ExternalDisk` From 4b68682c8189ccfb96393ed893af6ed4cee32765 Mon Sep 17 00:00:00 2001 From: Greg Neagle Date: Thu, 20 Sep 2018 10:38:39 -0700 Subject: [PATCH 6/6] Add Post Date to listing output --- installinstallmacos.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/installinstallmacos.py b/installinstallmacos.py index 863159c..ffb3abb 100755 --- a/installinstallmacos.py +++ b/installinstallmacos.py @@ -30,6 +30,7 @@ import os import plistlib import subprocess import sys +import time import urlparse import xattr from xml.dom import minidom @@ -323,7 +324,7 @@ def os_installer_product_info(catalog, workdir, ignore_cache=False): filename = get_server_metadata(catalog, product_key, workdir) product_info[product_key] = parse_server_metadata(filename) product = catalog['Products'][product_key] - product_info[product_key]['PostDate'] = str(product['PostDate']) + product_info[product_key]['PostDate'] = product['PostDate'] distributions = product['Distributions'] dist_url = distributions.get('English') or distributions.get('en') try: @@ -414,14 +415,17 @@ def main(): exit(-1) # display a menu of choices (some seed catalogs have multiple installers) - print '%2s %12s %10s %8s %s' % ('#', 'ProductID', 'Version', - 'Build', 'Title') + print '%2s %12s %10s %8s %11s %s' % ('#', 'ProductID', 'Version', + 'Build', 'Post Date', 'Title') for index, product_id in enumerate(product_info): - print '%2s %12s %10s %8s %s' % (index+1, - product_id, - product_info[product_id]['version'], - product_info[product_id]['BUILD'], - product_info[product_id]['title']) + print '%2s %12s %10s %8s %11s %s' % ( + index + 1, + product_id, + product_info[product_id]['version'], + product_info[product_id]['BUILD'], + product_info[product_id]['PostDate'].strftime('%Y-%m-%d'), + product_info[product_id]['title'] + ) answer = raw_input( '\nChoose a product to download (1-%s): ' % len(product_info))