From fc4a43d39aabcb6996b60d80015f9f6aa278132f Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Sat, 26 Apr 2014 23:49:42 +0200 Subject: [PATCH] osxbundle: split and optimize bundling script Move the code that copies the dylib's to the bundle to a new script (dylib-unhell.py) which is called by osxbundle.py. dylib-unhell is about 20x faster than the previous implementation. This is accomplished by removing superflous shell-out operations which are kept track of using an in memory tree of all the needed dependencies. Moreover the shell-outs have been further optimized by not requiring a complete shell for every operation and just using subprocess.call (which is equivalent to Popen). --- TOOLS/dylib-unhell.py | 93 +++++++++++++++++++++++++++++++++++++++++++ TOOLS/osxbundle.py | 67 +------------------------------ 2 files changed, 94 insertions(+), 66 deletions(-) create mode 100755 TOOLS/dylib-unhell.py diff --git a/TOOLS/dylib-unhell.py b/TOOLS/dylib-unhell.py new file mode 100755 index 0000000000..764d6c351b --- /dev/null +++ b/TOOLS/dylib-unhell.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +import re +import os +import sys +import shutil +import subprocess +from functools import partial + +sys_re = re.compile("^/System") +usr_re = re.compile("^/usr/lib/") +exe_re = re.compile("@executable_path") + +def is_user_lib(objfile, libname): + return not sys_re.match(libname) and \ + not usr_re.match(libname) and \ + not exe_re.match(libname) and \ + not "libobjc" in libname and \ + not "libSystem" in libname and \ + not "libc" in libname and \ + not "libgcc" in libname and \ + not os.path.basename(objfile) in libname + +def otool(objfile): + command = "otool -L %s | grep -e '\t' | awk '{ print $1 }'" % objfile + output = subprocess.check_output(command, shell = True) + return filter(partial(is_user_lib, objfile), output.split()) + +def install_name_tool_change(old, new, objfile): + subprocess.call(["install_name_tool", "-change", old, new, objfile]) + +def install_name_tool_id(name, objfile): + subprocess.call(["install_name_tool", "-id", name, objfile]) + +def libraries(objfile, result = dict()): + libs_list = otool(objfile) + result[objfile] = set(libs_list) + + for lib in libs_list: + if lib not in result: + libraries(lib, result) + + return result + +def leafs(libs_dict, processed = []): + result = [] + processed = set(processed) + + for objfile, libs in libs_dict.items(): + if libs <= processed: + result.append(objfile) + + return result + +def lib_path(binary): + return os.path.join(os.path.dirname(binary), 'lib') + +def lib_name(lib): + return os.path.join("@executable_path", "lib", os.path.basename(lib)) + +def process_libraries(libs_dict, binary, processed = []): + ls = leafs(libs_dict, processed) + diff = set(ls) - set(processed) + if diff == set(): + return + + for src in diff: + name = lib_name(src) + dst = os.path.join(lib_path(binary), os.path.basename(src)) + + shutil.copy(src, dst) + os.chmod(dst, 0o755) + install_name_tool_id(name, dst) + + if src in libs_dict[binary]: + install_name_tool_change(src, name, binary) + + for p in processed: + if p in libs_dict[src]: + install_name_tool_change(p, lib_name(p), dst) + + process_libraries(libs_dict, binary, ls) + +def main(): + binary = os.path.abspath(sys.argv[1]) + if not os.path.exists(lib_path(binary)): + os.makedirs(lib_path(binary)) + libs = libraries(binary) + print(libs) + process_libraries(libs, binary) + +if __name__ == "__main__": + main() diff --git a/TOOLS/osxbundle.py b/TOOLS/osxbundle.py index ec9427b9db..3053fec7b0 100755 --- a/TOOLS/osxbundle.py +++ b/TOOLS/osxbundle.py @@ -1,35 +1,13 @@ #!/usr/bin/env python import os -import re import shutil import sys from optparse import OptionParser -from textwrap import dedent def sh(command): return os.popen(command).read() -def dylib_lst(input_file): - return sh("otool -L %s | grep -e '\t' | awk '{ print $1 }'" % input_file) - -sys_re = re.compile("/System") -exe_re = re.compile("@executable_path") -binary_name = sys.argv[1] - -def is_user_lib(libname, input_file): - return not sys_re.match(libname) and \ - not exe_re.match(libname) and \ - not "libobjc" in libname and \ - not "libSystem" in libname and \ - not "libgcc" in libname and \ - not os.path.basename(input_file) in libname and \ - not libname == '' - -def user_dylib_lst(input_file): - return [lib for lib in dylib_lst(input_file).split("\n") if - is_user_lib(lib, input_file)] - def bundle_path(binary_name): return "%s.app" % binary_name @@ -56,52 +34,9 @@ def copy_bundle(binary_name): def copy_binary(binary_name): shutil.copy(binary_name, target_binary(binary_name)) -def run_install_name_tool(target_file, dylib_path, dest_dir, root=True): - new_dylib_path = os.path.join("@executable_path", "lib", - os.path.basename(dylib_path)) - - sh("install_name_tool -change %s %s %s" % \ - (dylib_path, new_dylib_path, target_file)) - if root: - sh("install_name_tool -id %s %s" % \ - (new_dylib_path, os.path.join(dest_dir, - os.path.basename(dylib_path)))) - -def cp_dylibs(target_file, dest_dir): - for dylib_path in user_dylib_lst(target_file): - dylib_dest_path = os.path.join(dest_dir, os.path.basename(dylib_path)) - - try: - shutil.copy(dylib_path, dylib_dest_path) - except IOError: - if re.match("dylib$", target_file): - reinstall_what = target_file - else: - reinstall_what = dylib_path - - sys.exit(dedent("""\ - %s uses library %s which is not available anymore. - This is most likely because you uninstalled %s. - Please reinstall %s to fix it's dependencies.""" % \ - (target_file, dylib_path, dylib_path, reinstall_what) )) - - os.chmod(dylib_dest_path, 0o755) - cp_dylibs(dylib_dest_path, dest_dir) - -def fix_dylibs_paths(target_file, dest_dir, root=True): - for dylib_path in user_dylib_lst(target_file): - dylib_dest_path = os.path.join(dest_dir, os.path.basename(dylib_path)) - run_install_name_tool(target_file, dylib_path, dest_dir, root) - fix_dylibs_paths(dylib_dest_path, dest_dir, False) - def apply_plist_template(plist_file, version): sh("sed -i -e 's/${VERSION}/%s/g' %s" % (version, plist_file)) -def bundle_dependencies(binary_name): - lib_bundle_directory = os.path.join(target_directory(binary_name), "lib") - cp_dylibs(binary_name, lib_bundle_directory) - fix_dylibs_paths(target_binary(binary_name), lib_bundle_directory) - def main(): version = sh("./version.sh --print").strip() @@ -128,7 +63,7 @@ def main(): if options.deps: print("> bundling dependencies") - bundle_dependencies(binary_name) + sh(" ".join(["TOOLS/dylib-unhell.py", target_binary(binary_name)])) print("done.")