diff --git a/cx_Freeze/darwintools.py b/cx_Freeze/darwintools.py new file mode 100644 index 0000000000..467b438f4c --- /dev/null +++ b/cx_Freeze/darwintools.py @@ -0,0 +1,285 @@ +import os +import sys +import subprocess +import stat +from typing import List, Dict, Optional, Set + + +# need to deal with @executable_path, @loader_path, @rpath +# @executable_path - where ultimate calling executable is +# @loader_path - directory of current object +# @rpath - list of paths to check (earlier rpaths have higher priority, i believe) + + +def _isMachOFile(path: str) -> bool: + """Determines whether the file is a Mach-O file.""" + if not os.path.isfile(path): return False + p = subprocess.Popen(("file", path), stdout=subprocess.PIPE) + if "Mach-O" in p.stdout.readline().decode(): return True + return False + +class MachOReference: + def __init__(self, sourceFile: "DarwinFile", rawPath: str, resolvedPath: str): + self.sourceFile = sourceFile + self.rawPath = rawPath + self.resolvedPath = resolvedPath + self.isSystemFile = False # True if the target is a system file that will not be included in package + self.isCopied = False # True if the fie is being copied into the package + self.targetFile: DarwinFile = None # if the file is being copied into package, this is a refernece to the relevant DarwinFile + return + + def setTargetFile(self, darwinFile: "DarwinFile"): + self.targetFile = darwinFile + self.isCopied = True + return + +# a DarwinFile tracks a file referenced in the application, and record where it was ultimately moved to in the application bundle. +# should also safe a copy of the DarwinFile object, if any!, created for each referenced library + +class DarwinFile: + def __init__(self, originalObjectPath: str, referencingFile: Optional["DarwinFile"]=None): + self.originalObjectPath = os.path.abspath( originalObjectPath ) + self.copyDestinationPath: Optional[str] = None + self.commands: List[MachOCommand] = [] + self.loadCommands: List[MachOLoadCommand] = [] + self.rpathCommands: List[MachORPathCommand] = [] + self.referencingFile: Optional[DarwinFile] = None + self.libraryPathResolution: Dict[str, str] = {} + self._rpath: Optional[List[str]] = None + self.machReferenceDict: Dict[str, MachOReference] = {} + self.isMachO = False + + if not _isMachOFile(path=self.originalObjectPath): return + + self.isMachO = True + self.commands = MachOCommand._getMachOCommands(path=self.originalObjectPath) + self.loadCommands = [c for c in self.commands if isinstance(c, MachOLoadCommand)] + self.rpathCommands = [c for c in self.commands if isinstance(c, MachORPathCommand)] + self.referencingFile = referencingFile + + self.getRPath() + self.resolveLibraryPaths() + + for rawPath, resolvedPath in self.libraryPathResolution.items(): + if resolvedPath in self.machReferenceDict: + raise Exception("Dynamic libraries resolved to the same file?") + self.machReferenceDict[resolvedPath] = MachOReference(sourceFile=self, + rawPath=rawPath, resolvedPath=resolvedPath) + pass + return + + def __str__(self): + l = [] + # l.append("RPath Commands: {}".format(self.rpathCommands)) + # l.append("Load commands: {}".format(self.loadCommands)) + l.append("Mach-O File: {}".format(self.originalObjectPath)) + l.append("Resolved rpath:") + for rp in self.getRPath(): + l.append(" {}".format(rp)) + pass + l.append("Loaded libraries:") + for rp in self.libraryPathResolution: + l.append(" {} -> {}".format(rp, self.libraryPathResolution[rp])) + pass + return "\n".join(l) + + @staticmethod + def isExecutablePath(path: str) -> bool: + return path.startswith("@executable_path") + + @staticmethod + def isLoaderPath(path: str) -> bool: + return path.startswith("@loader_path") + + @staticmethod + def isRPath(path: str) -> bool: + return path.startswith("@rpath") + + def sourceDir(self) -> str: + return os.path.dirname( self.originalObjectPath ) + + def resolveLoader(self, path:str) -> Optional[str]: + if self.isLoaderPath(path=path): + return path.replace("@loader_path", self.sourceDir(), 1) + raise Exception("resolveLoader() called on bad path: {}".format(path)) + + + def resolveExecutable(self, path:str) -> str: + if self.isExecutablePath(path=path): + return path.replace("@executable_path", self.sourceDir(), 1) + raise Exception("resolveExecutable() called on bad path: {}".format(path)) + + def resolveRPath(self, path: str) -> str: + for rp in self.getRPath(): + testPath = os.path.abspath( path.replace("@rpath", rp, 1) ) + if _isMachOFile(testPath): return testPath + pass + raise Exception("resolveRPath() failed to resolve path: {}".format(path)) + + def getRPath(self) -> List[str]: + """Returns the rpath in effect for this file.""" + if self._rpath is not None: return self._rpath + rawPaths = [c.rPath for c in self.rpathCommands] + rpath = [] + for rp in rawPaths: + if os.path.isabs(rp): rpath.append(rp) + elif self.isLoaderPath(rp): rpath.append(self.resolveLoader(rp)) + elif self.isExecutablePath(rp): rpath.append(self.resolveExecutable(rp)) + pass + + rpath = [os.path.abspath(rp) for rp in rpath] + rpath = [rp for rp in rpath if os.path.exists(rp) ] + + if self.referencingFile is not None: + rpath = self.referencingFile.getRPath() + rpath + pass + self._rpath = rpath + return self._rpath + + def resolvePath(self, path) -> str: + """Resolves @executable_path, @loader_path, and @rpath references in a path.""" + if self.isLoaderPath(path): # replace @loader_path + return self.resolveLoader(path) + if self.isExecutablePath(path): # replace @executable_path + return self.resolveExecutable(path) + if self.isRPath(path): # replace @rpath + return self.resolveRPath(path) + if os.path.isabs(path): # just use the path, if it is absolute + return path + testPath = os.path.abspath( os.path.join(self.sourceDir(), path) ) + if _isMachOFile(path=testPath): return testPath + raise Exception("Could not resolve path: {}".format(path)) + + def resolveLibraryPaths(self): + for lc in self.loadCommands: + rawPath = lc.loadPath + resolvedPath = self.resolvePath(path=rawPath) + self.libraryPathResolution[rawPath]= resolvedPath + pass + return + + def getDependentFiles(self) -> List[str]: + dependents: List[str] = [] + for rp,ref in self.machReferenceDict.items(): + dependents.append(ref.resolvedPath) + pass + return dependents + + def getMachOReference(self, resolvedPath: str) -> MachOReference: + return self.machReferenceDict[resolvedPath] + + def setCopyDestination(self, destinationPath: str): + """Tell the Mach-O file its relative position (compared to executable) in the bundled package.""" + self.copyDestinationPath = destinationPath + return + + pass + +class MachOCommand: + def __init__(self, lines: List[str]): + self.lines = lines + return + + def __repr__(self): + return "" + + @staticmethod + def _getMachOCommands(path) -> List["MachOCommand"]: + """Returns a list of load commands in the specified file, based on otool.""" + shellCommand = 'otool -l "{}"'.format(path) + commands: List[MachOCommand] = [] + currentCommandLines = None + + # split the output into separate load commands + for line in os.popen(shellCommand): + line = line.strip() + if line[:12] == "Load command": + if currentCommandLines is not None: + commands.append(MachOCommand.parseLines(lines=currentCommandLines)) + pass + currentCommandLines = [] + if currentCommandLines is not None: + currentCommandLines.append(line) + pass + pass + if currentCommandLines is not None: + commands.append(currentCommandLines) + return commands + + @staticmethod + def parseLines(lines: List[str]) -> "MachOCommand": + if len(lines) < 2: return MachOCommand(lines=lines) + commandLinePieces = lines[1].split(" ") + if commandLinePieces[0] != "cmd": return MachOCommand(lines=lines) + if commandLinePieces[1] == "LC_LOAD_DYLIB": return MachOLoadCommand(lines=lines) + if commandLinePieces[1] == "LC_RPATH": return MachORPathCommand(lines=lines) + return MachOCommand(lines=lines) + + pass + +class MachOLoadCommand(MachOCommand): + def __init__(self, lines: List[str]): + super().__init__(lines=lines) + self.loadPath = None + if len(self.lines) < 4: return + pathline = self.lines[3] + pathline = pathline.strip() + if not pathline.startswith("name "): return + pathline = pathline[4:].strip() + pathline = pathline.split("(offset")[0].strip() + self.loadPath = pathline + return + + def getPath(self): return self.loadPath + + def __repr__(self): + return "".format(self.loadPath) + +class MachORPathCommand(MachOCommand): + def __init__(self, lines: List[str]): + super().__init__(lines=lines) + self.rPath = None + if len(self.lines) < 4: return + pathline = self.lines[3] + pathline = pathline.strip() + if not pathline.startswith("path "): return + pathline = pathline[4:].strip() + pathline = pathline.split("(offset")[0].strip() + self.rPath = pathline + return + + def __repr__(self): + return "".format(self.rPath) + pass + +def _printFile(machOFile: DarwinFile, seenFiles: Set[DarwinFile], level: int, noRecurse=False): + print("{}{} {}".format(level*"| ", machOFile.originalObjectPath, "(already seen)" if noRecurse else "")) + if noRecurse: return + for path, ref in machOFile.machReferenceDict.items(): + if not ref.isCopied: continue + mf = ref.targetFile + _printFile(mf, seenFiles=seenFiles, level=level+1, noRecurse=(mf in seenFiles)) + seenFiles.add(mf) + pass + return + +def printMachOFiles(fileList: List[DarwinFile]): + seenFiles = set() + for mf in fileList: + if mf not in seenFiles: + seenFiles.add(mf) + _printFile(mf, seenFiles=seenFiles, level=0) + pass + pass + return + +def changeLoadReference(fileName: str, oldReference: str, newReference: str, VERBOSE: bool=True): + if VERBOSE: + print("Redirecting load reference for <{}> {} -> {}".format(fileName, oldReference, newReference)) + original = os.stat(fileName).st_mode + newMode = original | stat.S_IWUSR + os.chmod(fileName, newMode) + subprocess.call(('install_name_tool', '-change', oldReference, newReference, fileName)) + os.chmod(fileName, original) + return + diff --git a/cx_Freeze/dist.py b/cx_Freeze/dist.py index b8dbd8b2a1..b5b2bcebfc 100644 --- a/cx_Freeze/dist.py +++ b/cx_Freeze/dist.py @@ -214,6 +214,8 @@ def run(self): metadata=metadata, zipIncludePackages=self.zip_include_packages, zipExcludePackages=self.zip_exclude_packages) + + self.freezer = freezer # keep freezer around so that its data case be used in bdist_mac phase freezer.Freeze() def set_source_location(self, name, *pathParts): diff --git a/cx_Freeze/freezer.py b/cx_Freeze/freezer.py index bd241f2553..bd2b1618ff 100644 --- a/cx_Freeze/freezer.py +++ b/cx_Freeze/freezer.py @@ -19,6 +19,7 @@ import zipfile import cx_Freeze +from cx_Freeze.darwintools import DarwinFile, MachOReference __all__ = [ "ConfigError", "ConstantsModule", "Executable", "Freezer" ] @@ -130,10 +131,27 @@ def _AddVersionResource(self, exe): stamp(fileName, versionInfo) def _CopyFile(self, source, target, copyDependentFiles, - includeMode = False, relativeSource = False): + includeMode = False, relativeSource = False, + machOReference: MachOReference = None): normalizedSource = os.path.normcase(os.path.normpath(source)) normalizedTarget = os.path.normcase(os.path.normpath(target)) + if normalizedTarget in self.filesCopied: + if sys.platform == "darwin" and (machOReference is not None): + # If file was already copied, and we are following a reference from a DarwinFile, then we need + # to tell the reference where the file was copied to. + if normalizedTarget not in self.darwinFileDict: + raise Exception("File \"{}\" already copied to, but no DarwinFile object found for it.".format(normalizedTarget)) + assert (normalizedTarget in self.darwinFileDict) + # confirm that the file already copied to this location came from the same source + targetFile: DarwinFile = self.darwinFileDict[normalizedTarget] + machOReference.setTargetFile(darwinFile=targetFile) + if targetFile.originalObjectPath != normalizedSource: + raise Exception("Attempting to copy two files to \"{}\" (\"{}\" and \"{}\")".format( + normalizedTarget, normalizedSource, targetFile.originalObjectPath + )) + assert( targetFile.originalObjectPath == normalizedSource ) + #TODO: we should check that targetFile has the same source path as the reference specifies (otherwise, we have multiple files being copied to the same target... return if normalizedSource == normalizedTarget: return @@ -147,22 +165,43 @@ def _CopyFile(self, source, target, copyDependentFiles, if includeMode: shutil.copymode(source, target) self.filesCopied[normalizedTarget] = None + + newDarwinFile = None + if sys.platform == "darwin": + # Create a DarwinFile file object to represent the file being copied. + referencingFile = None + if machOReference is not None: + referencingFile = machOReference.sourceFile + newDarwinFile = DarwinFile(originalObjectPath=source, referencingFile=referencingFile) + newDarwinFile.copyDestinationPath = normalizedTarget + if machOReference is not None: + machOReference.setTargetFile(darwinFile=newDarwinFile) + self.darwinFiles.append(newDarwinFile) + self.darwinFileDict[normalizedTarget] = newDarwinFile + pass + if copyDependentFiles \ and source not in self.finder.exclude_dependent_files: # Always copy dependent files on root directory # to allow to set relative reference + sourceDir = os.path.dirname(source) + if sys.platform == 'darwin': targetDir = self.targetDir - sourceDir = os.path.dirname(source) - for source in self._GetDependentFiles(source): - if relativeSource and os.path.isabs(source) and \ - os.path.commonpath((source, sourceDir)) == sourceDir: - relative = os.path.relpath(source, sourceDir) - target = os.path.join(targetDir, relative) - else: - target = os.path.join(targetDir, os.path.basename(source)) - self._CopyFile(source, target, copyDependentFiles, - relativeSource=relativeSource) + for dependent_file in self._GetDependentFiles(source, darwinFile=newDarwinFile): + target = os.path.join(targetDir, os.path.basename(dependent_file)) + self._CopyFile(dependent_file, target, copyDependentFiles, + machOReference=newDarwinFile.getMachOReference(resolvedPath=dependent_file)) + else: + for dependent_file in self._GetDependentFiles(source, darwinFile=newDarwinFile): + if relativeSource and os.path.isabs(dependent_file) and \ + os.path.commonpath((dependent_file, sourceDir)) == sourceDir: + relative = os.path.relpath(dependent_file, sourceDir) + target = os.path.join(targetDir, relative) + else: + target = os.path.join(targetDir, os.path.basename(dependent_file)) + self._CopyFile(dependent_file, target, copyDependentFiles, + relativeSource=relativeSource) def _CreateDirectory(self, path): if not os.path.isdir(path): @@ -273,7 +312,7 @@ def _GetDefaultBinPathExcludes(self): return ["/lib", "/lib32", "/lib64", "/usr/lib", "/usr/lib32", "/usr/lib64"] - def _GetDependentFiles(self, path): + def _GetDependentFiles(self, path, darwinFile: DarwinFile = None): """Return the file's dependencies using platform-specific tools (the imagehlp library on Windows, otool on Mac OS X and ldd on Linux); limit this list by the exclusion lists as needed""" @@ -297,19 +336,16 @@ def _GetDependentFiles(self, path): os.environ["PATH"] = origPath else: dependentFiles = [] + elif sys.platform == "darwin": + dependentFiles = darwinFile.getDependentFiles() else: if not os.access(path, os.X_OK): self.dependentFiles[path] = [] return [] dependentFiles = [] - if sys.platform == "darwin": - command = 'otool -L "%s"' % path - splitString = " (compatibility" - dependentFileIndex = 0 - else: - command = 'ldd "%s"' % path - splitString = " => " - dependentFileIndex = 1 + command = 'ldd "%s"' % path + splitString = " => " + dependentFileIndex = 1 for line in os.popen(command): parts = line.expandtabs().strip().split(splitString) if len(parts) != 2: @@ -331,38 +367,11 @@ def _GetDependentFiles(self, path): dependentFile = dependentFile[:pos].strip() if dependentFile: dependentFiles.append(dependentFile) - if sys.platform == "darwin": - # Make library paths absolute. This is needed to use - # cx_Freeze on OSX in e.g. a conda-based distribution. - # Note that with @rpath we just assume Python's lib dir, - # which should work in most cases. - dependentFiles = [p.replace('@loader_path', dirname) - for p in dependentFiles] - dependentFiles = [p.replace('@rpath', sys.prefix + '/lib') - for p in dependentFiles] - if sys.platform == "darwin": - dependentFiles = [self._CheckDependentFile(f, dirname) - for f in dependentFiles if self._ShouldCopyFile(f)] - else: - dependentFiles = [os.path.normcase(f) - for f in dependentFiles if self._ShouldCopyFile(f)] + + dependentFiles = [f for f in dependentFiles if self._ShouldCopyFile(f)] self.dependentFiles[path] = dependentFiles return dependentFiles - def _CheckDependentFile(self, dependentFile, dirname): - """If the file does not exist, try to locate it in the directory of the - parent file (this is to workaround an issue in how otool returns - dependencies. See issue #292. - https://github.com/anthony-tuininga/cx_Freeze/issues/292""" - if os.path.isfile(dependentFile): - return dependentFile - basename = os.path.basename(dependentFile) - joined = os.path.join(dirname, basename) - if os.path.isfile(joined): - return joined - raise FileNotFoundError("otool returned a dependent file that " - "could not be found: " + dependentFile) - def _GetModuleFinder(self, argsSource = None): if argsSource is None: argsSource = self @@ -634,6 +643,8 @@ def Freeze(self): self.filesCopied = {} self.linkerWarnings = {} self.msvcRuntimeDir = None + self.darwinFiles = [] + self.darwinFileDict = {} self.finder = self._GetModuleFinder() for executable in self.executables: diff --git a/cx_Freeze/macdist.py b/cx_Freeze/macdist.py index 69b8445ac9..ec7160d61b 100644 --- a/cx_Freeze/macdist.py +++ b/cx_Freeze/macdist.py @@ -6,6 +6,8 @@ from cx_Freeze.common import normalize_to_list +from cx_Freeze.darwintools import changeLoadReference + __all__ = ["bdist_dmg", "bdist_mac"] @@ -169,78 +171,27 @@ def setAbsoluteReferencePaths(self, path=None): 'install_name_tool', '-change', lib, replacement, filename)) - def setRelativeReferencePaths(self): - """ Create a list of all the Mach-O binaries in Contents/MacOS. - Then, check if they contain references to other files in - that dir. If so, make those references relative. """ - files = [] - for root, dirs, dir_files in os.walk(self.binDir): - for f in dir_files: - p = subprocess.Popen(("file", os.path.join(root, f)), stdout=subprocess.PIPE) - if b"Mach-O" in p.stdout.readline(): - files.append(os.path.join(root, f).replace(self.binDir + "/", "")) - - for fileName in files: - - filePath = os.path.join(self.binDir, fileName) - - # ensure write permissions - mode = os.stat(filePath).st_mode - if not (mode & stat.S_IWUSR): - os.chmod(filePath, mode | stat.S_IWUSR) - - # let the file itself know its place - subprocess.call(('install_name_tool', '-id', - os.path.join('@executable_path', fileName), filePath)) - - # find the references: call otool -L on the file - otool = subprocess.Popen(('otool', '-L', filePath), - stdout=subprocess.PIPE) - references = otool.stdout.readlines()[1:] - - for reference in references: - - # find the actual referenced file name - referencedFile = reference.decode().strip().split()[0] - - if referencedFile.startswith('@executable_path'): - # the referencedFile is already a relative path (to the executable) - continue - - if self.rpath_lib_folder is not None: - referencedFile = str(referencedFile).replace("@rpath", self.rpath_lib_folder) - - # the output of otool on archive contain self referencing - # content inside parantheses. - if not os.path.exists(referencedFile): - print("skip unknown file {} ".format(referencedFile)) - continue - - path, name = os.path.split(referencedFile) - - # some referenced files have not previously been copied to the - # executable directory - the assumption is that you don't need - # to copy anything from /usr (but should from /usr/local) or - # /System, just from folders like /opt this fix should probably - # be elsewhere though - if (name not in files - and (not path.startswith('/usr') - or path.startswith('/usr/local')) - and not path.startswith('/System')): - print(referencedFile) - try: - self.copy_file(referencedFile, os.path.join(self.binDir, name)) - except DistutilsFileError as e: - print("issue copying {} to {} error {} skipping".format(referencedFile, os.path.join(self.binDir, name), e)) - else: - files.append(name) - - # see if we provide the referenced file; - # if so, change the reference - if name in files: - newReference = os.path.join('@executable_path', name) - subprocess.call(('install_name_tool', '-change', - referencedFile, newReference, filePath)) + def setRelativeReferencePaths(self, buildDir: str, binDir: str): + """ Make all the references from included Mach-O files to other included + Mach-O files relative. """ + + #TODO: Do an initial pass through the DarwinFiles to see if any references on DarwinFiles copied into the + # bundle that were not already set--in which case we set them? + + for mf in self.darwinFiles: + relativeMFDest = os.path.relpath(mf.copyDestinationPath, buildDir) + filePathInBinDir = os.path.join(binDir, relativeMFDest) + for path, ref in mf.machReferenceDict.items(): + if not ref.isCopied: continue + rawPath = ref.rawPath # this is the reference in the machO file that needs to be updated + targFile = ref.targetFile + absoluteDest = targFile.copyDestinationPath # this is the absolute in the build directory + relativeDest = os.path.relpath(absoluteDest, buildDir) + exePath = "@executable_path/{}".format(relativeDest) + changeLoadReference(filePathInBinDir,oldReference=rawPath,newReference=exePath, VERBOSE=False) + pass + pass + return def find_qt_menu_nib(self): """Returns a location of a qt_menu.nib folder, or None if this is not @@ -291,6 +242,7 @@ def prepare_qt_app(self): def run(self): self.run_command('build') build = self.get_finalized_command('build') + freezer = self.get_finalized_command('build_exe').freezer # Define the paths within the application bundle self.bundleDir = os.path.join(build.build_base, @@ -337,7 +289,8 @@ def run(self): self.execute(self.create_plist, ()) # Make all references to libraries relative - self.execute(self.setRelativeReferencePaths, ()) + self.darwinFiles = freezer.darwinFiles + self.execute(self.setRelativeReferencePaths, (os.path.abspath(build.build_exe), os.path.abspath(self.binDir))) # Make library references absolute if enabled if self.absolute_reference_path: