From 36cccc569aaae179c6a6d4ed442cbff2bcc7fb07 Mon Sep 17 00:00:00 2001 From: cainesi Date: Mon, 9 Nov 2020 20:17:28 -0500 Subject: [PATCH] Improved the resolution of dependencies in darwin MachO files. (#590) --- cx_Freeze/darwintools.py | 454 ++++++++++++++++++ cx_Freeze/dist.py | 3 + cx_Freeze/freezer.py | 125 +++-- cx_Freeze/macdist.py | 107 ++--- cx_Freeze/samples/PyQt5_plugins/setup.py | 94 ++++ .../samples/PyQt5_plugins/test_script.py | 9 + .../samples/PyQt5_plugins/test_script2.py | 2 + 7 files changed, 671 insertions(+), 123 deletions(-) create mode 100644 cx_Freeze/darwintools.py create mode 100644 cx_Freeze/samples/PyQt5_plugins/setup.py create mode 100644 cx_Freeze/samples/PyQt5_plugins/test_script.py create mode 100644 cx_Freeze/samples/PyQt5_plugins/test_script2.py diff --git a/cx_Freeze/darwintools.py b/cx_Freeze/darwintools.py new file mode 100644 index 000000000..76637922f --- /dev/null +++ b/cx_Freeze/darwintools.py @@ -0,0 +1,454 @@ +import os +import subprocess +import stat +from typing import List, Tuple, ItemsView, Dict, Optional, Set, Iterable + + +# In a MachO file, need to deal specially with links that use @executable_path, +# @loader_path, @rpath +# +# @executable_path - where ultimate calling executable is +# @loader_path - directory of current object +# @rpath - list of paths to check (earlier rpaths have higher priority, i believe) +# +# Resolving these variables (particularly @rpath) requires tracing through the sequence +# linked MachO files leading the the current file, to determine which directories are +# included in the current rpath. + +class DarwinException(Exception): + pass + +def _isMachOFile(path: str) -> bool: + """Determines whether the file is a Mach-O file.""" + if not os.path.isfile(path): + return False + p = subprocess.Popen(("file", path), stdout=subprocess.PIPE) + if "Mach-O" in p.stdout.readline().decode(): + return True + return False + +class MachOReference: + """Represents a linking reference from MachO file to another file.""" + def __init__(self, sourceFile: "DarwinFile", rawPath: str, resolvedPath: str): + """ + :param sourceFile: DarwinFile object for file in which the reference was found + :param rawPath: The path that appears in the file (may include @rpath, etc.) + :param resolvedPath: The path resolved to an explicit path to a file on system. + """ + self.sourceFile: "DarwinFile" = sourceFile + self.rawPath: str = rawPath + self.resolvedPath: str = resolvedPath + + # isSystemFile is True if the target is a system file that will not be + # included in package + self.isSystemFile = False + # True if the file is being copied into the package + self.isCopied = False + # reference to target DarwinFile (but only if file is copied into app) + self.targetFile: Optional[DarwinFile] = None + return + + def setTargetFile(self, darwinFile: "DarwinFile"): + self.targetFile = darwinFile + self.isCopied = True + return + +class DarwinFile: + """A DarwinFile object tracks a file referenced in the application, and record where it was + ultimately moved to in the application bundle. Should also save a copy of the DarwinFile + object, if any!, created for each referenced library.""" + + def __init__(self, originalFilePath: str, + referencingFile: Optional["DarwinFile"]=None): + """ + :param originalFilePath: The original path of the DarwinFile (before copying into app) + :param referencingFile: DarwinFile object representing the referencing source file + """ + self.originalFilePath = os.path.realpath(originalFilePath) + self._buildPath: Optional[str] = None # path to file in build directory + self.commands: List[MachOCommand] = [] + self.loadCommands: List[MachOLoadCommand] = [] + self.rpathCommands: List[MachORPathCommand] = [] + + # note -- if file gets referenced twice (or more), it will only be the first + # reference that gets recorded. + self.referencingFile: Optional[DarwinFile] = None + self.libraryPathResolution: Dict[str, str] = {} + self._rpath: Optional[List[str]] = None + + # dictionary of MachOReference objects, by their resolved paths + self.machOReferenceDict: Dict[str, MachOReference] = {} + self.isMachO = False + + if not _isMachOFile(path=self.originalFilePath): + return + + # if this is a MachO file, extract linking information from it + self.isMachO = True + self.commands = MachOCommand._getMachOCommands(forFileAtPath=self.originalFilePath) + self.loadCommands = [c for c in self.commands if isinstance(c, MachOLoadCommand)] + self.rpathCommands = [c for c in self.commands if isinstance(c, MachORPathCommand)] + self.referencingFile = referencingFile + + self.getRPath() + self.resolveLibraryPaths() + + for rawPath, resolvedPath in self.libraryPathResolution.items(): + if resolvedPath in self.machOReferenceDict: + raise DarwinException("Dynamic libraries resolved to the same file?") + self.machOReferenceDict[resolvedPath] = MachOReference(sourceFile=self, + rawPath=rawPath, + resolvedPath=resolvedPath) + pass + return + + def __str__(self): + l = [] + # l.append("RPath Commands: {}".format(self.rpathCommands)) + # l.append("Load commands: {}".format(self.loadCommands)) + l.append("Mach-O File: {}".format(self.originalFilePath)) + l.append("Resolved rpath:") + for rp in self.getRPath(): + l.append(" {}".format(rp)) + pass + l.append("Loaded libraries:") + for rp in self.libraryPathResolution: + l.append(" {} -> {}".format(rp, self.libraryPathResolution[rp])) + pass + return "\n".join(l) + + def setBuildPath(self, path: str): + self._buildPath = path + return + + def getBuildPath(self) -> Optional[str]: + return self._buildPath + + @staticmethod + def isExecutablePath(path: str) -> bool: + return path.startswith("@executable_path") + + @staticmethod + def isLoaderPath(path: str) -> bool: + return path.startswith("@loader_path") + + @staticmethod + def isRPath(path: str) -> bool: + return path.startswith("@rpath") + + def sourceDir(self) -> str: + return os.path.dirname(self.originalFilePath) + + def resolveLoader(self, path:str) -> Optional[str]: + """Resolve a path that includes @loader_path. @loader_path represents the directory in which + the DarwinFile is located.""" + if self.isLoaderPath(path=path): + return path.replace("@loader_path", self.sourceDir(), 1) + raise DarwinException("resolveLoader() called on bad path: {}".format(path)) + + + def resolveExecutable(self, path:str) -> str: + """@executable_path should resolve to the directory where the original executable was located. + By default, we set that to the directory of the library, so it would resolve in the same was as if + linked from an executable in the same directory. + """ + # consider making this resolve to the directory of the target script instead? + if self.isExecutablePath(path=path): + return path.replace("@executable_path", self.sourceDir(), 1) + raise DarwinException("resolveExecutable() called on bad path: {}".format(path)) + + def resolveRPath(self, path: str) -> str: + for rp in self.getRPath(): + testPath = os.path.abspath( path.replace("@rpath", rp, 1) ) + if _isMachOFile(testPath): + return testPath + pass + raise DarwinException("resolveRPath() failed to resolve path: {}".format(path)) + + def getRPath(self) -> List[str]: + """Returns the rpath in effect for this file. Determined by rpath commands in this file + and (recursively) the chain of files that referenced this file.""" + if self._rpath is not None: + return self._rpath + rawPaths = [c.rPath for c in self.rpathCommands] + rpath = [] + for rp in rawPaths: + if os.path.isabs(rp): rpath.append(rp) + elif self.isLoaderPath(rp): rpath.append(self.resolveLoader(rp)) + elif self.isExecutablePath(rp): rpath.append(self.resolveExecutable(rp)) + pass + + rpath = [os.path.abspath(rp) for rp in rpath] + rpath = [rp for rp in rpath if os.path.exists(rp) ] + + if self.referencingFile is not None: + rpath = self.referencingFile.getRPath() + rpath + pass + self._rpath = rpath + return self._rpath + + def resolvePath(self, path) -> str: + """Resolves any @executable_path, @loader_path, and @rpath references + in a path.""" + if self.isLoaderPath(path): # replace @loader_path + return self.resolveLoader(path) + if self.isExecutablePath(path): # replace @executable_path + return self.resolveExecutable(path) + if self.isRPath(path): # replace @rpath + return self.resolveRPath(path) + if os.path.isabs(path): # just use the path, if it is absolute + return path + testPath = os.path.abspath( os.path.join(self.sourceDir(), path) ) + if _isMachOFile(path=testPath): + return testPath + raise DarwinException("Could not resolve path: {}".format(path)) + + def resolveLibraryPaths(self): + for lc in self.loadCommands: + rawPath = lc.loadPath + resolvedPath = self.resolvePath(path=rawPath) + self.libraryPathResolution[rawPath]= resolvedPath + pass + return + + def getDependentFilePaths(self) -> List[str]: + dependents: List[str] = [] + for rp,ref in self.machOReferenceDict.items(): + dependents.append(ref.resolvedPath) + pass + return dependents + + def getMachOReference(self, resolvedPath: str) -> MachOReference: + if resolvedPath not in self.machOReferenceDict: + raise DarwinException("Path {} is not a path referenced from DarwinFile".format(resolvedPath)) + return self.machOReferenceDict[resolvedPath] + + def getMachOReferences(self) -> Iterable[Tuple[str, MachOReference]]: + return self.machOReferenceDict.items() + + def setCopyDestination(self, destinationPath: str): + """Tell the Mach-O file its relative position (compared to executable) + in the bundled package.""" + self._buildPath = destinationPath + return + + pass + +class MachOCommand: + """Represents a load command in a MachO file.""" + def __init__(self, lines: List[str]): + self.lines = lines + return + + def __repr__(self): + return "" + + @staticmethod + def _getMachOCommands(forFileAtPath: str) -> List["MachOCommand"]: + """Returns a list of load commands in the specified file, using otool.""" + shellCommand = 'otool -l "{}"'.format(forFileAtPath) + commands: List[MachOCommand] = [] + currentCommandLines = None + + # split the output into separate load commands + for line in os.popen(shellCommand): + line = line.strip() + if line[:12] == "Load command": + if currentCommandLines is not None: + commands.append(MachOCommand.parseLines(lines=currentCommandLines)) + pass + currentCommandLines = [] + if currentCommandLines is not None: + currentCommandLines.append(line) + pass + pass + if currentCommandLines is not None: + commands.append(currentCommandLines) + return commands + + @staticmethod + def parseLines(lines: List[str]) -> "MachOCommand": + if len(lines) < 2: + return MachOCommand(lines=lines) + commandLinePieces = lines[1].split(" ") + if commandLinePieces[0] != "cmd": + return MachOCommand(lines=lines) + if commandLinePieces[1] == "LC_LOAD_DYLIB": + return MachOLoadCommand(lines=lines) + if commandLinePieces[1] == "LC_RPATH": + return MachORPathCommand(lines=lines) + return MachOCommand(lines=lines) + + pass + +class MachOLoadCommand(MachOCommand): + def __init__(self, lines: List[str]): + super().__init__(lines=lines) + self.loadPath = None + if len(self.lines) < 4: + return + pathline = self.lines[3] + pathline = pathline.strip() + if not pathline.startswith("name "): + return + pathline = pathline[4:].strip() + pathline = pathline.split("(offset")[0].strip() + self.loadPath = pathline + return + + def getPath(self): + return self.loadPath + + def __repr__(self): + return "".format(self.loadPath) + +class MachORPathCommand(MachOCommand): + def __init__(self, lines: List[str]): + super().__init__(lines=lines) + self.rPath = None + if len(self.lines) < 4: + return + pathline = self.lines[3] + pathline = pathline.strip() + if not pathline.startswith("path "): + return + pathline = pathline[4:].strip() + pathline = pathline.split("(offset")[0].strip() + self.rPath = pathline + return + + def __repr__(self): + return "".format(self.rPath) + pass + +def _printFile(darwinFile: DarwinFile, seenFiles: Set[DarwinFile], + level: int, noRecurse=False): + """Utility function to prints details about a DarwinFile and (optionally) recursively + any other DarwinFiles that it references.""" + print("{}{} {}".format(level *"| ", darwinFile.originalFilePath, + "(already seen)" if noRecurse else "")) + if noRecurse: + return + for path, ref in darwinFile.machOReferenceDict.items(): + if not ref.isCopied: continue + mf = ref.targetFile + _printFile(mf, seenFiles=seenFiles, level=level+1, noRecurse=(mf in seenFiles)) + seenFiles.add(mf) + pass + return + +def printMachOFiles(fileList: List[DarwinFile]): + seenFiles = set() + for mf in fileList: + if mf not in seenFiles: + seenFiles.add(mf) + _printFile(mf, seenFiles=seenFiles, level=0) + pass + pass + return + +def changeLoadReference(fileName: str, oldReference: str, newReference: str, + VERBOSE: bool=True): + """Utility function that uses intall_name_tool to change oldReference to + newReference in the machO file specified by fileName.""" + if VERBOSE: + print("Redirecting load reference for <{}> {} -> {}".format(fileName, + oldReference, + newReference)) + original = os.stat(fileName).st_mode + newMode = original | stat.S_IWUSR + os.chmod(fileName, newMode) + subprocess.call(('install_name_tool', '-change', oldReference, newReference, fileName)) + os.chmod(fileName, original) + return + +class DarwinFileTracker: + """Object to track the DarwinFiles that have been added during a freeze.""" + + def __init__(self): + # a list of DarwinFile objects for files being copied into project + self._targetFileList: List[DarwinFile] = [] + + # a dictionary mapping (build directory) target paths to DarwinFile objects + self._targetFileDict: Dict[str, DarwinFile] = {} + + self._sourceFileDict: Dict[str, DarwinFile] = {} + + # a mapping of (source location) paths to the MacOReferences to them + self._referenceCache: Dict[str, MachOReference] = {} + return + + def __iter__(self) -> Iterable[DarwinFile]: + return iter(self._targetFileList) + + def pathIsAlreadyCopiedTo(self, targetPath: str) -> bool: + """Check if the given targetPath has already has a file copied to it.""" + if targetPath in self._targetFileDict: return True + return False + + def getDarwinFile(self, sourcePath: str, targetPath: str) -> DarwinFile: + """Gets the DarwinFile for file copied from sourcePath to targetPath. If either (i) nothing, + or (ii) a different file has been copied to targetPath, raises a DarwinException.""" + + # check that the file has been copied to + if targetPath not in self._targetFileDict: + raise DarwinException( + "File \"{}\" already copied to, but no DarwinFile object found for it.".format(targetPath)) + + # check that the target file came from the specified source + targetDarwinFile: DarwinFile = self._targetFileDict[targetPath] + realSource = os.path.realpath(sourcePath) + targetRealSource = os.path.realpath(targetDarwinFile.originalFilePath) + if realSource != targetRealSource: + exceptionString = \ +"""Attempting to copy two files to "{}" + source 1: "{}" (real: "{}") + source 2: "{}" (real: "{}") +(This may be caused by including modules in the zip file that rely on binary libraries with the same name.)""" + exceptionString = exceptionString.format(targetPath, + targetDarwinFile.originalFilePath, targetRealSource, + sourcePath, realSource) + + raise DarwinException(exceptionString) + return targetDarwinFile + + + def recordCopiedFile(self, targetPath:str, darwinFile: DarwinFile): + """Record that a DarwinFile is being copied to a given path. If the same file has already been copied + to that path, do nothing. If a different file has been copied to that bath, raise a DarwinException.""" + if self.pathIsAlreadyCopiedTo(targetPath=targetPath): + raise DarwinException("addFile() called with targetPath already copied to (targetPath=\"{}\")".format(targetPath)) + + self._targetFileList.append(darwinFile) + self._targetFileDict[targetPath] = darwinFile + self._sourceFileDict[darwinFile.originalFilePath] = darwinFile + return + + def cacheReferenceTo(self, path: str, machOReference: MachOReference): + self._referenceCache[path] = machOReference + return + + def getCachedReferenceTo(self, path: str) -> Optional[MachOReference]: + if path in self._referenceCache: + return self._referenceCache[path] + return None + + def finalizeReferences(self): + """ + Goes through the stored list of target files, and updates references. + Normally the references may not be updated automatically, if _CopyFile is called without + copyDependentFiles=True. + """ + + for df in self._targetFileList: # DarwinFile + for path, ref in df.getMachOReferences(): # path and corresponding MachOReference to path + if not ref.isCopied: + # if reference not already marked as copied, check if it points to a file that is + # being copied and, if so, use that to complete the reference + realTargetPath = os.path.realpath( path ) + if realTargetPath in self._sourceFileDict: + ref.setTargetFile(self._sourceFileDict[realTargetPath]) + pass + pass + pass + return \ No newline at end of file diff --git a/cx_Freeze/dist.py b/cx_Freeze/dist.py index b8dbd8b2a..1efa5089d 100644 --- a/cx_Freeze/dist.py +++ b/cx_Freeze/dist.py @@ -214,6 +214,9 @@ def run(self): metadata=metadata, zipIncludePackages=self.zip_include_packages, zipExcludePackages=self.zip_exclude_packages) + + # keep freezer around so that its data case be used in bdist_mac phase + self.freezer = freezer freezer.Freeze() def set_source_location(self, name, *pathParts): diff --git a/cx_Freeze/freezer.py b/cx_Freeze/freezer.py index 940c2f89d..1d71fc855 100644 --- a/cx_Freeze/freezer.py +++ b/cx_Freeze/freezer.py @@ -17,9 +17,11 @@ import sys import sysconfig import time +from typing import Optional import zipfile import cx_Freeze +from cx_Freeze.darwintools import DarwinFile, MachOReference, DarwinFileTracker __all__ = [ "ConfigError", "ConstantsModule", "Executable", "Freezer" ] @@ -130,10 +132,18 @@ def _AddVersionResource(self, exe): stamp(fileName, versionInfo) def _CopyFile(self, source, target, copyDependentFiles, - includeMode = False, relativeSource = False): + includeMode = False, relativeSource = False, + machOReference: Optional[MachOReference] = None): normalizedSource = os.path.normcase(os.path.normpath(source)) normalizedTarget = os.path.normcase(os.path.normpath(target)) + if normalizedTarget in self.filesCopied: + if sys.platform == "darwin" and (machOReference is not None): + # If file was already copied, and we are following a reference from a DarwinFile, then we need + # to tell the reference where the file was copied to. + targetDarwinFile = self.darwinTracker.getDarwinFile(sourcePath=normalizedSource, + targetPath=normalizedTarget) + machOReference.setTargetFile(darwinFile=targetDarwinFile) return if normalizedSource == normalizedTarget: return @@ -147,6 +157,21 @@ def _CopyFile(self, source, target, copyDependentFiles, if includeMode: shutil.copymode(source, target) self.filesCopied[normalizedTarget] = None + + newDarwinFile = None + if sys.platform == "darwin": + # The file was not previously copied, so need to create a DarwinFile file object to + # represent the file being copied. + referencingFile = None + if machOReference is not None: + referencingFile = machOReference.sourceFile + newDarwinFile = DarwinFile(originalFilePath=source, referencingFile=referencingFile) + newDarwinFile.setBuildPath( normalizedTarget ) + if machOReference is not None: + machOReference.setTargetFile(darwinFile=newDarwinFile) + self.darwinTracker.recordCopiedFile(targetPath=normalizedTarget, darwinFile=newDarwinFile) + pass + if copyDependentFiles \ and source not in self.finder.exclude_dependent_files: # TODO: relativeSource for other platforms @@ -154,18 +179,24 @@ def _CopyFile(self, source, target, copyDependentFiles, relativeSource = False # Always copy dependent files on root directory # to allow to set relative reference + sourceDir = os.path.dirname(source) + if sys.platform == 'darwin': targetDir = self.targetDir - sourceDir = os.path.dirname(source) - for source in self._GetDependentFiles(source): - if relativeSource and os.path.isabs(source) and \ - os.path.commonpath((source, sourceDir)) == sourceDir: - relative = os.path.relpath(source, sourceDir) - target = os.path.join(targetDir, relative) - else: - target = os.path.join(targetDir, os.path.basename(source)) - self._CopyFile(source, target, copyDependentFiles, - relativeSource=relativeSource) + for dependent_file in self._GetDependentFiles(source, darwinFile=newDarwinFile): + target = os.path.join(targetDir, os.path.basename(dependent_file)) + self._CopyFile(dependent_file, target, copyDependentFiles=True, + machOReference=newDarwinFile.getMachOReference(resolvedPath=dependent_file)) + else: + for dependent_file in self._GetDependentFiles(source, darwinFile=newDarwinFile): + if relativeSource and os.path.isabs(dependent_file) and \ + os.path.commonpath((dependent_file, sourceDir)) == sourceDir: + relative = os.path.relpath(dependent_file, sourceDir) + target = os.path.join(targetDir, relative) + else: + target = os.path.join(targetDir, os.path.basename(dependent_file)) + self._CopyFile(dependent_file, target, copyDependentFiles, + relativeSource=relativeSource) def _CreateDirectory(self, path): if not os.path.isdir(path): @@ -209,8 +240,12 @@ def _FreezeExecutable(self, exe): target_dir = os.path.join(os.path.dirname(exe.targetName), "lib") for source in dependent_files: target = os.path.join(target_dir, os.path.basename(source)) - self._CopyFile(source, target, - copyDependentFiles=True, includeMode=True) + if sys.platform == "darwin": + self._CopyFile(source, target, copyDependentFiles=True, includeMode=True, + machOReference=self.darwinTracker.getCachedReferenceTo(path=source)) + else: + self._CopyFile(source, target, + copyDependentFiles=True, includeMode=True) self._CopyFile(exe.base, exe.targetName, copyDependentFiles=False, includeMode=True) if not os.access(exe.targetName, os.W_OK): @@ -278,7 +313,7 @@ def _GetDefaultBinPathExcludes(self): return ["/lib", "/lib32", "/lib64", "/usr/lib", "/usr/lib32", "/usr/lib64"] - def _GetDependentFiles(self, path): + def _GetDependentFiles(self, path, darwinFile: DarwinFile = None): """Return the file's dependencies using platform-specific tools (the imagehlp library on Windows, otool on Mac OS X and ldd on Linux); limit this list by the exclusion lists as needed""" @@ -302,19 +337,25 @@ def _GetDependentFiles(self, path): os.environ["PATH"] = origPath else: dependentFiles = [] + elif sys.platform == "darwin": + # if darwinFile is None, create a temporary DarwinFile object for the path, just + # so we can read its dependencies + if darwinFile is None: + darwinFile = DarwinFile(originalFilePath=path, referencingFile=None) + dependentFiles = darwinFile.getDependentFilePaths() + + # cache the MachOReferences to the dependencies, so they can be called up later + # in _CopyFile if copying a dependency without an explicit reference provided + for depFilePath, ref in darwinFile.getMachOReferences(): + self.darwinTracker.cacheReferenceTo(path=depFilePath, machOReference=ref) else: if not os.access(path, os.X_OK): self.dependentFiles[path] = [] return [] dependentFiles = [] - if sys.platform == "darwin": - command = 'otool -L "%s"' % path - splitString = " (compatibility" - dependentFileIndex = 0 - else: - command = 'ldd "%s"' % path - splitString = " => " - dependentFileIndex = 1 + command = 'ldd "%s"' % path + splitString = " => " + dependentFileIndex = 1 for line in os.popen(command): parts = line.expandtabs().strip().split(splitString) if len(parts) != 2: @@ -336,38 +377,12 @@ def _GetDependentFiles(self, path): dependentFile = dependentFile[:pos].strip() if dependentFile: dependentFiles.append(dependentFile) - if sys.platform == "darwin": - # Make library paths absolute. This is needed to use - # cx_Freeze on OSX in e.g. a conda-based distribution. - # Note that with @rpath we just assume Python's lib dir, - # which should work in most cases. - dependentFiles = [p.replace('@loader_path', dirname) - for p in dependentFiles] - dependentFiles = [p.replace('@rpath', sys.prefix + '/lib') - for p in dependentFiles] - if sys.platform == "darwin": - dependentFiles = [self._CheckDependentFile(f, dirname) - for f in dependentFiles if self._ShouldCopyFile(f)] - else: - dependentFiles = [os.path.normcase(f) - for f in dependentFiles if self._ShouldCopyFile(f)] + + dependentFiles = [os.path.normcase(f) for f in dependentFiles if + self._ShouldCopyFile(f)] self.dependentFiles[path] = dependentFiles return dependentFiles - def _CheckDependentFile(self, dependentFile, dirname): - """If the file does not exist, try to locate it in the directory of the - parent file (this is to workaround an issue in how otool returns - dependencies. See issue #292. - https://github.com/anthony-tuininga/cx_Freeze/issues/292""" - if os.path.isfile(dependentFile): - return dependentFile - basename = os.path.basename(dependentFile) - joined = os.path.join(dirname, basename) - if os.path.isfile(joined): - return joined - raise FileNotFoundError("otool returned a dependent file that " - "could not be found: " + dependentFile) - def _GetModuleFinder(self, argsSource = None): if argsSource is None: argsSource = self @@ -642,6 +657,10 @@ def Freeze(self): self.linkerWarnings = {} self.msvcRuntimeDir = None + self.darwinTracker: Optional[DarwinFileTracker] = None + if sys.platform == "darwin": + self.darwinTracker = DarwinFileTracker() + self.finder = self._GetModuleFinder() for executable in self.executables: self._FreezeExecutable(executable) @@ -677,6 +696,10 @@ def Freeze(self): copyDependentFiles=True, relativeSource=True) + if sys.platform == "darwin": + self.darwinTracker.finalizeReferences() + return + class ConfigError(Exception): diff --git a/cx_Freeze/macdist.py b/cx_Freeze/macdist.py index 69b8445ac..75bc10fba 100644 --- a/cx_Freeze/macdist.py +++ b/cx_Freeze/macdist.py @@ -6,6 +6,8 @@ from cx_Freeze.common import normalize_to_list +from cx_Freeze.darwintools import changeLoadReference, DarwinFile, DarwinFileTracker + __all__ = ["bdist_dmg", "bdist_mac"] @@ -169,78 +171,37 @@ def setAbsoluteReferencePaths(self, path=None): 'install_name_tool', '-change', lib, replacement, filename)) - def setRelativeReferencePaths(self): - """ Create a list of all the Mach-O binaries in Contents/MacOS. - Then, check if they contain references to other files in - that dir. If so, make those references relative. """ - files = [] - for root, dirs, dir_files in os.walk(self.binDir): - for f in dir_files: - p = subprocess.Popen(("file", os.path.join(root, f)), stdout=subprocess.PIPE) - if b"Mach-O" in p.stdout.readline(): - files.append(os.path.join(root, f).replace(self.binDir + "/", "")) - - for fileName in files: - - filePath = os.path.join(self.binDir, fileName) - - # ensure write permissions - mode = os.stat(filePath).st_mode - if not (mode & stat.S_IWUSR): - os.chmod(filePath, mode | stat.S_IWUSR) - - # let the file itself know its place - subprocess.call(('install_name_tool', '-id', - os.path.join('@executable_path', fileName), filePath)) - - # find the references: call otool -L on the file - otool = subprocess.Popen(('otool', '-L', filePath), - stdout=subprocess.PIPE) - references = otool.stdout.readlines()[1:] - - for reference in references: - - # find the actual referenced file name - referencedFile = reference.decode().strip().split()[0] - - if referencedFile.startswith('@executable_path'): - # the referencedFile is already a relative path (to the executable) + def setRelativeReferencePaths(self, buildDir: str, binDir: str): + """ Make all the references from included Mach-O files to other included + Mach-O files relative. """ + + # TODO: Do an initial pass through the DarwinFiles to see if any references on DarwinFiles copied into the + # bundle that were not already set--in which case we set them? + + for darwinFile in self.darwinTracker: + # get the relative path to darwinFile in build directory + relativeCopyDestination = os.path.relpath(darwinFile.getBuildPath(), buildDir) + # figure out directory where it will go in binary directory + # for .app bundle, this would be the Content/MacOS subdirectory in bundle + filePathInBinDir = os.path.join(binDir, relativeCopyDestination) + + # for each file that this darwinFile references, update the reference as necessary + # if the file is copied into the binary package, change the refernce to be relative to + # @executable_path (so an .app bundle will work wherever it is moved) + for path, machORef in darwinFile.machOReferenceDict.items(): + if not machORef.isCopied: + # referenced file not copied -- assume this is a system file that will also be + # present on the user's machine, and do not change reference continue - - if self.rpath_lib_folder is not None: - referencedFile = str(referencedFile).replace("@rpath", self.rpath_lib_folder) - - # the output of otool on archive contain self referencing - # content inside parantheses. - if not os.path.exists(referencedFile): - print("skip unknown file {} ".format(referencedFile)) - continue - - path, name = os.path.split(referencedFile) - - # some referenced files have not previously been copied to the - # executable directory - the assumption is that you don't need - # to copy anything from /usr (but should from /usr/local) or - # /System, just from folders like /opt this fix should probably - # be elsewhere though - if (name not in files - and (not path.startswith('/usr') - or path.startswith('/usr/local')) - and not path.startswith('/System')): - print(referencedFile) - try: - self.copy_file(referencedFile, os.path.join(self.binDir, name)) - except DistutilsFileError as e: - print("issue copying {} to {} error {} skipping".format(referencedFile, os.path.join(self.binDir, name), e)) - else: - files.append(name) - - # see if we provide the referenced file; - # if so, change the reference - if name in files: - newReference = os.path.join('@executable_path', name) - subprocess.call(('install_name_tool', '-change', - referencedFile, newReference, filePath)) + rawPath = machORef.rawPath # this is the reference in the machO file that needs to be updated + referencedDarwinFile: DarwinFile = machORef.targetFile + absoluteBuildDest = referencedDarwinFile.getBuildPath() # this is where file copied in build dir + relativeBuildDest = os.path.relpath(absoluteBuildDest, buildDir) + exePath = "@executable_path/{}".format(relativeBuildDest) + changeLoadReference(filePathInBinDir, oldReference=rawPath,newReference=exePath, VERBOSE=False) + pass + pass + return def find_qt_menu_nib(self): """Returns a location of a qt_menu.nib folder, or None if this is not @@ -291,6 +252,7 @@ def prepare_qt_app(self): def run(self): self.run_command('build') build = self.get_finalized_command('build') + freezer: "freezer.Freezer" = self.get_finalized_command('build_exe').freezer # Define the paths within the application bundle self.bundleDir = os.path.join(build.build_base, @@ -337,7 +299,8 @@ def run(self): self.execute(self.create_plist, ()) # Make all references to libraries relative - self.execute(self.setRelativeReferencePaths, ()) + self.darwinTracker: DarwinFileTracker = freezer.darwinTracker + self.execute(self.setRelativeReferencePaths, (os.path.abspath(build.build_exe), os.path.abspath(self.binDir))) # Make library references absolute if enabled if self.absolute_reference_path: diff --git a/cx_Freeze/samples/PyQt5_plugins/setup.py b/cx_Freeze/samples/PyQt5_plugins/setup.py new file mode 100644 index 000000000..a740d92a5 --- /dev/null +++ b/cx_Freeze/samples/PyQt5_plugins/setup.py @@ -0,0 +1,94 @@ +import sys, os, site +from cx_Freeze import setup, Executable +from typing import List, Tuple + +PACKAGENAME = "TEST" +INCLUDE_QT = True # whether to include certain PyQT-related files + + + +def getQtPluginIncludes(pluginList: List[str]) -> List[Tuple[str,str]]: + includes = [] + for ppath in pluginList: + includes.append(_getInclude(ppath)) + pass + return includes + +def _getInclude(pluginPath: str) -> Tuple[str,str]: + foundPath = None + + if sys.platform == "darwin": + packagesDirs = [c for c in sys.path if c.find("site-packages")!=-1] + else: + packagesDirs = site.getsitepackages() # search site packages locations to see if we can find required .dll + pass + for pdir in packagesDirs: + testPath = os.path.join(pdir, os.path.join("PyQt5", "Qt", "plugins", pluginPath )) + bname = os.path.basename(pluginPath) + + # print("Checking for {} at {}".format(bname, testPath)) + if os.path.exists(testPath): + foundPath = testPath + # print("DLL Found") + break + pass + if foundPath is None: + print("Error, could not find: {}".format(pluginPath)) + sys.exit(1) + + return (foundPath, pluginPath) + + + +# force the inclusion of certain plugins that cx-freeze cannot find on its own +requiredPlugins = [ + "styles/libqmacstyle.dylib", + "sqldrivers/libqsqlite.dylib", + "printsupport/libcocoaprintersupport.dylib" +] + +if not INCLUDE_QT: requiredPlugins = [] +include_files = [] + +include_files += getQtPluginIncludes(pluginList=requiredPlugins) + +keyFiles = [] + +build_options = {"build_exe":"cx_build/", # subdirectory to do build in + "build_base": "cx_build_dists" # subdirectory to place .app and .dmg packages in + } +# Cause the PyQt5 to be included in the package in the old way (only specifically required files). +# This makes the package *much* smaller. +zip_include_packages = ["PyQt5"] +extraPackages = ["PyQt5.sip"] # force PyQt5.sip to be included. + +if not INCLUDE_QT: + zip_include_packages = [] + extraPackages = [] + + +build_exe_options = { "include_files": include_files, + "zip_include_packages": zip_include_packages, + "packages": extraPackages, + } + +bdist_mac_options = { + "bundle_name": "Test", +} + +bdist_dmg_options = { + "volume_label": PACKAGENAME, +} + +exe = Executable(script = "test_script.py") +exe2 = Executable(script = "test_script2.py") + + +setup( name = "Test application", + author = "[author]", + maintainer = "[maintainer]", + maintainer_email = "[email]", + options = {"build":build_options, "build_exe" : build_exe_options, "bdist_mac": bdist_mac_options, + "bdist_dmg": bdist_dmg_options}, + executables = [exe, exe2] + ) diff --git a/cx_Freeze/samples/PyQt5_plugins/test_script.py b/cx_Freeze/samples/PyQt5_plugins/test_script.py new file mode 100644 index 000000000..0a25e33a4 --- /dev/null +++ b/cx_Freeze/samples/PyQt5_plugins/test_script.py @@ -0,0 +1,9 @@ +# import PyQt5.QtGui +# +# from PyQt5.QtWidgets import QApplication +# +# app = QApplication([]) +# +# app.exec_() + +print("hi") \ No newline at end of file diff --git a/cx_Freeze/samples/PyQt5_plugins/test_script2.py b/cx_Freeze/samples/PyQt5_plugins/test_script2.py new file mode 100644 index 000000000..dddb8602b --- /dev/null +++ b/cx_Freeze/samples/PyQt5_plugins/test_script2.py @@ -0,0 +1,2 @@ + +print("bye") \ No newline at end of file