imphook.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
  1. #-----------------------------------------------------------------------------
  2. # Copyright (c) 2005-2021, PyInstaller Development Team.
  3. #
  4. # Distributed under the terms of the GNU General Public License (version 2
  5. # or later) with exception for distributing the bootloader.
  6. #
  7. # The full license is in the file COPYING.txt, distributed with this software.
  8. #
  9. # SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
  10. #-----------------------------------------------------------------------------
  11. """
  12. Code related to processing of import hooks.
  13. """
  14. import glob
  15. import os.path
  16. import sys
  17. import weakref
  18. from PyInstaller import log as logging
  19. from PyInstaller.building.utils import format_binaries_and_datas
  20. from PyInstaller.compat import expand_path, importlib_load_source
  21. from PyInstaller.depend.imphookapi import PostGraphAPI
  22. from PyInstaller.exceptions import ImportErrorWhenRunningHook
  23. logger = logging.getLogger(__name__)
  24. # Safety check: Hook module names need to be unique. Duplicate names might occur if the cached PyuModuleGraph has an
  25. # issue.
  26. HOOKS_MODULE_NAMES = set()
  27. class ModuleHookCache(dict):
  28. """
  29. Cache of lazily loadable hook script objects.
  30. This cache is implemented as a `dict` subclass mapping from the fully-qualified names of all modules with at
  31. least one hook script to lists of `ModuleHook` instances encapsulating these scripts. As a `dict` subclass,
  32. all cached module names and hook scripts are accessible via standard dictionary operations.
  33. Attributes
  34. ----------
  35. module_graph : ModuleGraph
  36. Current module graph.
  37. _hook_module_name_prefix : str
  38. String prefixing the names of all in-memory modules lazily loaded from cached hook scripts. See also the
  39. `hook_module_name_prefix` parameter passed to the `ModuleHook.__init__()` method.
  40. """
  41. _cache_id_next = 0
  42. """
  43. 0-based identifier unique to the next `ModuleHookCache` to be instantiated.
  44. This identifier is incremented on each instantiation of a new `ModuleHookCache` to isolate in-memory modules of
  45. lazily loaded hook scripts in that cache to the same cache-specific namespace, preventing edge-case collisions
  46. with existing in-memory modules in other caches.
  47. """
  48. def __init__(self, module_graph, hook_dirs):
  49. """
  50. Cache all hook scripts in the passed directories.
  51. **Order of caching is significant** with respect to hooks for the same module, as the values of this
  52. dictionary are lists. Hooks for the same module will be run in the order in which they are cached. Previously
  53. cached hooks are always preserved rather than overidden.
  54. By default, official hooks are cached _before_ user-defined hooks. For modules with both official and
  55. user-defined hooks, this implies that the former take priority over and hence will be loaded _before_ the
  56. latter.
  57. Parameters
  58. ----------
  59. module_graph : ModuleGraph
  60. Current module graph.
  61. hook_dirs : list
  62. List of the absolute or relative paths of all directories containing **hook scripts** (i.e.,
  63. Python scripts with filenames matching `hook-{module_name}.py`, where `{module_name}` is the module
  64. hooked by that script) to be cached.
  65. """
  66. super().__init__()
  67. # To avoid circular references and hence increased memory consumption, a weak rather than strong reference is
  68. # stored to the passed graph. Since this graph is guaranteed to live longer than this cache,
  69. # this is guaranteed to be safe.
  70. self.module_graph = weakref.proxy(module_graph)
  71. # String unique to this cache prefixing the names of all in-memory modules lazily loaded from cached hook
  72. # scripts, privatized for safety.
  73. self._hook_module_name_prefix = '__PyInstaller_hooks_{}_'.format(ModuleHookCache._cache_id_next)
  74. ModuleHookCache._cache_id_next += 1
  75. # Cache all hook scripts in the passed directories.
  76. self._cache_hook_dirs(hook_dirs)
  77. def _cache_hook_dirs(self, hook_dirs):
  78. """
  79. Cache all hook scripts in the passed directories.
  80. Parameters
  81. ----------
  82. hook_dirs : list
  83. List of the absolute or relative paths of all directories containing hook scripts to be cached.
  84. """
  85. for hook_dir in hook_dirs:
  86. # Canonicalize this directory's path and validate its existence.
  87. hook_dir = os.path.abspath(expand_path(hook_dir))
  88. if not os.path.isdir(hook_dir):
  89. raise FileNotFoundError('Hook directory "{}" not found.'.format(hook_dir))
  90. # For each hook script in this directory...
  91. hook_filenames = glob.glob(os.path.join(hook_dir, 'hook-*.py'))
  92. for hook_filename in hook_filenames:
  93. # Fully-qualified name of this hook's corresponding module, constructed by removing the "hook-" prefix
  94. # and ".py" suffix.
  95. module_name = os.path.basename(hook_filename)[5:-3]
  96. if module_name in self:
  97. logger.warning(
  98. "Several hooks defined for module %r. Please take care they do not conflict.", module_name
  99. )
  100. # Lazily loadable hook object.
  101. module_hook = ModuleHook(
  102. module_graph=self.module_graph,
  103. module_name=module_name,
  104. hook_filename=hook_filename,
  105. hook_module_name_prefix=self._hook_module_name_prefix,
  106. )
  107. # Add this hook to this module's list of hooks.
  108. module_hooks = self.setdefault(module_name, [])
  109. module_hooks.append(module_hook)
  110. def remove_modules(self, *module_names):
  111. """
  112. Remove the passed modules and all hook scripts cached for these modules from this cache.
  113. Parameters
  114. ----------
  115. module_names : list
  116. List of all fully-qualified module names to be removed.
  117. """
  118. for module_name in module_names:
  119. # Unload this module's hook script modules from memory. Since these are top-level pure-Python modules cached
  120. # only in the "sys.modules" dictionary, popping these modules from this dictionary suffices to garbage
  121. # collect these modules.
  122. module_hooks = self.get(module_name, [])
  123. for module_hook in module_hooks:
  124. sys.modules.pop(module_hook.hook_module_name, None)
  125. # Remove this module and its hook script objects from this cache.
  126. self.pop(module_name, None)
  127. # Dictionary mapping the names of magic attributes required by the "ModuleHook" class to 2-tuples "(default_type,
  128. # sanitizer_func)", where:
  129. #
  130. # * "default_type" is the type to which that attribute will be initialized when that hook is lazily loaded.
  131. # * "sanitizer_func" is the callable sanitizing the original value of that attribute defined by that hook into a
  132. # safer value consumable by "ModuleHook" callers if any or "None" if the original value requires no sanitization.
  133. #
  134. # To avoid subtleties in the ModuleHook.__getattr__() method, this dictionary is declared as a module rather than a
  135. # class attribute. If declared as a class attribute and then undefined (...for whatever reason), attempting to access
  136. # this attribute from that method would produce infinite recursion.
  137. _MAGIC_MODULE_HOOK_ATTRS = {
  138. # Collections in which order is insignificant. This includes:
  139. #
  140. # * "datas", sanitized from hook-style 2-tuple lists defined by hooks into TOC-style 2-tuple sets consumable by
  141. # "ModuleHook" callers.
  142. # * "binaries", sanitized in the same way.
  143. 'datas': (set, format_binaries_and_datas),
  144. 'binaries': (set, format_binaries_and_datas),
  145. 'excludedimports': (set, None),
  146. # Collections in which order is significant. This includes:
  147. #
  148. # * "hiddenimports", as order of importation is significant. On module importation, hook scripts are loaded and hook
  149. # functions declared by these scripts are called. As these scripts and functions can have side effects dependent
  150. # on module importation order, module importation itself can have side effects dependent on this order!
  151. 'hiddenimports': (list, None),
  152. }
  153. class ModuleHook(object):
  154. """
  155. Cached object encapsulating a lazy loadable hook script.
  156. This object exposes public attributes (e.g., `datas`) of the underlying hook script as attributes of the same
  157. name of this object. On the first access of any such attribute, this hook script is lazily loaded into an
  158. in-memory private module reused on subsequent accesses. These dynamic attributes are referred to as "magic." All
  159. other static attributes of this object (e.g., `hook_module_name`) are referred to as "non-magic."
  160. Attributes (Magic)
  161. ----------
  162. datas : set
  163. Set of `TOC`-style 2-tuples `(target_file, source_file)` for all external non-executable files required by
  164. the module being hooked, converted from the `datas` list of hook-style 2-tuples `(source_dir_or_glob,
  165. target_dir)` defined by this hook script.
  166. binaries : set
  167. Set of `TOC`-style 2-tuples `(target_file, source_file)` for all external executable files required by the
  168. module being hooked, converted from the `binaries` list of hook-style 2-tuples `(source_dir_or_glob,
  169. target_dir)` defined by this hook script.
  170. excludedimports : set
  171. Set of the fully-qualified names of all modules imported by the module being hooked to be ignored rather than
  172. imported from that module, converted from the `excludedimports` list defined by this hook script. These
  173. modules will only be "locally" rather than "globally" ignored. These modules will remain importable from all
  174. modules other than the module being hooked.
  175. hiddenimports : set
  176. Set of the fully-qualified names of all modules imported by the module being hooked that are _not_
  177. automatically detectable by PyInstaller (usually due to being dynamically imported in that module),
  178. converted from the `hiddenimports` list defined by this hook script.
  179. Attributes (Non-magic)
  180. ----------
  181. module_graph : ModuleGraph
  182. Current module graph.
  183. module_name : str
  184. Name of the module hooked by this hook script.
  185. hook_filename : str
  186. Absolute or relative path of this hook script.
  187. hook_module_name : str
  188. Name of the in-memory module of this hook script's interpreted contents.
  189. _hook_module : module
  190. In-memory module of this hook script's interpreted contents, lazily loaded on the first call to the
  191. `_load_hook_module()` method _or_ `None` if this method has yet to be accessed.
  192. """
  193. #-- Magic --
  194. def __init__(self, module_graph, module_name, hook_filename, hook_module_name_prefix):
  195. """
  196. Initialize this metadata.
  197. Parameters
  198. ----------
  199. module_graph : ModuleGraph
  200. Current module graph.
  201. module_name : str
  202. Name of the module hooked by this hook script.
  203. hook_filename : str
  204. Absolute or relative path of this hook script.
  205. hook_module_name_prefix : str
  206. String prefixing the name of the in-memory module for this hook script. To avoid namespace clashes with
  207. similar modules created by other `ModuleHook` objects in other `ModuleHookCache` containers, this string
  208. _must_ be unique to the `ModuleHookCache` container containing this `ModuleHook` object. If this string
  209. is non-unique, an existing in-memory module will be erroneously reused when lazily loading this hook
  210. script, thus erroneously resanitizing previously sanitized hook script attributes (e.g., `datas`) with
  211. the `format_binaries_and_datas()` helper.
  212. """
  213. # Note that the passed module graph is already a weak reference, avoiding circular reference issues. See
  214. # ModuleHookCache.__init__(). TODO: Add a failure message
  215. assert isinstance(module_graph, weakref.ProxyTypes)
  216. self.module_graph = module_graph
  217. self.module_name = module_name
  218. self.hook_filename = hook_filename
  219. # Name of the in-memory module fabricated to refer to this hook script.
  220. self.hook_module_name = hook_module_name_prefix + self.module_name.replace('.', '_')
  221. # Safety check, see above
  222. global HOOKS_MODULE_NAMES
  223. if self.hook_module_name in HOOKS_MODULE_NAMES:
  224. # When self._shallow is true, this class never loads the hook and sets the attributes to empty values
  225. self._shallow = True
  226. else:
  227. self._shallow = False
  228. HOOKS_MODULE_NAMES.add(self.hook_module_name)
  229. # Attributes subsequently defined by the _load_hook_module() method.
  230. self._hook_module = None
  231. def __getattr__(self, attr_name):
  232. """
  233. Get the magic attribute with the passed name (e.g., `datas`) from this lazily loaded hook script if any _or_
  234. raise `AttributeError` otherwise.
  235. This special method is called only for attributes _not_ already defined by this object. This includes
  236. undefined attributes and the first attempt to access magic attributes.
  237. This special method is _not_ called for subsequent attempts to access magic attributes. The first attempt to
  238. access magic attributes defines corresponding instance variables accessible via the `self.__dict__` instance
  239. dictionary (e.g., as `self.datas`) without calling this method. This approach also allows magic attributes to
  240. be deleted from this object _without_ defining the `__delattr__()` special method.
  241. See Also
  242. ----------
  243. Class docstring for supported magic attributes.
  244. """
  245. # If this is a magic attribute, initialize this attribute by lazy loading this hook script and then return
  246. # this attribute. To avoid recursion, the superclass method rather than getattr() is called.
  247. if attr_name in _MAGIC_MODULE_HOOK_ATTRS:
  248. self._load_hook_module()
  249. return super().__getattr__(attr_name)
  250. # Else, this is an undefined attribute. Raise an exception.
  251. else:
  252. raise AttributeError(attr_name)
  253. def __setattr__(self, attr_name, attr_value):
  254. """
  255. Set the attribute with the passed name to the passed value.
  256. If this is a magic attribute, this hook script will be lazily loaded before setting this attribute. Unlike
  257. `__getattr__()`, this special method is called to set _any_ attribute -- including magic, non-magic,
  258. and undefined attributes.
  259. See Also
  260. ----------
  261. Class docstring for supported magic attributes.
  262. """
  263. # If this is a magic attribute, initialize this attribute by lazy loading this hook script before overwriting
  264. # this attribute.
  265. if attr_name in _MAGIC_MODULE_HOOK_ATTRS:
  266. self._load_hook_module()
  267. # Set this attribute to the passed value. To avoid recursion, the superclass method rather than setattr() is
  268. # called.
  269. return super().__setattr__(attr_name, attr_value)
  270. #-- Loading --
  271. def _load_hook_module(self):
  272. """
  273. Lazily load this hook script into an in-memory private module.
  274. This method (and, indeed, this class) preserves all attributes and functions defined by this hook script as
  275. is, ensuring sane behaviour in hook functions _not_ expecting unplanned external modification. Instead,
  276. this method copies public attributes defined by this hook script (e.g., `binaries`) into private attributes
  277. of this object, which the special `__getattr__()` and `__setattr__()` methods safely expose to external
  278. callers. For public attributes _not_ defined by this hook script, the corresponding private attributes will
  279. be assigned sane defaults. For some public attributes defined by this hook script, the corresponding private
  280. attributes will be transformed into objects more readily and safely consumed elsewhere by external callers.
  281. See Also
  282. ----------
  283. Class docstring for supported attributes.
  284. """
  285. # If this hook script module has already been loaded, or we are _shallow, noop.
  286. if self._hook_module is not None or self._shallow:
  287. if self._shallow:
  288. self._hook_module = True # Not None
  289. # Inform the user
  290. logger.debug(
  291. 'Skipping module hook %r from %r because a hook for %s has already been loaded.',
  292. *os.path.split(self.hook_filename)[::-1], self.module_name
  293. )
  294. # Set the default attributes to empty instances of the type.
  295. for attr_name, (attr_type, _) in _MAGIC_MODULE_HOOK_ATTRS.items():
  296. super().__setattr__(attr_name, attr_type())
  297. return
  298. # Load and execute the hook script. Even if mechanisms from the import machinery are used, this does not import
  299. # the hook as the module.
  300. head, tail = os.path.split(self.hook_filename)
  301. logger.info('Loading module hook %r from %r...', tail, head)
  302. try:
  303. self._hook_module = importlib_load_source(self.hook_module_name, self.hook_filename)
  304. except ImportError:
  305. logger.debug("Hook failed with:", exc_info=True)
  306. raise ImportErrorWhenRunningHook(self.hook_module_name, self.hook_filename)
  307. # Copy hook script attributes into magic attributes exposed as instance variables of the current "ModuleHook"
  308. # instance.
  309. for attr_name, (default_type, sanitizer_func) in _MAGIC_MODULE_HOOK_ATTRS.items():
  310. # Unsanitized value of this attribute.
  311. attr_value = getattr(self._hook_module, attr_name, None)
  312. # If this attribute is undefined, expose a sane default instead.
  313. if attr_value is None:
  314. attr_value = default_type()
  315. # Else if this attribute requires sanitization, do so.
  316. elif sanitizer_func is not None:
  317. attr_value = sanitizer_func(attr_value)
  318. # Else, expose the unsanitized value of this attribute.
  319. # Expose this attribute as an instance variable of the same name.
  320. setattr(self, attr_name, attr_value)
  321. #-- Hooks --
  322. def post_graph(self, analysis):
  323. """
  324. Call the **post-graph hook** (i.e., `hook()` function) defined by this hook script, if any.
  325. Parameters
  326. ----------
  327. analysis: build_main.Analysis
  328. Analysis that calls the hook
  329. This method is intended to be called _after_ the module graph for this application is constructed.
  330. """
  331. # Lazily load this hook script into an in-memory module.
  332. self._load_hook_module()
  333. # Call this hook script's hook() function, which modifies attributes accessed by subsequent methods and hence
  334. # must be called first.
  335. self._process_hook_func(analysis)
  336. # Order is insignificant here.
  337. self._process_hidden_imports()
  338. self._process_excluded_imports()
  339. def _process_hook_func(self, analysis):
  340. """
  341. Call this hook's `hook()` function if defined.
  342. Parameters
  343. ----------
  344. analysis: build_main.Analysis
  345. Analysis that calls the hook
  346. """
  347. # If this hook script defines no hook() function, noop.
  348. if not hasattr(self._hook_module, 'hook'):
  349. return
  350. # Call this hook() function.
  351. hook_api = PostGraphAPI(module_name=self.module_name, module_graph=self.module_graph, analysis=analysis)
  352. try:
  353. self._hook_module.hook(hook_api)
  354. except ImportError:
  355. logger.debug("Hook failed with:", exc_info=True)
  356. raise ImportErrorWhenRunningHook(self.hook_module_name, self.hook_filename)
  357. # Update all magic attributes modified by the prior call.
  358. self.datas.update(set(hook_api._added_datas))
  359. self.binaries.update(set(hook_api._added_binaries))
  360. self.hiddenimports.extend(hook_api._added_imports)
  361. # FIXME: Deleted imports should be appended to self.excludedimports rather than handled here. However, see the
  362. # _process_excluded_imports() FIXME below for a sensible alternative.
  363. for deleted_module_name in hook_api._deleted_imports:
  364. # Remove the graph link between the hooked module and item. This removes the 'item' node from the graph if
  365. # no other links go to it (no other modules import it)
  366. self.module_graph.removeReference(hook_api.node, deleted_module_name)
  367. def _process_hidden_imports(self):
  368. """
  369. Add all imports listed in this hook script's `hiddenimports` attribute to the module graph as if directly
  370. imported by this hooked module.
  371. These imports are typically _not_ implicitly detectable by PyInstaller and hence must be explicitly defined
  372. by hook scripts.
  373. """
  374. # For each hidden import required by the module being hooked...
  375. for import_module_name in self.hiddenimports:
  376. try:
  377. # Graph node for this module. Do not implicitly create namespace packages for non-existent packages.
  378. caller = self.module_graph.find_node(self.module_name, create_nspkg=False)
  379. # Manually import this hidden import from this module.
  380. self.module_graph.import_hook(import_module_name, caller)
  381. # If this hidden import is unimportable, print a non-fatal warning. Hidden imports often become
  382. # desynchronized from upstream packages and hence are only "soft" recommendations.
  383. except ImportError:
  384. logger.warning('Hidden import "%s" not found!', import_module_name)
  385. # FIXME: This is pretty... intense. Attempting to cleanly "undo" prior module graph operations is a recipe for
  386. # subtle edge cases and difficult-to-debug issues. It would be both safer and simpler to prevent these
  387. # imports from being added to the graph in the first place. To do so:
  388. #
  389. # * Remove the _process_excluded_imports() method below.
  390. # * Remove the PostGraphAPI.del_imports() method, which cannot reasonably be supported by the following solution,
  391. # appears to be currently broken, and (in any case) is not called anywhere in the PyInstaller codebase.
  392. # * Override the ModuleGraph._safe_import_hook() superclass method with a new PyiModuleGraph._safe_import_hook()
  393. # subclass method resembling:
  394. #
  395. # def _safe_import_hook(
  396. # self, target_module_name, source_module, fromlist,
  397. # level=DEFAULT_IMPORT_LEVEL, attr=None):
  398. #
  399. # if source_module.identifier in self._module_hook_cache:
  400. # for module_hook in self._module_hook_cache[
  401. # source_module.identifier]:
  402. # if target_module_name in module_hook.excludedimports:
  403. # return []
  404. #
  405. # return super()._safe_import_hook(
  406. # target_module_name, source_module, fromlist,
  407. # level=level, attr=attr)
  408. def _process_excluded_imports(self):
  409. """
  410. 'excludedimports' is a list of Python module names that PyInstaller should not detect as dependency of this
  411. module name.
  412. So remove all import-edges from the current module (and it's submodules) to the given `excludedimports` (end
  413. their submodules).
  414. """
  415. def find_all_package_nodes(name):
  416. mods = [name]
  417. name += '.'
  418. for subnode in self.module_graph.nodes():
  419. if subnode.identifier.startswith(name):
  420. mods.append(subnode.identifier)
  421. return mods
  422. # If this hook excludes no imports, noop.
  423. if not self.excludedimports:
  424. return
  425. # Collect all submodules of this module.
  426. hooked_mods = find_all_package_nodes(self.module_name)
  427. # Collect all dependencies and their submodules
  428. # TODO: Optimize this by using a pattern and walking the graph only once.
  429. for item in set(self.excludedimports):
  430. excluded_node = self.module_graph.find_node(item, create_nspkg=False)
  431. if excluded_node is None:
  432. logger.info("Import to be excluded not found: %r", item)
  433. continue
  434. imports_to_remove = set(find_all_package_nodes(item))
  435. # Remove references between module nodes, as though they would not be imported from 'name'. Note: Doing this
  436. # in a nested loop is less efficient than collecting all import to remove first, but log messages are easier
  437. # to understand since related to the "Excluding ..." message above.
  438. for src in hooked_mods:
  439. # modules, this `src` does import
  440. references = set(node.identifier for node in self.module_graph.outgoing(src))
  441. # Remove all of these imports which are also in "imports_to_remove".
  442. for dest in imports_to_remove & references:
  443. self.module_graph.removeReference(src, dest)
  444. logger.debug("Excluding import of %s from module %s", dest, src)
  445. class AdditionalFilesCache(object):
  446. """
  447. Cache for storing what binaries and datas were pushed by what modules when import hooks were processed.
  448. """
  449. def __init__(self):
  450. self._binaries = {}
  451. self._datas = {}
  452. def add(self, modname, binaries, datas):
  453. self._binaries.setdefault(modname, [])
  454. self._binaries[modname].extend(binaries or [])
  455. self._datas.setdefault(modname, [])
  456. self._datas[modname].extend(datas or [])
  457. def __contains__(self, name):
  458. return name in self._binaries or name in self._datas
  459. def binaries(self, modname):
  460. """
  461. Return list of binaries for given module name.
  462. """
  463. return self._binaries[modname]
  464. def datas(self, modname):
  465. """
  466. Return list of datas for given module name.
  467. """
  468. return self._datas[modname]