1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 """Python modules manipulation utility functions.
20
21 :type PY_SOURCE_EXTS: tuple(str)
22 :var PY_SOURCE_EXTS: list of possible python source file extension
23
24 :type STD_LIB_DIR: str
25 :var STD_LIB_DIR: directory where standard modules are located
26
27 :type BUILTIN_MODULES: dict
28 :var BUILTIN_MODULES: dictionary with builtin module names as key
29 """
30
31 __docformat__ = "restructuredtext en"
32
33 import sys
34 import os
35 from os.path import (splitext, join, abspath, isdir, dirname, exists,
36 basename, expanduser, normcase, realpath)
37 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
38 from distutils.sysconfig import get_config_var, get_python_lib, get_python_version
39 from distutils.errors import DistutilsPlatformError
40
41 from six import PY3
42 from six.moves import map, range
43
44 try:
45 import zipimport
46 except ImportError:
47 zipimport = None
48
49 ZIPFILE = object()
50
51 from logilab.common import STD_BLACKLIST, _handle_blacklist
52
53
54
55
56
57
58
59 if sys.platform.startswith('win'):
60 PY_SOURCE_EXTS = ('py', 'pyw')
61 PY_COMPILED_EXTS = ('dll', 'pyd')
62 else:
63 PY_SOURCE_EXTS = ('py',)
64 PY_COMPILED_EXTS = ('so',)
65
66 try:
67 STD_LIB_DIR = get_python_lib(standard_lib=True)
68
69
70 except DistutilsPlatformError:
71 STD_LIB_DIR = '//'
72
73 EXT_LIB_DIR = get_python_lib()
74
75 BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
76
77
79 """exception raised when we are not able to get a python
80 source file for a precompiled file
81 """
82
85 self.module = module
86 self.obj = obj
87 self._imported = None
88
90 if self._imported is None:
91 self._imported = getattr(load_module_from_name(self.module),
92 self.obj)
93 return self._imported
94
96 try:
97 return super(LazyObject, self).__getattribute__(attr)
98 except AttributeError as ex:
99 return getattr(self._getobj(), attr)
100
102 return self._getobj()(*args, **kwargs)
103
104
106 """Load a Python module from its name.
107
108 :type dotted_name: str
109 :param dotted_name: python name of a module or package
110
111 :type path: list or None
112 :param path:
113 optional list of path where the module or package should be
114 searched (use sys.path if nothing or None is given)
115
116 :type use_sys: bool
117 :param use_sys:
118 boolean indicating whether the sys.modules dictionary should be
119 used or not
120
121
122 :raise ImportError: if the module or package is not found
123
124 :rtype: module
125 :return: the loaded module
126 """
127 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
128
129
131 """Load a python module from its splitted name.
132
133 :type parts: list(str) or tuple(str)
134 :param parts:
135 python name of a module or package splitted on '.'
136
137 :type path: list or None
138 :param path:
139 optional list of path where the module or package should be
140 searched (use sys.path if nothing or None is given)
141
142 :type use_sys: bool
143 :param use_sys:
144 boolean indicating whether the sys.modules dictionary should be used or not
145
146 :raise ImportError: if the module or package is not found
147
148 :rtype: module
149 :return: the loaded module
150 """
151 if use_sys:
152 try:
153 return sys.modules['.'.join(parts)]
154 except KeyError:
155 pass
156 modpath = []
157 prevmodule = None
158 for part in parts:
159 modpath.append(part)
160 curname = '.'.join(modpath)
161 module = None
162 if len(modpath) != len(parts):
163
164 module = sys.modules.get(curname)
165 elif use_sys:
166
167 module = sys.modules.get(curname)
168 if module is None:
169 mp_file, mp_filename, mp_desc = find_module(part, path)
170 try:
171 module = load_module(curname, mp_file, mp_filename, mp_desc)
172 finally:
173 if mp_file is not None:
174 mp_file.close()
175 if prevmodule:
176 setattr(prevmodule, part, module)
177 _file = getattr(module, '__file__', '')
178 prevmodule = module
179 if not _file and _is_namespace(curname):
180 continue
181 if not _file and len(modpath) != len(parts):
182 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) )
183 path = [dirname( _file )]
184 return module
185
186
188 """Load a Python module from it's path.
189
190 :type filepath: str
191 :param filepath: path to the python module or package
192
193 :type path: list or None
194 :param path:
195 optional list of path where the module or package should be
196 searched (use sys.path if nothing or None is given)
197
198 :type use_sys: bool
199 :param use_sys:
200 boolean indicating whether the sys.modules dictionary should be
201 used or not
202
203
204 :raise ImportError: if the module or package is not found
205
206 :rtype: module
207 :return: the loaded module
208 """
209 modpath = modpath_from_file(filepath, extrapath)
210 return load_module_from_modpath(modpath, path, use_sys)
211
212
214 """check there are some __init__.py all along the way"""
215 modpath = []
216 for part in mod_path:
217 modpath.append(part)
218 path = join(path, part)
219 if not _is_namespace('.'.join(modpath)) and not _has_init(path):
220 return False
221 return True
222
223
225 return realpath(expanduser(path))
226
227
229 if PY3:
230 return filename
231 else:
232 if filename.endswith(".pyc"):
233 return filename[:-1]
234 return filename
235
236
238 """given a file path return the corresponding splitted module's name
239 (i.e name of a module or package splitted on '.')
240
241 :type filename: str
242 :param filename: file's path for which we want the module's name
243
244 :type extrapath: dict
245 :param extrapath:
246 optional extra search path, with path as key and package name for the path
247 as value. This is usually useful to handle package splitted in multiple
248 directories using __path__ trick.
249
250
251 :raise ImportError:
252 if the corresponding module's name has not been found
253
254 :rtype: list(str)
255 :return: the corresponding splitted module's name
256 """
257 filename = _path_from_filename(filename)
258 filename = _canonicalize_path(filename)
259 base = os.path.splitext(filename)[0]
260
261 if extrapath is not None:
262 for path_ in map(_canonicalize_path, extrapath):
263 path = abspath(path_)
264 if path and normcase(base[:len(path)]) == normcase(path):
265 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
266 if pkg]
267 if _check_init(path, submodpath[:-1]):
268 return extrapath[path_].split('.') + submodpath
269
270 for path in map(_canonicalize_path, sys.path):
271 if path and normcase(base).startswith(path):
272 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
273 if _check_init(path, modpath[:-1]):
274 return modpath
275
276 raise ImportError('Unable to find module for %s in %s' % (
277 filename, ', \n'.join(sys.path)))
278
279
281 """given a mod path (i.e. splitted module / package name), return the
282 corresponding file, giving priority to source file over precompiled
283 file if it exists
284
285 :type modpath: list or tuple
286 :param modpath:
287 splitted module's name (i.e name of a module or package splitted
288 on '.')
289 (this means explicit relative imports that start with dots have
290 empty strings in this list!)
291
292 :type path: list or None
293 :param path:
294 optional list of path where the module or package should be
295 searched (use sys.path if nothing or None is given)
296
297 :type context_file: str or None
298 :param context_file:
299 context file to consider, necessary if the identifier has been
300 introduced using a relative import unresolvable in the actual
301 context (i.e. modutils)
302
303 :raise ImportError: if there is no such module in the directory
304
305 :rtype: str or None
306 :return:
307 the path to the module's file or None if it's an integrated
308 builtin module such as 'sys'
309 """
310 if context_file is not None:
311 context = dirname(context_file)
312 else:
313 context = context_file
314 if modpath[0] == 'xml':
315
316 try:
317 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
318 except ImportError:
319 return _file_from_modpath(modpath, path, context)
320 elif modpath == ['os', 'path']:
321
322 return os.path.__file__
323 return _file_from_modpath(modpath, path, context)
324
325
326
328 """given a dotted name return the module part of the name :
329
330 >>> get_module_part('logilab.common.modutils.get_module_part')
331 'logilab.common.modutils'
332
333 :type dotted_name: str
334 :param dotted_name: full name of the identifier we are interested in
335
336 :type context_file: str or None
337 :param context_file:
338 context file to consider, necessary if the identifier has been
339 introduced using a relative import unresolvable in the actual
340 context (i.e. modutils)
341
342
343 :raise ImportError: if there is no such module in the directory
344
345 :rtype: str or None
346 :return:
347 the module part of the name or None if we have not been able at
348 all to import the given name
349
350 XXX: deprecated, since it doesn't handle package precedence over module
351 (see #10066)
352 """
353
354 if dotted_name.startswith('os.path'):
355 return 'os.path'
356 parts = dotted_name.split('.')
357 if context_file is not None:
358
359
360 if parts[0] in BUILTIN_MODULES:
361 if len(parts) > 2:
362 raise ImportError(dotted_name)
363 return parts[0]
364
365 path = None
366 starti = 0
367 if parts[0] == '':
368 assert context_file is not None, \
369 'explicit relative import, but no context_file?'
370 path = []
371 starti = 1
372 while parts[starti] == '':
373 starti += 1
374 context_file = dirname(context_file)
375 for i in range(starti, len(parts)):
376 try:
377 file_from_modpath(parts[starti:i+1],
378 path=path, context_file=context_file)
379 except ImportError:
380 if not i >= max(1, len(parts) - 2):
381 raise
382 return '.'.join(parts[:i])
383 return dotted_name
384
385
387 """given a package directory return a list of all available python
388 modules in the package and its subpackages
389
390 :type package: str
391 :param package: the python name for the package
392
393 :type src_directory: str
394 :param src_directory:
395 path of the directory corresponding to the package
396
397 :type blacklist: list or tuple
398 :param blacklist:
399 optional list of files or directory to ignore, default to
400 the value of `logilab.common.STD_BLACKLIST`
401
402 :rtype: list
403 :return:
404 the list of all available python modules in the package and its
405 subpackages
406 """
407 modules = []
408 for directory, dirnames, filenames in os.walk(src_directory):
409 _handle_blacklist(blacklist, dirnames, filenames)
410
411 if not '__init__.py' in filenames:
412 dirnames[:] = ()
413 continue
414 if directory != src_directory:
415 dir_package = directory[len(src_directory):].replace(os.sep, '.')
416 modules.append(package + dir_package)
417 for filename in filenames:
418 if _is_python_file(filename) and filename != '__init__.py':
419 src = join(directory, filename)
420 module = package + src[len(src_directory):-3]
421 modules.append(module.replace(os.sep, '.'))
422 return modules
423
424
425
427 """given a package directory return a list of all available python
428 module's files in the package and its subpackages
429
430 :type src_directory: str
431 :param src_directory:
432 path of the directory corresponding to the package
433
434 :type blacklist: list or tuple
435 :param blacklist:
436 optional list of files or directory to ignore, default to the value of
437 `logilab.common.STD_BLACKLIST`
438
439 :rtype: list
440 :return:
441 the list of all available python module's files in the package and
442 its subpackages
443 """
444 files = []
445 for directory, dirnames, filenames in os.walk(src_directory):
446 _handle_blacklist(blacklist, dirnames, filenames)
447
448 if not '__init__.py' in filenames:
449 dirnames[:] = ()
450 continue
451 for filename in filenames:
452 if _is_python_file(filename):
453 src = join(directory, filename)
454 files.append(src)
455 return files
456
457
459 """given a python module's file name return the matching source file
460 name (the filename will be returned identically if it's a already an
461 absolute path to a python source file...)
462
463 :type filename: str
464 :param filename: python module's file name
465
466
467 :raise NoSourceFile: if no source file exists on the file system
468
469 :rtype: str
470 :return: the absolute path of the source file if it exists
471 """
472 base, orig_ext = splitext(abspath(filename))
473 for ext in PY_SOURCE_EXTS:
474 source_path = '%s.%s' % (base, ext)
475 if exists(source_path):
476 return source_path
477 if include_no_ext and not orig_ext and exists(base):
478 return base
479 raise NoSourceFile(filename)
480
481
483 """remove submodules of `directories` from `sys.modules`"""
484 cleaned = []
485 for modname, module in list(sys.modules.items()):
486 modfile = getattr(module, '__file__', None)
487 if modfile:
488 for directory in directories:
489 if modfile.startswith(directory):
490 cleaned.append(modname)
491 del sys.modules[modname]
492 break
493 return cleaned
494
495
497 """
498 rtype: bool
499 return: True if the filename is a python source file
500 """
501 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
502
503
505 """try to guess if a module is a standard python module (by default,
506 see `std_path` parameter's description)
507
508 :type modname: str
509 :param modname: name of the module we are interested in
510
511 :type std_path: list(str) or tuple(str)
512 :param std_path: list of path considered as standard
513
514
515 :rtype: bool
516 :return:
517 true if the module:
518 - is located on the path listed in one of the directory in `std_path`
519 - is a built-in module
520
521 Note: this function is known to return wrong values when inside virtualenv.
522 See https://www.logilab.org/ticket/294756.
523 """
524 modname = modname.split('.')[0]
525 try:
526 filename = file_from_modpath([modname])
527 except ImportError as ex:
528
529
530 return False
531
532
533 if filename is None:
534
535 return not _is_namespace(modname)
536 filename = abspath(filename)
537 if filename.startswith(EXT_LIB_DIR):
538 return False
539 for path in std_path:
540 if filename.startswith(abspath(path)):
541 return True
542 return False
543
544
545
547 """return true if the given module name is relative to the given
548 file name
549
550 :type modname: str
551 :param modname: name of the module we are interested in
552
553 :type from_file: str
554 :param from_file:
555 path of the module from which modname has been imported
556
557 :rtype: bool
558 :return:
559 true if the module has been imported relatively to `from_file`
560 """
561 if not isdir(from_file):
562 from_file = dirname(from_file)
563 if from_file in sys.path:
564 return False
565 try:
566 find_module(modname.split('.')[0], [from_file])
567 return True
568 except ImportError:
569 return False
570
571
572
573
575 """given a mod path (i.e. splitted module / package name), return the
576 corresponding file
577
578 this function is used internally, see `file_from_modpath`'s
579 documentation for more information
580 """
581 assert len(modpath) > 0
582 if context is not None:
583 try:
584 mtype, mp_filename = _module_file(modpath, [context])
585 except ImportError:
586 mtype, mp_filename = _module_file(modpath, path)
587 else:
588 mtype, mp_filename = _module_file(modpath, path)
589 if mtype == PY_COMPILED:
590 try:
591 return get_source_file(mp_filename)
592 except NoSourceFile:
593 return mp_filename
594 elif mtype == C_BUILTIN:
595
596 return None
597 elif mtype == PKG_DIRECTORY:
598 mp_filename = _has_init(mp_filename)
599 return mp_filename
600
602 for filepath, importer in pic.items():
603 if importer is not None:
604 if importer.find_module(modpath[0]):
605 if not importer.find_module('/'.join(modpath)):
606 raise ImportError('No module named %s in %s/%s' % (
607 '.'.join(modpath[1:]), filepath, modpath))
608 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
609 raise ImportError('No module named %s' % '.'.join(modpath))
610
611 try:
612 import pkg_resources
613 except ImportError:
614 pkg_resources = None
615
616
620
621
623 """get a module type / file path
624
625 :type modpath: list or tuple
626 :param modpath:
627 splitted module's name (i.e name of a module or package splitted
628 on '.'), with leading empty strings for explicit relative import
629
630 :type path: list or None
631 :param path:
632 optional list of path where the module or package should be
633 searched (use sys.path if nothing or None is given)
634
635
636 :rtype: tuple(int, str)
637 :return: the module type flag and the file path for a module
638 """
639
640 try:
641 pic = sys.path_importer_cache
642 _path = (path is None and sys.path or path)
643 for __path in _path:
644 if not __path in pic:
645 try:
646 pic[__path] = zipimport.zipimporter(__path)
647 except zipimport.ZipImportError:
648 pic[__path] = None
649 checkeggs = True
650 except AttributeError:
651 checkeggs = False
652
653 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules):
654
655
656 module = sys.modules[modpath.pop(0)]
657
658
659 path = list(module.__path__)
660 if not modpath:
661 return C_BUILTIN, None
662 imported = []
663 while modpath:
664 modname = modpath[0]
665
666
667
668
669
670
671
672
673
674 try:
675 _, mp_filename, mp_desc = find_module(modname, path)
676 except ImportError:
677 if checkeggs:
678 return _search_zip(modpath, pic)[:2]
679 raise
680 else:
681 if checkeggs and mp_filename:
682 fullabspath = [abspath(x) for x in _path]
683 try:
684 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
685 emtype, emp_filename, zippath = _search_zip(modpath, pic)
686 if pathindex > _path.index(zippath):
687
688 return emtype, emp_filename
689 except ValueError:
690
691 pass
692 except ImportError:
693 pass
694 checkeggs = False
695 imported.append(modpath.pop(0))
696 mtype = mp_desc[2]
697 if modpath:
698 if mtype != PKG_DIRECTORY:
699 raise ImportError('No module %s in %s' % ('.'.join(modpath),
700 '.'.join(imported)))
701
702
703 try:
704 with open(join(mp_filename, '__init__.py')) as stream:
705 data = stream.read(4096)
706 except IOError:
707 path = [mp_filename]
708 else:
709 if 'pkgutil' in data and 'extend_path' in data:
710
711
712 path = [join(p, *imported) for p in sys.path
713 if isdir(join(p, *imported))]
714 else:
715 path = [mp_filename]
716 return mtype, mp_filename
717
719 """return true if the given filename should be considered as a python file
720
721 .pyc and .pyo are ignored
722 """
723 for ext in ('.py', '.so', '.pyd', '.pyw'):
724 if filename.endswith(ext):
725 return True
726 return False
727
728
730 """if the given directory has a valid __init__ file, return its path,
731 else return None
732 """
733 mod_or_pack = join(directory, '__init__')
734 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
735 if exists(mod_or_pack + '.' + ext):
736 return mod_or_pack + '.' + ext
737 return None
738