]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | """ |
2 | :mod:`jedi.inference.imports` is here to resolve import statements and return | |
3 | the modules/classes/functions/whatever, which they stand for. However there's | |
4 | not any actual importing done. This module is about finding modules in the | |
5 | filesystem. This can be quite tricky sometimes, because Python imports are not | |
6 | always that simple. | |
7 | ||
8 | This module also supports import autocompletion, which means to complete | |
9 | statements like ``from datetim`` (cursor at the end would return ``datetime``). | |
10 | """ | |
11 | import os | |
12 | from pathlib import Path | |
13 | ||
14 | from parso.python import tree | |
15 | from parso.tree import search_ancestor | |
16 | ||
17 | from jedi import debug | |
18 | from jedi import settings | |
19 | from jedi.file_io import FolderIO | |
20 | from jedi.parser_utils import get_cached_code_lines | |
21 | from jedi.inference import sys_path | |
22 | from jedi.inference import helpers | |
23 | from jedi.inference import compiled | |
24 | from jedi.inference import analysis | |
25 | from jedi.inference.utils import unite | |
26 | from jedi.inference.cache import inference_state_method_cache | |
27 | from jedi.inference.names import ImportName, SubModuleName | |
28 | from jedi.inference.base_value import ValueSet, NO_VALUES | |
29 | from jedi.inference.gradual.typeshed import import_module_decorator, \ | |
30 | create_stub_module, parse_stub_module | |
31 | from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo | |
32 | from jedi.plugins import plugin_manager | |
33 | ||
34 | ||
35 | class ModuleCache: | |
36 | def __init__(self): | |
37 | self._name_cache = {} | |
38 | ||
39 | def add(self, string_names, value_set): | |
40 | if string_names is not None: | |
41 | self._name_cache[string_names] = value_set | |
42 | ||
43 | def get(self, string_names): | |
44 | return self._name_cache.get(string_names) | |
45 | ||
46 | ||
47 | # This memoization is needed, because otherwise we will infinitely loop on | |
48 | # certain imports. | |
49 | @inference_state_method_cache(default=NO_VALUES) | |
50 | def infer_import(context, tree_name): | |
51 | module_context = context.get_root_context() | |
52 | from_import_name, import_path, level, values = \ | |
53 | _prepare_infer_import(module_context, tree_name) | |
54 | if values: | |
55 | ||
56 | if from_import_name is not None: | |
57 | values = values.py__getattribute__( | |
58 | from_import_name, | |
59 | name_context=context, | |
60 | analysis_errors=False | |
61 | ) | |
62 | ||
63 | if not values: | |
64 | path = import_path + (from_import_name,) | |
65 | importer = Importer(context.inference_state, path, module_context, level) | |
66 | values = importer.follow() | |
67 | debug.dbg('after import: %s', values) | |
68 | return values | |
69 | ||
70 | ||
71 | @inference_state_method_cache(default=[]) | |
72 | def goto_import(context, tree_name): | |
73 | module_context = context.get_root_context() | |
74 | from_import_name, import_path, level, values = \ | |
75 | _prepare_infer_import(module_context, tree_name) | |
76 | if not values: | |
77 | return [] | |
78 | ||
79 | if from_import_name is not None: | |
80 | names = unite([ | |
81 | c.goto( | |
82 | from_import_name, | |
83 | name_context=context, | |
84 | analysis_errors=False | |
85 | ) for c in values | |
86 | ]) | |
87 | # Avoid recursion on the same names. | |
88 | if names and not any(n.tree_name is tree_name for n in names): | |
89 | return names | |
90 | ||
91 | path = import_path + (from_import_name,) | |
92 | importer = Importer(context.inference_state, path, module_context, level) | |
93 | values = importer.follow() | |
94 | return set(s.name for s in values) | |
95 | ||
96 | ||
97 | def _prepare_infer_import(module_context, tree_name): | |
98 | import_node = search_ancestor(tree_name, 'import_name', 'import_from') | |
99 | import_path = import_node.get_path_for_name(tree_name) | |
100 | from_import_name = None | |
101 | try: | |
102 | from_names = import_node.get_from_names() | |
103 | except AttributeError: | |
104 | # Is an import_name | |
105 | pass | |
106 | else: | |
107 | if len(from_names) + 1 == len(import_path): | |
108 | # We have to fetch the from_names part first and then check | |
109 | # if from_names exists in the modules. | |
110 | from_import_name = import_path[-1] | |
111 | import_path = from_names | |
112 | ||
113 | importer = Importer(module_context.inference_state, tuple(import_path), | |
114 | module_context, import_node.level) | |
115 | ||
116 | return from_import_name, tuple(import_path), import_node.level, importer.follow() | |
117 | ||
118 | ||
119 | def _add_error(value, name, message): | |
120 | if hasattr(name, 'parent') and value is not None: | |
121 | analysis.add(value, 'import-error', name, message) | |
122 | else: | |
123 | debug.warning('ImportError without origin: ' + message) | |
124 | ||
125 | ||
126 | def _level_to_base_import_path(project_path, directory, level): | |
127 | """ | |
128 | In case the level is outside of the currently known package (something like | |
129 | import .....foo), we can still try our best to help the user for | |
130 | completions. | |
131 | """ | |
132 | for i in range(level - 1): | |
133 | old = directory | |
134 | directory = os.path.dirname(directory) | |
135 | if old == directory: | |
136 | return None, None | |
137 | ||
138 | d = directory | |
139 | level_import_paths = [] | |
140 | # Now that we are on the level that the user wants to be, calculate the | |
141 | # import path for it. | |
142 | while True: | |
143 | if d == project_path: | |
144 | return level_import_paths, d | |
145 | dir_name = os.path.basename(d) | |
146 | if dir_name: | |
147 | level_import_paths.insert(0, dir_name) | |
148 | d = os.path.dirname(d) | |
149 | else: | |
150 | return None, directory | |
151 | ||
152 | ||
153 | class Importer: | |
154 | def __init__(self, inference_state, import_path, module_context, level=0): | |
155 | """ | |
156 | An implementation similar to ``__import__``. Use `follow` | |
157 | to actually follow the imports. | |
158 | ||
159 | *level* specifies whether to use absolute or relative imports. 0 (the | |
160 | default) means only perform absolute imports. Positive values for level | |
161 | indicate the number of parent directories to search relative to the | |
162 | directory of the module calling ``__import__()`` (see PEP 328 for the | |
163 | details). | |
164 | ||
165 | :param import_path: List of namespaces (strings or Names). | |
166 | """ | |
167 | debug.speed('import %s %s' % (import_path, module_context)) | |
168 | self._inference_state = inference_state | |
169 | self.level = level | |
170 | self._module_context = module_context | |
171 | ||
172 | self._fixed_sys_path = None | |
173 | self._infer_possible = True | |
174 | if level: | |
175 | base = module_context.get_value().py__package__() | |
176 | # We need to care for two cases, the first one is if it's a valid | |
177 | # Python import. This import has a properly defined module name | |
178 | # chain like `foo.bar.baz` and an import in baz is made for | |
179 | # `..lala.` It can then resolve to `foo.bar.lala`. | |
180 | # The else here is a heuristic for all other cases, if for example | |
181 | # in `foo` you search for `...bar`, it's obviously out of scope. | |
182 | # However since Jedi tries to just do it's best, we help the user | |
183 | # here, because he might have specified something wrong in his | |
184 | # project. | |
185 | if level <= len(base): | |
186 | # Here we basically rewrite the level to 0. | |
187 | base = tuple(base) | |
188 | if level > 1: | |
189 | base = base[:-level + 1] | |
190 | import_path = base + tuple(import_path) | |
191 | else: | |
192 | path = module_context.py__file__() | |
193 | project_path = self._inference_state.project.path | |
194 | import_path = list(import_path) | |
195 | if path is None: | |
196 | # If no path is defined, our best guess is that the current | |
197 | # file is edited by a user on the current working | |
198 | # directory. We need to add an initial path, because it | |
199 | # will get removed as the name of the current file. | |
200 | directory = project_path | |
201 | else: | |
202 | directory = os.path.dirname(path) | |
203 | ||
204 | base_import_path, base_directory = _level_to_base_import_path( | |
205 | project_path, directory, level, | |
206 | ) | |
207 | if base_directory is None: | |
208 | # Everything is lost, the relative import does point | |
209 | # somewhere out of the filesystem. | |
210 | self._infer_possible = False | |
211 | else: | |
212 | self._fixed_sys_path = [base_directory] | |
213 | ||
214 | if base_import_path is None: | |
215 | if import_path: | |
216 | _add_error( | |
217 | module_context, import_path[0], | |
218 | message='Attempted relative import beyond top-level package.' | |
219 | ) | |
220 | else: | |
221 | import_path = base_import_path + import_path | |
222 | self.import_path = import_path | |
223 | ||
224 | @property | |
225 | def _str_import_path(self): | |
226 | """Returns the import path as pure strings instead of `Name`.""" | |
227 | return tuple( | |
228 | name.value if isinstance(name, tree.Name) else name | |
229 | for name in self.import_path | |
230 | ) | |
231 | ||
232 | def _sys_path_with_modifications(self, is_completion): | |
233 | if self._fixed_sys_path is not None: | |
234 | return self._fixed_sys_path | |
235 | ||
236 | return ( | |
237 | # For import completions we don't want to see init paths, but for | |
238 | # inference we want to show the user as much as possible. | |
239 | # See GH #1446. | |
240 | self._inference_state.get_sys_path(add_init_paths=not is_completion) | |
241 | + [ | |
242 | str(p) for p | |
243 | in sys_path.check_sys_path_modifications(self._module_context) | |
244 | ] | |
245 | ) | |
246 | ||
247 | def follow(self): | |
248 | if not self.import_path: | |
249 | if self._fixed_sys_path: | |
250 | # This is a bit of a special case, that maybe should be | |
251 | # revisited. If the project path is wrong or the user uses | |
252 | # relative imports the wrong way, we might end up here, where | |
253 | # the `fixed_sys_path == project.path` in that case we kind of | |
254 | # use the project.path.parent directory as our path. This is | |
255 | # usually not a problem, except if imports in other places are | |
256 | # using the same names. Example: | |
257 | # | |
258 | # foo/ < #1 | |
259 | # - setup.py | |
260 | # - foo/ < #2 | |
261 | # - __init__.py | |
262 | # - foo.py < #3 | |
263 | # | |
264 | # If the top foo is our project folder and somebody uses | |
265 | # `from . import foo` in `setup.py`, it will resolve to foo #2, | |
266 | # which means that the import for foo.foo is cached as | |
267 | # `__init__.py` (#2) and not as `foo.py` (#3). This is usually | |
268 | # not an issue, because this case is probably pretty rare, but | |
269 | # might be an issue for some people. | |
270 | # | |
271 | # However for most normal cases where we work with different | |
272 | # file names, this code path hits where we basically change the | |
273 | # project path to an ancestor of project path. | |
274 | from jedi.inference.value.namespace import ImplicitNamespaceValue | |
275 | import_path = (os.path.basename(self._fixed_sys_path[0]),) | |
276 | ns = ImplicitNamespaceValue( | |
277 | self._inference_state, | |
278 | string_names=import_path, | |
279 | paths=self._fixed_sys_path, | |
280 | ) | |
281 | return ValueSet({ns}) | |
282 | return NO_VALUES | |
283 | if not self._infer_possible: | |
284 | return NO_VALUES | |
285 | ||
286 | # Check caches first | |
287 | from_cache = self._inference_state.stub_module_cache.get(self._str_import_path) | |
288 | if from_cache is not None: | |
289 | return ValueSet({from_cache}) | |
290 | from_cache = self._inference_state.module_cache.get(self._str_import_path) | |
291 | if from_cache is not None: | |
292 | return from_cache | |
293 | ||
294 | sys_path = self._sys_path_with_modifications(is_completion=False) | |
295 | ||
296 | return import_module_by_names( | |
297 | self._inference_state, self.import_path, sys_path, self._module_context | |
298 | ) | |
299 | ||
300 | def _get_module_names(self, search_path=None, in_module=None): | |
301 | """ | |
302 | Get the names of all modules in the search_path. This means file names | |
303 | and not names defined in the files. | |
304 | """ | |
305 | if search_path is None: | |
306 | sys_path = self._sys_path_with_modifications(is_completion=True) | |
307 | else: | |
308 | sys_path = search_path | |
309 | return list(iter_module_names( | |
310 | self._inference_state, self._module_context, sys_path, | |
311 | module_cls=ImportName if in_module is None else SubModuleName, | |
312 | add_builtin_modules=search_path is None and in_module is None, | |
313 | )) | |
314 | ||
315 | def completion_names(self, inference_state, only_modules=False): | |
316 | """ | |
317 | :param only_modules: Indicates wheter it's possible to import a | |
318 | definition that is not defined in a module. | |
319 | """ | |
320 | if not self._infer_possible: | |
321 | return [] | |
322 | ||
323 | names = [] | |
324 | if self.import_path: | |
325 | # flask | |
326 | if self._str_import_path == ('flask', 'ext'): | |
327 | # List Flask extensions like ``flask_foo`` | |
328 | for mod in self._get_module_names(): | |
329 | modname = mod.string_name | |
330 | if modname.startswith('flask_'): | |
331 | extname = modname[len('flask_'):] | |
332 | names.append(ImportName(self._module_context, extname)) | |
333 | # Now the old style: ``flaskext.foo`` | |
334 | for dir in self._sys_path_with_modifications(is_completion=True): | |
335 | flaskext = os.path.join(dir, 'flaskext') | |
336 | if os.path.isdir(flaskext): | |
337 | names += self._get_module_names([flaskext]) | |
338 | ||
339 | values = self.follow() | |
340 | for value in values: | |
341 | # Non-modules are not completable. | |
342 | if value.api_type not in ('module', 'namespace'): # not a module | |
343 | continue | |
344 | if not value.is_compiled(): | |
345 | # sub_modules_dict is not implemented for compiled modules. | |
346 | names += value.sub_modules_dict().values() | |
347 | ||
348 | if not only_modules: | |
349 | from jedi.inference.gradual.conversion import convert_values | |
350 | ||
351 | both_values = values | convert_values(values) | |
352 | for c in both_values: | |
353 | for filter in c.get_filters(): | |
354 | names += filter.values() | |
355 | else: | |
356 | if self.level: | |
357 | # We only get here if the level cannot be properly calculated. | |
358 | names += self._get_module_names(self._fixed_sys_path) | |
359 | else: | |
360 | # This is just the list of global imports. | |
361 | names += self._get_module_names() | |
362 | return names | |
363 | ||
364 | ||
365 | def import_module_by_names(inference_state, import_names, sys_path=None, | |
366 | module_context=None, prefer_stubs=True): | |
367 | if sys_path is None: | |
368 | sys_path = inference_state.get_sys_path() | |
369 | ||
370 | str_import_names = tuple( | |
371 | i.value if isinstance(i, tree.Name) else i | |
372 | for i in import_names | |
373 | ) | |
374 | value_set = [None] | |
375 | for i, name in enumerate(import_names): | |
376 | value_set = ValueSet.from_sets([ | |
377 | import_module( | |
378 | inference_state, | |
379 | str_import_names[:i+1], | |
380 | parent_module_value, | |
381 | sys_path, | |
382 | prefer_stubs=prefer_stubs, | |
383 | ) for parent_module_value in value_set | |
384 | ]) | |
385 | if not value_set: | |
386 | message = 'No module named ' + '.'.join(str_import_names) | |
387 | if module_context is not None: | |
388 | _add_error(module_context, name, message) | |
389 | else: | |
390 | debug.warning(message) | |
391 | return NO_VALUES | |
392 | return value_set | |
393 | ||
394 | ||
395 | @plugin_manager.decorate() | |
396 | @import_module_decorator | |
397 | def import_module(inference_state, import_names, parent_module_value, sys_path): | |
398 | """ | |
399 | This method is very similar to importlib's `_gcd_import`. | |
400 | """ | |
401 | if import_names[0] in settings.auto_import_modules: | |
402 | module = _load_builtin_module(inference_state, import_names, sys_path) | |
403 | if module is None: | |
404 | return NO_VALUES | |
405 | return ValueSet([module]) | |
406 | ||
407 | module_name = '.'.join(import_names) | |
408 | if parent_module_value is None: | |
409 | # Override the sys.path. It works only good that way. | |
410 | # Injecting the path directly into `find_module` did not work. | |
411 | file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info( | |
412 | string=import_names[-1], | |
413 | full_name=module_name, | |
414 | sys_path=sys_path, | |
415 | is_global_search=True, | |
416 | ) | |
417 | if is_pkg is None: | |
418 | return NO_VALUES | |
419 | else: | |
420 | paths = parent_module_value.py__path__() | |
421 | if paths is None: | |
422 | # The module might not be a package. | |
423 | return NO_VALUES | |
424 | ||
425 | file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info( | |
426 | string=import_names[-1], | |
427 | path=paths, | |
428 | full_name=module_name, | |
429 | is_global_search=False, | |
430 | ) | |
431 | if is_pkg is None: | |
432 | return NO_VALUES | |
433 | ||
434 | if isinstance(file_io_or_ns, ImplicitNSInfo): | |
435 | from jedi.inference.value.namespace import ImplicitNamespaceValue | |
436 | module = ImplicitNamespaceValue( | |
437 | inference_state, | |
438 | string_names=tuple(file_io_or_ns.name.split('.')), | |
439 | paths=file_io_or_ns.paths, | |
440 | ) | |
441 | elif file_io_or_ns is None: | |
442 | module = _load_builtin_module(inference_state, import_names, sys_path) | |
443 | if module is None: | |
444 | return NO_VALUES | |
445 | else: | |
446 | module = _load_python_module( | |
447 | inference_state, file_io_or_ns, | |
448 | import_names=import_names, | |
449 | is_package=is_pkg, | |
450 | ) | |
451 | ||
452 | if parent_module_value is None: | |
453 | debug.dbg('global search_module %s: %s', import_names[-1], module) | |
454 | else: | |
455 | debug.dbg('search_module %s in paths %s: %s', module_name, paths, module) | |
456 | return ValueSet([module]) | |
457 | ||
458 | ||
459 | def _load_python_module(inference_state, file_io, | |
460 | import_names=None, is_package=False): | |
461 | module_node = inference_state.parse( | |
462 | file_io=file_io, | |
463 | cache=True, | |
464 | diff_cache=settings.fast_parser, | |
465 | cache_path=settings.cache_directory, | |
466 | ) | |
467 | ||
468 | from jedi.inference.value import ModuleValue | |
469 | return ModuleValue( | |
470 | inference_state, module_node, | |
471 | file_io=file_io, | |
472 | string_names=import_names, | |
473 | code_lines=get_cached_code_lines(inference_state.grammar, file_io.path), | |
474 | is_package=is_package, | |
475 | ) | |
476 | ||
477 | ||
478 | def _load_builtin_module(inference_state, import_names=None, sys_path=None): | |
479 | project = inference_state.project | |
480 | if sys_path is None: | |
481 | sys_path = inference_state.get_sys_path() | |
482 | if not project._load_unsafe_extensions: | |
483 | safe_paths = project._get_base_sys_path(inference_state) | |
484 | sys_path = [p for p in sys_path if p in safe_paths] | |
485 | ||
486 | dotted_name = '.'.join(import_names) | |
487 | assert dotted_name is not None | |
488 | module = compiled.load_module(inference_state, dotted_name=dotted_name, sys_path=sys_path) | |
489 | if module is None: | |
490 | # The file might raise an ImportError e.g. and therefore not be | |
491 | # importable. | |
492 | return None | |
493 | return module | |
494 | ||
495 | ||
496 | def load_module_from_path(inference_state, file_io, import_names=None, is_package=None): | |
497 | """ | |
498 | This should pretty much only be used for get_modules_containing_name. It's | |
499 | here to ensure that a random path is still properly loaded into the Jedi | |
500 | module structure. | |
501 | """ | |
502 | path = Path(file_io.path) | |
503 | if import_names is None: | |
504 | e_sys_path = inference_state.get_sys_path() | |
505 | import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path) | |
506 | else: | |
507 | assert isinstance(is_package, bool) | |
508 | ||
509 | is_stub = path.suffix == '.pyi' | |
510 | if is_stub: | |
511 | folder_io = file_io.get_parent_folder() | |
512 | if folder_io.path.endswith('-stubs'): | |
513 | folder_io = FolderIO(folder_io.path[:-6]) | |
514 | if path.name == '__init__.pyi': | |
515 | python_file_io = folder_io.get_file_io('__init__.py') | |
516 | else: | |
517 | python_file_io = folder_io.get_file_io(import_names[-1] + '.py') | |
518 | ||
519 | try: | |
520 | v = load_module_from_path( | |
521 | inference_state, python_file_io, | |
522 | import_names, is_package=is_package | |
523 | ) | |
524 | values = ValueSet([v]) | |
525 | except FileNotFoundError: | |
526 | values = NO_VALUES | |
527 | ||
528 | return create_stub_module( | |
529 | inference_state, inference_state.latest_grammar, values, | |
530 | parse_stub_module(inference_state, file_io), file_io, import_names | |
531 | ) | |
532 | else: | |
533 | module = _load_python_module( | |
534 | inference_state, file_io, | |
535 | import_names=import_names, | |
536 | is_package=is_package, | |
537 | ) | |
538 | inference_state.module_cache.add(import_names, ValueSet([module])) | |
539 | return module | |
540 | ||
541 | ||
542 | def load_namespace_from_path(inference_state, folder_io): | |
543 | import_names, is_package = sys_path.transform_path_to_dotted( | |
544 | inference_state.get_sys_path(), | |
545 | Path(folder_io.path) | |
546 | ) | |
547 | from jedi.inference.value.namespace import ImplicitNamespaceValue | |
548 | return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path]) | |
549 | ||
550 | ||
551 | def follow_error_node_imports_if_possible(context, name): | |
552 | error_node = tree.search_ancestor(name, 'error_node') | |
553 | if error_node is not None: | |
554 | # Get the first command start of a started simple_stmt. The error | |
555 | # node is sometimes a small_stmt and sometimes a simple_stmt. Check | |
556 | # for ; leaves that start a new statements. | |
557 | start_index = 0 | |
558 | for index, n in enumerate(error_node.children): | |
559 | if n.start_pos > name.start_pos: | |
560 | break | |
561 | if n == ';': | |
562 | start_index = index + 1 | |
563 | nodes = error_node.children[start_index:] | |
564 | first_name = nodes[0].get_first_leaf().value | |
565 | ||
566 | # Make it possible to infer stuff like `import foo.` or | |
567 | # `from foo.bar`. | |
568 | if first_name in ('from', 'import'): | |
569 | is_import_from = first_name == 'from' | |
570 | level, names = helpers.parse_dotted_names( | |
571 | nodes, | |
572 | is_import_from=is_import_from, | |
573 | until_node=name, | |
574 | ) | |
575 | return Importer( | |
576 | context.inference_state, names, context.get_root_context(), level).follow() | |
577 | return None | |
578 | ||
579 | ||
580 | def iter_module_names(inference_state, module_context, search_path, | |
581 | module_cls=ImportName, add_builtin_modules=True): | |
582 | """ | |
583 | Get the names of all modules in the search_path. This means file names | |
584 | and not names defined in the files. | |
585 | """ | |
586 | # add builtin module names | |
587 | if add_builtin_modules: | |
588 | for name in inference_state.compiled_subprocess.get_builtin_module_names(): | |
589 | yield module_cls(module_context, name) | |
590 | ||
591 | for name in inference_state.compiled_subprocess.iter_module_names(search_path): | |
592 | yield module_cls(module_context, name) |