]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | """ |
2 | Environments are a way to activate different Python versions or Virtualenvs for | |
3 | static analysis. The Python binary in that environment is going to be executed. | |
4 | """ | |
5 | import os | |
6 | import sys | |
7 | import hashlib | |
8 | import filecmp | |
9 | from collections import namedtuple | |
10 | from shutil import which | |
11 | ||
12 | from jedi.cache import memoize_method, time_cache | |
13 | from jedi.inference.compiled.subprocess import CompiledSubprocess, \ | |
14 | InferenceStateSameProcess, InferenceStateSubprocess | |
15 | ||
16 | import parso | |
17 | ||
18 | _VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match] | |
19 | ||
20 | _SUPPORTED_PYTHONS = ['3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6'] | |
21 | _SAFE_PATHS = ['/usr/bin', '/usr/local/bin'] | |
22 | _CONDA_VAR = 'CONDA_PREFIX' | |
23 | _CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor) | |
24 | ||
25 | ||
26 | class InvalidPythonEnvironment(Exception): | |
27 | """ | |
28 | If you see this exception, the Python executable or Virtualenv you have | |
29 | been trying to use is probably not a correct Python version. | |
30 | """ | |
31 | ||
32 | ||
33 | class _BaseEnvironment: | |
34 | @memoize_method | |
35 | def get_grammar(self): | |
36 | version_string = '%s.%s' % (self.version_info.major, self.version_info.minor) | |
37 | return parso.load_grammar(version=version_string) | |
38 | ||
39 | @property | |
40 | def _sha256(self): | |
41 | try: | |
42 | return self._hash | |
43 | except AttributeError: | |
44 | self._hash = _calculate_sha256_for_file(self.executable) | |
45 | return self._hash | |
46 | ||
47 | ||
48 | def _get_info(): | |
49 | return ( | |
50 | sys.executable, | |
51 | sys.prefix, | |
52 | sys.version_info[:3], | |
53 | ) | |
54 | ||
55 | ||
56 | class Environment(_BaseEnvironment): | |
57 | """ | |
58 | This class is supposed to be created by internal Jedi architecture. You | |
59 | should not create it directly. Please use create_environment or the other | |
60 | functions instead. It is then returned by that function. | |
61 | """ | |
62 | _subprocess = None | |
63 | ||
64 | def __init__(self, executable, env_vars=None): | |
65 | self._start_executable = executable | |
66 | self._env_vars = env_vars | |
67 | # Initialize the environment | |
68 | self._get_subprocess() | |
69 | ||
70 | def _get_subprocess(self): | |
71 | if self._subprocess is not None and not self._subprocess.is_crashed: | |
72 | return self._subprocess | |
73 | ||
74 | try: | |
75 | self._subprocess = CompiledSubprocess(self._start_executable, | |
76 | env_vars=self._env_vars) | |
77 | info = self._subprocess._send(None, _get_info) | |
78 | except Exception as exc: | |
79 | raise InvalidPythonEnvironment( | |
80 | "Could not get version information for %r: %r" % ( | |
81 | self._start_executable, | |
82 | exc)) | |
83 | ||
84 | # Since it could change and might not be the same(?) as the one given, | |
85 | # set it here. | |
86 | self.executable = info[0] | |
87 | """ | |
88 | The Python executable, matches ``sys.executable``. | |
89 | """ | |
90 | self.path = info[1] | |
91 | """ | |
92 | The path to an environment, matches ``sys.prefix``. | |
93 | """ | |
94 | self.version_info = _VersionInfo(*info[2]) | |
95 | """ | |
96 | Like :data:`sys.version_info`: a tuple to show the current | |
97 | Environment's Python version. | |
98 | """ | |
99 | return self._subprocess | |
100 | ||
101 | def __repr__(self): | |
102 | version = '.'.join(str(i) for i in self.version_info) | |
103 | return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path) | |
104 | ||
105 | def get_inference_state_subprocess(self, inference_state): | |
106 | return InferenceStateSubprocess(inference_state, self._get_subprocess()) | |
107 | ||
108 | @memoize_method | |
109 | def get_sys_path(self): | |
110 | """ | |
111 | The sys path for this environment. Does not include potential | |
112 | modifications from e.g. appending to :data:`sys.path`. | |
113 | ||
114 | :returns: list of str | |
115 | """ | |
116 | # It's pretty much impossible to generate the sys path without actually | |
117 | # executing Python. The sys path (when starting with -S) itself depends | |
118 | # on how the Python version was compiled (ENV variables). | |
119 | # If you omit -S when starting Python (normal case), additionally | |
120 | # site.py gets executed. | |
121 | return self._get_subprocess().get_sys_path() | |
122 | ||
123 | ||
124 | class _SameEnvironmentMixin: | |
125 | def __init__(self): | |
126 | self._start_executable = self.executable = sys.executable | |
127 | self.path = sys.prefix | |
128 | self.version_info = _VersionInfo(*sys.version_info[:3]) | |
129 | self._env_vars = None | |
130 | ||
131 | ||
132 | class SameEnvironment(_SameEnvironmentMixin, Environment): | |
133 | pass | |
134 | ||
135 | ||
136 | class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment): | |
137 | def get_inference_state_subprocess(self, inference_state): | |
138 | return InferenceStateSameProcess(inference_state) | |
139 | ||
140 | def get_sys_path(self): | |
141 | return sys.path | |
142 | ||
143 | ||
144 | def _get_virtual_env_from_var(env_var='VIRTUAL_ENV'): | |
145 | """Get virtualenv environment from VIRTUAL_ENV environment variable. | |
146 | ||
147 | It uses `safe=False` with ``create_environment``, because the environment | |
148 | variable is considered to be safe / controlled by the user solely. | |
149 | """ | |
150 | var = os.environ.get(env_var) | |
151 | if var: | |
152 | # Under macOS in some cases - notably when using Pipenv - the | |
153 | # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of | |
154 | # /path/to/env so we need to fully resolve the paths in order to | |
155 | # compare them. | |
156 | if os.path.realpath(var) == os.path.realpath(sys.prefix): | |
157 | return _try_get_same_env() | |
158 | ||
159 | try: | |
160 | return create_environment(var, safe=False) | |
161 | except InvalidPythonEnvironment: | |
162 | pass | |
163 | ||
164 | ||
165 | def _calculate_sha256_for_file(path): | |
166 | sha256 = hashlib.sha256() | |
167 | with open(path, 'rb') as f: | |
168 | for block in iter(lambda: f.read(filecmp.BUFSIZE), b''): | |
169 | sha256.update(block) | |
170 | return sha256.hexdigest() | |
171 | ||
172 | ||
173 | def get_default_environment(): | |
174 | """ | |
175 | Tries to return an active Virtualenv or conda environment. | |
176 | If there is no VIRTUAL_ENV variable or no CONDA_PREFIX variable set | |
177 | set it will return the latest Python version installed on the system. This | |
178 | makes it possible to use as many new Python features as possible when using | |
179 | autocompletion and other functionality. | |
180 | ||
181 | :returns: :class:`.Environment` | |
182 | """ | |
183 | virtual_env = _get_virtual_env_from_var() | |
184 | if virtual_env is not None: | |
185 | return virtual_env | |
186 | ||
187 | conda_env = _get_virtual_env_from_var(_CONDA_VAR) | |
188 | if conda_env is not None: | |
189 | return conda_env | |
190 | ||
191 | return _try_get_same_env() | |
192 | ||
193 | ||
194 | def _try_get_same_env(): | |
195 | env = SameEnvironment() | |
196 | if not os.path.basename(env.executable).lower().startswith('python'): | |
197 | # This tries to counter issues with embedding. In some cases (e.g. | |
198 | # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This | |
199 | # happens, because for Mac a function called `_NSGetExecutablePath` is | |
200 | # used and for Windows `GetModuleFileNameW`. These are both platform | |
201 | # specific functions. For all other systems sys.executable should be | |
202 | # alright. However here we try to generalize: | |
203 | # | |
204 | # 1. Check if the executable looks like python (heuristic) | |
205 | # 2. In case it's not try to find the executable | |
206 | # 3. In case we don't find it use an interpreter environment. | |
207 | # | |
208 | # The last option will always work, but leads to potential crashes of | |
209 | # Jedi - which is ok, because it happens very rarely and even less, | |
210 | # because the code below should work for most cases. | |
211 | if os.name == 'nt': | |
212 | # The first case would be a virtualenv and the second a normal | |
213 | # Python installation. | |
214 | checks = (r'Scripts\python.exe', 'python.exe') | |
215 | else: | |
216 | # For unix it looks like Python is always in a bin folder. | |
217 | checks = ( | |
218 | 'bin/python%s.%s' % (sys.version_info[0], sys.version[1]), | |
219 | 'bin/python%s' % (sys.version_info[0]), | |
220 | 'bin/python', | |
221 | ) | |
222 | for check in checks: | |
223 | guess = os.path.join(sys.exec_prefix, check) | |
224 | if os.path.isfile(guess): | |
225 | # Bingo - We think we have our Python. | |
226 | return Environment(guess) | |
227 | # It looks like there is no reasonable Python to be found. | |
228 | return InterpreterEnvironment() | |
229 | # If no virtualenv is found, use the environment we're already | |
230 | # using. | |
231 | return env | |
232 | ||
233 | ||
234 | def get_cached_default_environment(): | |
235 | var = os.environ.get('VIRTUAL_ENV') or os.environ.get(_CONDA_VAR) | |
236 | environment = _get_cached_default_environment() | |
237 | ||
238 | # Under macOS in some cases - notably when using Pipenv - the | |
239 | # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of | |
240 | # /path/to/env so we need to fully resolve the paths in order to | |
241 | # compare them. | |
242 | if var and os.path.realpath(var) != os.path.realpath(environment.path): | |
243 | _get_cached_default_environment.clear_cache() | |
244 | return _get_cached_default_environment() | |
245 | return environment | |
246 | ||
247 | ||
248 | @time_cache(seconds=10 * 60) # 10 Minutes | |
249 | def _get_cached_default_environment(): | |
250 | try: | |
251 | return get_default_environment() | |
252 | except InvalidPythonEnvironment: | |
253 | # It's possible that `sys.executable` is wrong. Typically happens | |
254 | # when Jedi is used in an executable that embeds Python. For further | |
255 | # information, have a look at: | |
256 | # https://github.com/davidhalter/jedi/issues/1531 | |
257 | return InterpreterEnvironment() | |
258 | ||
259 | ||
260 | def find_virtualenvs(paths=None, *, safe=True, use_environment_vars=True): | |
261 | """ | |
262 | :param paths: A list of paths in your file system to be scanned for | |
263 | Virtualenvs. It will search in these paths and potentially execute the | |
264 | Python binaries. | |
265 | :param safe: Default True. In case this is False, it will allow this | |
266 | function to execute potential `python` environments. An attacker might | |
267 | be able to drop an executable in a path this function is searching by | |
268 | default. If the executable has not been installed by root, it will not | |
269 | be executed. | |
270 | :param use_environment_vars: Default True. If True, the VIRTUAL_ENV | |
271 | variable will be checked if it contains a valid VirtualEnv. | |
272 | CONDA_PREFIX will be checked to see if it contains a valid conda | |
273 | environment. | |
274 | ||
275 | :yields: :class:`.Environment` | |
276 | """ | |
277 | if paths is None: | |
278 | paths = [] | |
279 | ||
280 | _used_paths = set() | |
281 | ||
282 | if use_environment_vars: | |
283 | # Using this variable should be safe, because attackers might be | |
284 | # able to drop files (via git) but not environment variables. | |
285 | virtual_env = _get_virtual_env_from_var() | |
286 | if virtual_env is not None: | |
287 | yield virtual_env | |
288 | _used_paths.add(virtual_env.path) | |
289 | ||
290 | conda_env = _get_virtual_env_from_var(_CONDA_VAR) | |
291 | if conda_env is not None: | |
292 | yield conda_env | |
293 | _used_paths.add(conda_env.path) | |
294 | ||
295 | for directory in paths: | |
296 | if not os.path.isdir(directory): | |
297 | continue | |
298 | ||
299 | directory = os.path.abspath(directory) | |
300 | for path in os.listdir(directory): | |
301 | path = os.path.join(directory, path) | |
302 | if path in _used_paths: | |
303 | # A path shouldn't be inferred twice. | |
304 | continue | |
305 | _used_paths.add(path) | |
306 | ||
307 | try: | |
308 | executable = _get_executable_path(path, safe=safe) | |
309 | yield Environment(executable) | |
310 | except InvalidPythonEnvironment: | |
311 | pass | |
312 | ||
313 | ||
314 | def find_system_environments(*, env_vars=None): | |
315 | """ | |
316 | Ignores virtualenvs and returns the Python versions that were installed on | |
317 | your system. This might return nothing, if you're running Python e.g. from | |
318 | a portable version. | |
319 | ||
320 | The environments are sorted from latest to oldest Python version. | |
321 | ||
322 | :yields: :class:`.Environment` | |
323 | """ | |
324 | for version_string in _SUPPORTED_PYTHONS: | |
325 | try: | |
326 | yield get_system_environment(version_string, env_vars=env_vars) | |
327 | except InvalidPythonEnvironment: | |
328 | pass | |
329 | ||
330 | ||
331 | # TODO: this function should probably return a list of environments since | |
332 | # multiple Python installations can be found on a system for the same version. | |
333 | def get_system_environment(version, *, env_vars=None): | |
334 | """ | |
335 | Return the first Python environment found for a string of the form 'X.Y' | |
336 | where X and Y are the major and minor versions of Python. | |
337 | ||
338 | :raises: :exc:`.InvalidPythonEnvironment` | |
339 | :returns: :class:`.Environment` | |
340 | """ | |
341 | exe = which('python' + version) | |
342 | if exe: | |
343 | if exe == sys.executable: | |
344 | return SameEnvironment() | |
345 | return Environment(exe) | |
346 | ||
347 | if os.name == 'nt': | |
348 | for exe in _get_executables_from_windows_registry(version): | |
349 | try: | |
350 | return Environment(exe, env_vars=env_vars) | |
351 | except InvalidPythonEnvironment: | |
352 | pass | |
353 | raise InvalidPythonEnvironment("Cannot find executable python%s." % version) | |
354 | ||
355 | ||
356 | def create_environment(path, *, safe=True, env_vars=None): | |
357 | """ | |
358 | Make it possible to manually create an Environment object by specifying a | |
359 | Virtualenv path or an executable path and optional environment variables. | |
360 | ||
361 | :raises: :exc:`.InvalidPythonEnvironment` | |
362 | :returns: :class:`.Environment` | |
363 | """ | |
364 | if os.path.isfile(path): | |
365 | _assert_safe(path, safe) | |
366 | return Environment(path, env_vars=env_vars) | |
367 | return Environment(_get_executable_path(path, safe=safe), env_vars=env_vars) | |
368 | ||
369 | ||
370 | def _get_executable_path(path, safe=True): | |
371 | """ | |
372 | Returns None if it's not actually a virtual env. | |
373 | """ | |
374 | ||
375 | if os.name == 'nt': | |
376 | python = os.path.join(path, 'Scripts', 'python.exe') | |
377 | else: | |
378 | python = os.path.join(path, 'bin', 'python') | |
379 | if not os.path.exists(python): | |
380 | raise InvalidPythonEnvironment("%s seems to be missing." % python) | |
381 | ||
382 | _assert_safe(python, safe) | |
383 | return python | |
384 | ||
385 | ||
386 | def _get_executables_from_windows_registry(version): | |
387 | import winreg | |
388 | ||
389 | # TODO: support Python Anaconda. | |
390 | sub_keys = [ | |
391 | r'SOFTWARE\Python\PythonCore\{version}\InstallPath', | |
392 | r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath', | |
393 | r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath', | |
394 | r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath' | |
395 | ] | |
396 | for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]: | |
397 | for sub_key in sub_keys: | |
398 | sub_key = sub_key.format(version=version) | |
399 | try: | |
400 | with winreg.OpenKey(root_key, sub_key) as key: | |
401 | prefix = winreg.QueryValueEx(key, '')[0] | |
402 | exe = os.path.join(prefix, 'python.exe') | |
403 | if os.path.isfile(exe): | |
404 | yield exe | |
405 | except WindowsError: | |
406 | pass | |
407 | ||
408 | ||
409 | def _assert_safe(executable_path, safe): | |
410 | if safe and not _is_safe(executable_path): | |
411 | raise InvalidPythonEnvironment( | |
412 | "The python binary is potentially unsafe.") | |
413 | ||
414 | ||
415 | def _is_safe(executable_path): | |
416 | # Resolve sym links. A venv typically is a symlink to a known Python | |
417 | # binary. Only virtualenvs copy symlinks around. | |
418 | real_path = os.path.realpath(executable_path) | |
419 | ||
420 | if _is_unix_safe_simple(real_path): | |
421 | return True | |
422 | ||
423 | # Just check the list of known Python versions. If it's not in there, | |
424 | # it's likely an attacker or some Python that was not properly | |
425 | # installed in the system. | |
426 | for environment in find_system_environments(): | |
427 | if environment.executable == real_path: | |
428 | return True | |
429 | ||
430 | # If the versions don't match, just compare the binary files. If we | |
431 | # don't do that, only venvs will be working and not virtualenvs. | |
432 | # venvs are symlinks while virtualenvs are actual copies of the | |
433 | # Python files. | |
434 | # This still means that if the system Python is updated and the | |
435 | # virtualenv's Python is not (which is probably never going to get | |
436 | # upgraded), it will not work with Jedi. IMO that's fine, because | |
437 | # people should just be using venv. ~ dave | |
438 | if environment._sha256 == _calculate_sha256_for_file(real_path): | |
439 | return True | |
440 | return False | |
441 | ||
442 | ||
443 | def _is_unix_safe_simple(real_path): | |
444 | if _is_unix_admin(): | |
445 | # In case we are root, just be conservative and | |
446 | # only execute known paths. | |
447 | return any(real_path.startswith(p) for p in _SAFE_PATHS) | |
448 | ||
449 | uid = os.stat(real_path).st_uid | |
450 | # The interpreter needs to be owned by root. This means that it wasn't | |
451 | # written by a user and therefore attacking Jedi is not as simple. | |
452 | # The attack could look like the following: | |
453 | # 1. A user clones a repository. | |
454 | # 2. The repository has an innocent looking folder called foobar. jedi | |
455 | # searches for the folder and executes foobar/bin/python --version if | |
456 | # there's also a foobar/bin/activate. | |
457 | # 3. The attacker has gained code execution, since he controls | |
458 | # foobar/bin/python. | |
459 | return uid == 0 | |
460 | ||
461 | ||
462 | def _is_unix_admin(): | |
463 | try: | |
464 | return os.getuid() == 0 | |
465 | except AttributeError: | |
466 | return False # Windows |