]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | """distutils.util |
2 | ||
3 | Miscellaneous utility functions -- anything that doesn't fit into | |
4 | one of the other *util.py modules. | |
5 | """ | |
6 | ||
7 | import importlib.util | |
8 | import os | |
9 | import re | |
10 | import string | |
11 | import subprocess | |
12 | import sys | |
13 | import sysconfig | |
14 | import functools | |
15 | ||
16 | from .errors import DistutilsPlatformError, DistutilsByteCompileError | |
17 | from .dep_util import newer | |
18 | from .spawn import spawn | |
19 | from ._log import log | |
20 | ||
21 | ||
22 | def get_host_platform(): | |
23 | """ | |
24 | Return a string that identifies the current platform. Use this | |
25 | function to distinguish platform-specific build directories and | |
26 | platform-specific built distributions. | |
27 | """ | |
28 | ||
29 | # This function initially exposed platforms as defined in Python 3.9 | |
30 | # even with older Python versions when distutils was split out. | |
31 | # Now it delegates to stdlib sysconfig, but maintains compatibility. | |
32 | ||
33 | if sys.version_info < (3, 8): | |
34 | if os.name == 'nt': | |
35 | if '(arm)' in sys.version.lower(): | |
36 | return 'win-arm32' | |
37 | if '(arm64)' in sys.version.lower(): | |
38 | return 'win-arm64' | |
39 | ||
40 | if sys.version_info < (3, 9): | |
41 | if os.name == "posix" and hasattr(os, 'uname'): | |
42 | osname, host, release, version, machine = os.uname() | |
43 | if osname[:3] == "aix": | |
44 | from .py38compat import aix_platform | |
45 | ||
46 | return aix_platform(osname, version, release) | |
47 | ||
48 | return sysconfig.get_platform() | |
49 | ||
50 | ||
51 | def get_platform(): | |
52 | if os.name == 'nt': | |
53 | TARGET_TO_PLAT = { | |
54 | 'x86': 'win32', | |
55 | 'x64': 'win-amd64', | |
56 | 'arm': 'win-arm32', | |
57 | 'arm64': 'win-arm64', | |
58 | } | |
59 | target = os.environ.get('VSCMD_ARG_TGT_ARCH') | |
60 | return TARGET_TO_PLAT.get(target) or get_host_platform() | |
61 | return get_host_platform() | |
62 | ||
63 | ||
64 | if sys.platform == 'darwin': | |
65 | _syscfg_macosx_ver = None # cache the version pulled from sysconfig | |
66 | MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET' | |
67 | ||
68 | ||
69 | def _clear_cached_macosx_ver(): | |
70 | """For testing only. Do not call.""" | |
71 | global _syscfg_macosx_ver | |
72 | _syscfg_macosx_ver = None | |
73 | ||
74 | ||
75 | def get_macosx_target_ver_from_syscfg(): | |
76 | """Get the version of macOS latched in the Python interpreter configuration. | |
77 | Returns the version as a string or None if can't obtain one. Cached.""" | |
78 | global _syscfg_macosx_ver | |
79 | if _syscfg_macosx_ver is None: | |
80 | from distutils import sysconfig | |
81 | ||
82 | ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or '' | |
83 | if ver: | |
84 | _syscfg_macosx_ver = ver | |
85 | return _syscfg_macosx_ver | |
86 | ||
87 | ||
88 | def get_macosx_target_ver(): | |
89 | """Return the version of macOS for which we are building. | |
90 | ||
91 | The target version defaults to the version in sysconfig latched at time | |
92 | the Python interpreter was built, unless overridden by an environment | |
93 | variable. If neither source has a value, then None is returned""" | |
94 | ||
95 | syscfg_ver = get_macosx_target_ver_from_syscfg() | |
96 | env_ver = os.environ.get(MACOSX_VERSION_VAR) | |
97 | ||
98 | if env_ver: | |
99 | # Validate overridden version against sysconfig version, if have both. | |
100 | # Ensure that the deployment target of the build process is not less | |
101 | # than 10.3 if the interpreter was built for 10.3 or later. This | |
102 | # ensures extension modules are built with correct compatibility | |
103 | # values, specifically LDSHARED which can use | |
104 | # '-undefined dynamic_lookup' which only works on >= 10.3. | |
105 | if ( | |
106 | syscfg_ver | |
107 | and split_version(syscfg_ver) >= [10, 3] | |
108 | and split_version(env_ver) < [10, 3] | |
109 | ): | |
110 | my_msg = ( | |
111 | '$' + MACOSX_VERSION_VAR + ' mismatch: ' | |
112 | 'now "%s" but "%s" during configure; ' | |
113 | 'must use 10.3 or later' % (env_ver, syscfg_ver) | |
114 | ) | |
115 | raise DistutilsPlatformError(my_msg) | |
116 | return env_ver | |
117 | return syscfg_ver | |
118 | ||
119 | ||
120 | def split_version(s): | |
121 | """Convert a dot-separated string into a list of numbers for comparisons""" | |
122 | return [int(n) for n in s.split('.')] | |
123 | ||
124 | ||
125 | def convert_path(pathname): | |
126 | """Return 'pathname' as a name that will work on the native filesystem, | |
127 | i.e. split it on '/' and put it back together again using the current | |
128 | directory separator. Needed because filenames in the setup script are | |
129 | always supplied in Unix style, and have to be converted to the local | |
130 | convention before we can actually use them in the filesystem. Raises | |
131 | ValueError on non-Unix-ish systems if 'pathname' either starts or | |
132 | ends with a slash. | |
133 | """ | |
134 | if os.sep == '/': | |
135 | return pathname | |
136 | if not pathname: | |
137 | return pathname | |
138 | if pathname[0] == '/': | |
139 | raise ValueError("path '%s' cannot be absolute" % pathname) | |
140 | if pathname[-1] == '/': | |
141 | raise ValueError("path '%s' cannot end with '/'" % pathname) | |
142 | ||
143 | paths = pathname.split('/') | |
144 | while '.' in paths: | |
145 | paths.remove('.') | |
146 | if not paths: | |
147 | return os.curdir | |
148 | return os.path.join(*paths) | |
149 | ||
150 | ||
151 | # convert_path () | |
152 | ||
153 | ||
154 | def change_root(new_root, pathname): | |
155 | """Return 'pathname' with 'new_root' prepended. If 'pathname' is | |
156 | relative, this is equivalent to "os.path.join(new_root,pathname)". | |
157 | Otherwise, it requires making 'pathname' relative and then joining the | |
158 | two, which is tricky on DOS/Windows and Mac OS. | |
159 | """ | |
160 | if os.name == 'posix': | |
161 | if not os.path.isabs(pathname): | |
162 | return os.path.join(new_root, pathname) | |
163 | else: | |
164 | return os.path.join(new_root, pathname[1:]) | |
165 | ||
166 | elif os.name == 'nt': | |
167 | (drive, path) = os.path.splitdrive(pathname) | |
168 | if path[0] == '\\': | |
169 | path = path[1:] | |
170 | return os.path.join(new_root, path) | |
171 | ||
172 | raise DistutilsPlatformError(f"nothing known about platform '{os.name}'") | |
173 | ||
174 | ||
175 | @functools.lru_cache() | |
176 | def check_environ(): | |
177 | """Ensure that 'os.environ' has all the environment variables we | |
178 | guarantee that users can use in config files, command-line options, | |
179 | etc. Currently this includes: | |
180 | HOME - user's home directory (Unix only) | |
181 | PLAT - description of the current platform, including hardware | |
182 | and OS (see 'get_platform()') | |
183 | """ | |
184 | if os.name == 'posix' and 'HOME' not in os.environ: | |
185 | try: | |
186 | import pwd | |
187 | ||
188 | os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] | |
189 | except (ImportError, KeyError): | |
190 | # bpo-10496: if the current user identifier doesn't exist in the | |
191 | # password database, do nothing | |
192 | pass | |
193 | ||
194 | if 'PLAT' not in os.environ: | |
195 | os.environ['PLAT'] = get_platform() | |
196 | ||
197 | ||
198 | def subst_vars(s, local_vars): | |
199 | """ | |
200 | Perform variable substitution on 'string'. | |
201 | Variables are indicated by format-style braces ("{var}"). | |
202 | Variable is substituted by the value found in the 'local_vars' | |
203 | dictionary or in 'os.environ' if it's not in 'local_vars'. | |
204 | 'os.environ' is first checked/augmented to guarantee that it contains | |
205 | certain values: see 'check_environ()'. Raise ValueError for any | |
206 | variables not found in either 'local_vars' or 'os.environ'. | |
207 | """ | |
208 | check_environ() | |
209 | lookup = dict(os.environ) | |
210 | lookup.update((name, str(value)) for name, value in local_vars.items()) | |
211 | try: | |
212 | return _subst_compat(s).format_map(lookup) | |
213 | except KeyError as var: | |
214 | raise ValueError(f"invalid variable {var}") | |
215 | ||
216 | ||
217 | def _subst_compat(s): | |
218 | """ | |
219 | Replace shell/Perl-style variable substitution with | |
220 | format-style. For compatibility. | |
221 | """ | |
222 | ||
223 | def _subst(match): | |
224 | return f'{{{match.group(1)}}}' | |
225 | ||
226 | repl = re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) | |
227 | if repl != s: | |
228 | import warnings | |
229 | ||
230 | warnings.warn( | |
231 | "shell/Perl-style substitions are deprecated", | |
232 | DeprecationWarning, | |
233 | ) | |
234 | return repl | |
235 | ||
236 | ||
237 | def grok_environment_error(exc, prefix="error: "): | |
238 | # Function kept for backward compatibility. | |
239 | # Used to try clever things with EnvironmentErrors, | |
240 | # but nowadays str(exception) produces good messages. | |
241 | return prefix + str(exc) | |
242 | ||
243 | ||
244 | # Needed by 'split_quoted()' | |
245 | _wordchars_re = _squote_re = _dquote_re = None | |
246 | ||
247 | ||
248 | def _init_regex(): | |
249 | global _wordchars_re, _squote_re, _dquote_re | |
250 | _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) | |
251 | _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") | |
252 | _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') | |
253 | ||
254 | ||
255 | def split_quoted(s): | |
256 | """Split a string up according to Unix shell-like rules for quotes and | |
257 | backslashes. In short: words are delimited by spaces, as long as those | |
258 | spaces are not escaped by a backslash, or inside a quoted string. | |
259 | Single and double quotes are equivalent, and the quote characters can | |
260 | be backslash-escaped. The backslash is stripped from any two-character | |
261 | escape sequence, leaving only the escaped character. The quote | |
262 | characters are stripped from any quoted string. Returns a list of | |
263 | words. | |
264 | """ | |
265 | ||
266 | # This is a nice algorithm for splitting up a single string, since it | |
267 | # doesn't require character-by-character examination. It was a little | |
268 | # bit of a brain-bender to get it working right, though... | |
269 | if _wordchars_re is None: | |
270 | _init_regex() | |
271 | ||
272 | s = s.strip() | |
273 | words = [] | |
274 | pos = 0 | |
275 | ||
276 | while s: | |
277 | m = _wordchars_re.match(s, pos) | |
278 | end = m.end() | |
279 | if end == len(s): | |
280 | words.append(s[:end]) | |
281 | break | |
282 | ||
283 | if s[end] in string.whitespace: | |
284 | # unescaped, unquoted whitespace: now | |
285 | # we definitely have a word delimiter | |
286 | words.append(s[:end]) | |
287 | s = s[end:].lstrip() | |
288 | pos = 0 | |
289 | ||
290 | elif s[end] == '\\': | |
291 | # preserve whatever is being escaped; | |
292 | # will become part of the current word | |
293 | s = s[:end] + s[end + 1 :] | |
294 | pos = end + 1 | |
295 | ||
296 | else: | |
297 | if s[end] == "'": # slurp singly-quoted string | |
298 | m = _squote_re.match(s, end) | |
299 | elif s[end] == '"': # slurp doubly-quoted string | |
300 | m = _dquote_re.match(s, end) | |
301 | else: | |
302 | raise RuntimeError("this can't happen (bad char '%c')" % s[end]) | |
303 | ||
304 | if m is None: | |
305 | raise ValueError("bad string (mismatched %s quotes?)" % s[end]) | |
306 | ||
307 | (beg, end) = m.span() | |
308 | s = s[:beg] + s[beg + 1 : end - 1] + s[end:] | |
309 | pos = m.end() - 2 | |
310 | ||
311 | if pos >= len(s): | |
312 | words.append(s) | |
313 | break | |
314 | ||
315 | return words | |
316 | ||
317 | ||
318 | # split_quoted () | |
319 | ||
320 | ||
321 | def execute(func, args, msg=None, verbose=0, dry_run=0): | |
322 | """Perform some action that affects the outside world (eg. by | |
323 | writing to the filesystem). Such actions are special because they | |
324 | are disabled by the 'dry_run' flag. This method takes care of all | |
325 | that bureaucracy for you; all you have to do is supply the | |
326 | function to call and an argument tuple for it (to embody the | |
327 | "external action" being performed), and an optional message to | |
328 | print. | |
329 | """ | |
330 | if msg is None: | |
331 | msg = "{}{!r}".format(func.__name__, args) | |
332 | if msg[-2:] == ',)': # correct for singleton tuple | |
333 | msg = msg[0:-2] + ')' | |
334 | ||
335 | log.info(msg) | |
336 | if not dry_run: | |
337 | func(*args) | |
338 | ||
339 | ||
340 | def strtobool(val): | |
341 | """Convert a string representation of truth to true (1) or false (0). | |
342 | ||
343 | True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values | |
344 | are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if | |
345 | 'val' is anything else. | |
346 | """ | |
347 | val = val.lower() | |
348 | if val in ('y', 'yes', 't', 'true', 'on', '1'): | |
349 | return 1 | |
350 | elif val in ('n', 'no', 'f', 'false', 'off', '0'): | |
351 | return 0 | |
352 | else: | |
353 | raise ValueError("invalid truth value {!r}".format(val)) | |
354 | ||
355 | ||
356 | def byte_compile( # noqa: C901 | |
357 | py_files, | |
358 | optimize=0, | |
359 | force=0, | |
360 | prefix=None, | |
361 | base_dir=None, | |
362 | verbose=1, | |
363 | dry_run=0, | |
364 | direct=None, | |
365 | ): | |
366 | """Byte-compile a collection of Python source files to .pyc | |
367 | files in a __pycache__ subdirectory. 'py_files' is a list | |
368 | of files to compile; any files that don't end in ".py" are silently | |
369 | skipped. 'optimize' must be one of the following: | |
370 | 0 - don't optimize | |
371 | 1 - normal optimization (like "python -O") | |
372 | 2 - extra optimization (like "python -OO") | |
373 | If 'force' is true, all files are recompiled regardless of | |
374 | timestamps. | |
375 | ||
376 | The source filename encoded in each bytecode file defaults to the | |
377 | filenames listed in 'py_files'; you can modify these with 'prefix' and | |
378 | 'basedir'. 'prefix' is a string that will be stripped off of each | |
379 | source filename, and 'base_dir' is a directory name that will be | |
380 | prepended (after 'prefix' is stripped). You can supply either or both | |
381 | (or neither) of 'prefix' and 'base_dir', as you wish. | |
382 | ||
383 | If 'dry_run' is true, doesn't actually do anything that would | |
384 | affect the filesystem. | |
385 | ||
386 | Byte-compilation is either done directly in this interpreter process | |
387 | with the standard py_compile module, or indirectly by writing a | |
388 | temporary script and executing it. Normally, you should let | |
389 | 'byte_compile()' figure out to use direct compilation or not (see | |
390 | the source for details). The 'direct' flag is used by the script | |
391 | generated in indirect mode; unless you know what you're doing, leave | |
392 | it set to None. | |
393 | """ | |
394 | ||
395 | # nothing is done if sys.dont_write_bytecode is True | |
396 | if sys.dont_write_bytecode: | |
397 | raise DistutilsByteCompileError('byte-compiling is disabled.') | |
398 | ||
399 | # First, if the caller didn't force us into direct or indirect mode, | |
400 | # figure out which mode we should be in. We take a conservative | |
401 | # approach: choose direct mode *only* if the current interpreter is | |
402 | # in debug mode and optimize is 0. If we're not in debug mode (-O | |
403 | # or -OO), we don't know which level of optimization this | |
404 | # interpreter is running with, so we can't do direct | |
405 | # byte-compilation and be certain that it's the right thing. Thus, | |
406 | # always compile indirectly if the current interpreter is in either | |
407 | # optimize mode, or if either optimization level was requested by | |
408 | # the caller. | |
409 | if direct is None: | |
410 | direct = __debug__ and optimize == 0 | |
411 | ||
412 | # "Indirect" byte-compilation: write a temporary script and then | |
413 | # run it with the appropriate flags. | |
414 | if not direct: | |
415 | try: | |
416 | from tempfile import mkstemp | |
417 | ||
418 | (script_fd, script_name) = mkstemp(".py") | |
419 | except ImportError: | |
420 | from tempfile import mktemp | |
421 | ||
422 | (script_fd, script_name) = None, mktemp(".py") | |
423 | log.info("writing byte-compilation script '%s'", script_name) | |
424 | if not dry_run: | |
425 | if script_fd is not None: | |
426 | script = os.fdopen(script_fd, "w") | |
427 | else: | |
428 | script = open(script_name, "w") | |
429 | ||
430 | with script: | |
431 | script.write( | |
432 | """\ | |
433 | from distutils.util import byte_compile | |
434 | files = [ | |
435 | """ | |
436 | ) | |
437 | ||
438 | # XXX would be nice to write absolute filenames, just for | |
439 | # safety's sake (script should be more robust in the face of | |
440 | # chdir'ing before running it). But this requires abspath'ing | |
441 | # 'prefix' as well, and that breaks the hack in build_lib's | |
442 | # 'byte_compile()' method that carefully tacks on a trailing | |
443 | # slash (os.sep really) to make sure the prefix here is "just | |
444 | # right". This whole prefix business is rather delicate -- the | |
445 | # problem is that it's really a directory, but I'm treating it | |
446 | # as a dumb string, so trailing slashes and so forth matter. | |
447 | ||
448 | script.write(",\n".join(map(repr, py_files)) + "]\n") | |
449 | script.write( | |
450 | """ | |
451 | byte_compile(files, optimize=%r, force=%r, | |
452 | prefix=%r, base_dir=%r, | |
453 | verbose=%r, dry_run=0, | |
454 | direct=1) | |
455 | """ | |
456 | % (optimize, force, prefix, base_dir, verbose) | |
457 | ) | |
458 | ||
459 | cmd = [sys.executable] | |
460 | cmd.extend(subprocess._optim_args_from_interpreter_flags()) | |
461 | cmd.append(script_name) | |
462 | spawn(cmd, dry_run=dry_run) | |
463 | execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run) | |
464 | ||
465 | # "Direct" byte-compilation: use the py_compile module to compile | |
466 | # right here, right now. Note that the script generated in indirect | |
467 | # mode simply calls 'byte_compile()' in direct mode, a weird sort of | |
468 | # cross-process recursion. Hey, it works! | |
469 | else: | |
470 | from py_compile import compile | |
471 | ||
472 | for file in py_files: | |
473 | if file[-3:] != ".py": | |
474 | # This lets us be lazy and not filter filenames in | |
475 | # the "install_lib" command. | |
476 | continue | |
477 | ||
478 | # Terminology from the py_compile module: | |
479 | # cfile - byte-compiled file | |
480 | # dfile - purported source filename (same as 'file' by default) | |
481 | if optimize >= 0: | |
482 | opt = '' if optimize == 0 else optimize | |
483 | cfile = importlib.util.cache_from_source(file, optimization=opt) | |
484 | else: | |
485 | cfile = importlib.util.cache_from_source(file) | |
486 | dfile = file | |
487 | if prefix: | |
488 | if file[: len(prefix)] != prefix: | |
489 | raise ValueError( | |
490 | "invalid prefix: filename %r doesn't start with %r" | |
491 | % (file, prefix) | |
492 | ) | |
493 | dfile = dfile[len(prefix) :] | |
494 | if base_dir: | |
495 | dfile = os.path.join(base_dir, dfile) | |
496 | ||
497 | cfile_base = os.path.basename(cfile) | |
498 | if direct: | |
499 | if force or newer(file, cfile): | |
500 | log.info("byte-compiling %s to %s", file, cfile_base) | |
501 | if not dry_run: | |
502 | compile(file, cfile, dfile) | |
503 | else: | |
504 | log.debug("skipping byte-compilation of %s to %s", file, cfile_base) | |
505 | ||
506 | ||
507 | def rfc822_escape(header): | |
508 | """Return a version of the string escaped for inclusion in an | |
509 | RFC-822 header, by ensuring there are 8 spaces space after each newline. | |
510 | """ | |
511 | lines = header.split('\n') | |
512 | sep = '\n' + 8 * ' ' | |
513 | return sep.join(lines) |