]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | """ |
2 | Load setuptools configuration from ``pyproject.toml`` files. | |
3 | ||
4 | **PRIVATE MODULE**: API reserved for setuptools internal usage only. | |
5 | """ | |
6 | import logging | |
7 | import os | |
8 | import warnings | |
9 | from contextlib import contextmanager | |
10 | from functools import partial | |
11 | from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Set, Union | |
12 | ||
13 | from setuptools.errors import FileError, OptionError | |
14 | ||
15 | from . import expand as _expand | |
16 | from ._apply_pyprojecttoml import apply as _apply | |
17 | from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _WouldIgnoreField | |
18 | ||
19 | if TYPE_CHECKING: | |
20 | from setuptools.dist import Distribution # noqa | |
21 | ||
22 | _Path = Union[str, os.PathLike] | |
23 | _logger = logging.getLogger(__name__) | |
24 | ||
25 | ||
26 | def load_file(filepath: _Path) -> dict: | |
27 | from setuptools.extern import tomli # type: ignore | |
28 | ||
29 | with open(filepath, "rb") as file: | |
30 | return tomli.load(file) | |
31 | ||
32 | ||
33 | def validate(config: dict, filepath: _Path) -> bool: | |
34 | from . import _validate_pyproject as validator | |
35 | ||
36 | trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier") | |
37 | if hasattr(trove_classifier, "_disable_download"): | |
38 | # Improve reproducibility by default. See issue 31 for validate-pyproject. | |
39 | trove_classifier._disable_download() # type: ignore | |
40 | ||
41 | try: | |
42 | return validator.validate(config) | |
43 | except validator.ValidationError as ex: | |
44 | summary = f"configuration error: {ex.summary}" | |
45 | if ex.name.strip("`") != "project": | |
46 | # Probably it is just a field missing/misnamed, not worthy the verbosity... | |
47 | _logger.debug(summary) | |
48 | _logger.debug(ex.details) | |
49 | ||
50 | error = f"invalid pyproject.toml config: {ex.name}." | |
51 | raise ValueError(f"{error}\n{summary}") from None | |
52 | ||
53 | ||
54 | def apply_configuration( | |
55 | dist: "Distribution", | |
56 | filepath: _Path, | |
57 | ignore_option_errors=False, | |
58 | ) -> "Distribution": | |
59 | """Apply the configuration from a ``pyproject.toml`` file into an existing | |
60 | distribution object. | |
61 | """ | |
62 | config = read_configuration(filepath, True, ignore_option_errors, dist) | |
63 | return _apply(dist, config, filepath) | |
64 | ||
65 | ||
66 | def read_configuration( | |
67 | filepath: _Path, | |
68 | expand=True, | |
69 | ignore_option_errors=False, | |
70 | dist: Optional["Distribution"] = None, | |
71 | ): | |
72 | """Read given configuration file and returns options from it as a dict. | |
73 | ||
74 | :param str|unicode filepath: Path to configuration file in the ``pyproject.toml`` | |
75 | format. | |
76 | ||
77 | :param bool expand: Whether to expand directives and other computed values | |
78 | (i.e. post-process the given configuration) | |
79 | ||
80 | :param bool ignore_option_errors: Whether to silently ignore | |
81 | options, values of which could not be resolved (e.g. due to exceptions | |
82 | in directives such as file:, attr:, etc.). | |
83 | If False exceptions are propagated as expected. | |
84 | ||
85 | :param Distribution|None: Distribution object to which the configuration refers. | |
86 | If not given a dummy object will be created and discarded after the | |
87 | configuration is read. This is used for auto-discovery of packages and in the | |
88 | case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded. | |
89 | When ``expand=False`` this object is simply ignored. | |
90 | ||
91 | :rtype: dict | |
92 | """ | |
93 | filepath = os.path.abspath(filepath) | |
94 | ||
95 | if not os.path.isfile(filepath): | |
96 | raise FileError(f"Configuration file {filepath!r} does not exist.") | |
97 | ||
98 | asdict = load_file(filepath) or {} | |
99 | project_table = asdict.get("project", {}) | |
100 | tool_table = asdict.get("tool", {}) | |
101 | setuptools_table = tool_table.get("setuptools", {}) | |
102 | if not asdict or not (project_table or setuptools_table): | |
103 | return {} # User is not using pyproject to configure setuptools | |
104 | ||
105 | if setuptools_table: | |
106 | # TODO: Remove the following once the feature stabilizes: | |
107 | msg = "Support for `[tool.setuptools]` in `pyproject.toml` is still *beta*." | |
108 | warnings.warn(msg, _BetaConfiguration) | |
109 | ||
110 | # There is an overall sense in the community that making include_package_data=True | |
111 | # the default would be an improvement. | |
112 | # `ini2toml` backfills include_package_data=False when nothing is explicitly given, | |
113 | # therefore setting a default here is backwards compatible. | |
114 | orig_setuptools_table = setuptools_table.copy() | |
115 | if dist and getattr(dist, "include_package_data", None) is not None: | |
116 | setuptools_table.setdefault("include-package-data", dist.include_package_data) | |
117 | else: | |
118 | setuptools_table.setdefault("include-package-data", True) | |
119 | # Persist changes: | |
120 | asdict["tool"] = tool_table | |
121 | tool_table["setuptools"] = setuptools_table | |
122 | ||
123 | try: | |
124 | # Don't complain about unrelated errors (e.g. tools not using the "tool" table) | |
125 | subset = {"project": project_table, "tool": {"setuptools": setuptools_table}} | |
126 | validate(subset, filepath) | |
127 | except Exception as ex: | |
128 | # TODO: Remove the following once the feature stabilizes: | |
129 | if _skip_bad_config(project_table, orig_setuptools_table, dist): | |
130 | return {} | |
131 | # TODO: After the previous statement is removed the try/except can be replaced | |
132 | # by the _ignore_errors context manager. | |
133 | if ignore_option_errors: | |
134 | _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}") | |
135 | else: | |
136 | raise # re-raise exception | |
137 | ||
138 | if expand: | |
139 | root_dir = os.path.dirname(filepath) | |
140 | return expand_configuration(asdict, root_dir, ignore_option_errors, dist) | |
141 | ||
142 | return asdict | |
143 | ||
144 | ||
145 | def _skip_bad_config( | |
146 | project_cfg: dict, setuptools_cfg: dict, dist: Optional["Distribution"] | |
147 | ) -> bool: | |
148 | """Be temporarily forgiving with invalid ``pyproject.toml``""" | |
149 | # See pypa/setuptools#3199 and pypa/cibuildwheel#1064 | |
150 | ||
151 | if dist is None or ( | |
152 | dist.metadata.name is None | |
153 | and dist.metadata.version is None | |
154 | and dist.install_requires is None | |
155 | ): | |
156 | # It seems that the build is not getting any configuration from other places | |
157 | return False | |
158 | ||
159 | if setuptools_cfg: | |
160 | # If `[tool.setuptools]` is set, then `pyproject.toml` config is intentional | |
161 | return False | |
162 | ||
163 | given_config = set(project_cfg.keys()) | |
164 | popular_subset = {"name", "version", "python_requires", "requires-python"} | |
165 | if given_config <= popular_subset: | |
166 | # It seems that the docs in cibuildtool has been inadvertently encouraging users | |
167 | # to create `pyproject.toml` files that are not compliant with the standards. | |
168 | # Let's be forgiving for the time being. | |
169 | warnings.warn(_InvalidFile.message(), _InvalidFile, stacklevel=2) | |
170 | return True | |
171 | ||
172 | return False | |
173 | ||
174 | ||
175 | def expand_configuration( | |
176 | config: dict, | |
177 | root_dir: Optional[_Path] = None, | |
178 | ignore_option_errors: bool = False, | |
179 | dist: Optional["Distribution"] = None, | |
180 | ) -> dict: | |
181 | """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...) | |
182 | find their final values. | |
183 | ||
184 | :param dict config: Dict containing the configuration for the distribution | |
185 | :param str root_dir: Top-level directory for the distribution/project | |
186 | (the same directory where ``pyproject.toml`` is place) | |
187 | :param bool ignore_option_errors: see :func:`read_configuration` | |
188 | :param Distribution|None: Distribution object to which the configuration refers. | |
189 | If not given a dummy object will be created and discarded after the | |
190 | configuration is read. Used in the case a dynamic configuration | |
191 | (e.g. ``attr`` or ``cmdclass``). | |
192 | ||
193 | :rtype: dict | |
194 | """ | |
195 | return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand() | |
196 | ||
197 | ||
198 | class _ConfigExpander: | |
199 | def __init__( | |
200 | self, | |
201 | config: dict, | |
202 | root_dir: Optional[_Path] = None, | |
203 | ignore_option_errors: bool = False, | |
204 | dist: Optional["Distribution"] = None, | |
205 | ): | |
206 | self.config = config | |
207 | self.root_dir = root_dir or os.getcwd() | |
208 | self.project_cfg = config.get("project", {}) | |
209 | self.dynamic = self.project_cfg.get("dynamic", []) | |
210 | self.setuptools_cfg = config.get("tool", {}).get("setuptools", {}) | |
211 | self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {}) | |
212 | self.ignore_option_errors = ignore_option_errors | |
213 | self._dist = dist | |
214 | self._referenced_files: Set[str] = set() | |
215 | ||
216 | def _ensure_dist(self) -> "Distribution": | |
217 | from setuptools.dist import Distribution | |
218 | ||
219 | attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)} | |
220 | return self._dist or Distribution(attrs) | |
221 | ||
222 | def _process_field(self, container: dict, field: str, fn: Callable): | |
223 | if field in container: | |
224 | with _ignore_errors(self.ignore_option_errors): | |
225 | container[field] = fn(container[field]) | |
226 | ||
227 | def _canonic_package_data(self, field="package-data"): | |
228 | package_data = self.setuptools_cfg.get(field, {}) | |
229 | return _expand.canonic_package_data(package_data) | |
230 | ||
231 | def expand(self): | |
232 | self._expand_packages() | |
233 | self._canonic_package_data() | |
234 | self._canonic_package_data("exclude-package-data") | |
235 | ||
236 | # A distribution object is required for discovering the correct package_dir | |
237 | dist = self._ensure_dist() | |
238 | ctx = _EnsurePackagesDiscovered(dist, self.project_cfg, self.setuptools_cfg) | |
239 | with ctx as ensure_discovered: | |
240 | package_dir = ensure_discovered.package_dir | |
241 | self._expand_data_files() | |
242 | self._expand_cmdclass(package_dir) | |
243 | self._expand_all_dynamic(dist, package_dir) | |
244 | ||
245 | dist._referenced_files.update(self._referenced_files) | |
246 | return self.config | |
247 | ||
248 | def _expand_packages(self): | |
249 | packages = self.setuptools_cfg.get("packages") | |
250 | if packages is None or isinstance(packages, (list, tuple)): | |
251 | return | |
252 | ||
253 | find = packages.get("find") | |
254 | if isinstance(find, dict): | |
255 | find["root_dir"] = self.root_dir | |
256 | find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {}) | |
257 | with _ignore_errors(self.ignore_option_errors): | |
258 | self.setuptools_cfg["packages"] = _expand.find_packages(**find) | |
259 | ||
260 | def _expand_data_files(self): | |
261 | data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir) | |
262 | self._process_field(self.setuptools_cfg, "data-files", data_files) | |
263 | ||
264 | def _expand_cmdclass(self, package_dir: Mapping[str, str]): | |
265 | root_dir = self.root_dir | |
266 | cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir) | |
267 | self._process_field(self.setuptools_cfg, "cmdclass", cmdclass) | |
268 | ||
269 | def _expand_all_dynamic(self, dist: "Distribution", package_dir: Mapping[str, str]): | |
270 | special = ( # need special handling | |
271 | "version", | |
272 | "readme", | |
273 | "entry-points", | |
274 | "scripts", | |
275 | "gui-scripts", | |
276 | "classifiers", | |
277 | "dependencies", | |
278 | "optional-dependencies", | |
279 | ) | |
280 | # `_obtain` functions are assumed to raise appropriate exceptions/warnings. | |
281 | obtained_dynamic = { | |
282 | field: self._obtain(dist, field, package_dir) | |
283 | for field in self.dynamic | |
284 | if field not in special | |
285 | } | |
286 | obtained_dynamic.update( | |
287 | self._obtain_entry_points(dist, package_dir) or {}, | |
288 | version=self._obtain_version(dist, package_dir), | |
289 | readme=self._obtain_readme(dist), | |
290 | classifiers=self._obtain_classifiers(dist), | |
291 | dependencies=self._obtain_dependencies(dist), | |
292 | optional_dependencies=self._obtain_optional_dependencies(dist), | |
293 | ) | |
294 | # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value | |
295 | # might have already been set by setup.py/extensions, so avoid overwriting. | |
296 | updates = {k: v for k, v in obtained_dynamic.items() if v is not None} | |
297 | self.project_cfg.update(updates) | |
298 | ||
299 | def _ensure_previously_set(self, dist: "Distribution", field: str): | |
300 | previous = _PREVIOUSLY_DEFINED[field](dist) | |
301 | if previous is None and not self.ignore_option_errors: | |
302 | msg = ( | |
303 | f"No configuration found for dynamic {field!r}.\n" | |
304 | "Some dynamic fields need to be specified via `tool.setuptools.dynamic`" | |
305 | "\nothers must be specified via the equivalent attribute in `setup.py`." | |
306 | ) | |
307 | raise OptionError(msg) | |
308 | ||
309 | def _expand_directive( | |
310 | self, specifier: str, directive, package_dir: Mapping[str, str] | |
311 | ): | |
312 | from setuptools.extern.more_itertools import always_iterable # type: ignore | |
313 | ||
314 | with _ignore_errors(self.ignore_option_errors): | |
315 | root_dir = self.root_dir | |
316 | if "file" in directive: | |
317 | self._referenced_files.update(always_iterable(directive["file"])) | |
318 | return _expand.read_files(directive["file"], root_dir) | |
319 | if "attr" in directive: | |
320 | return _expand.read_attr(directive["attr"], package_dir, root_dir) | |
321 | raise ValueError(f"invalid `{specifier}`: {directive!r}") | |
322 | return None | |
323 | ||
324 | def _obtain(self, dist: "Distribution", field: str, package_dir: Mapping[str, str]): | |
325 | if field in self.dynamic_cfg: | |
326 | return self._expand_directive( | |
327 | f"tool.setuptools.dynamic.{field}", | |
328 | self.dynamic_cfg[field], | |
329 | package_dir, | |
330 | ) | |
331 | self._ensure_previously_set(dist, field) | |
332 | return None | |
333 | ||
334 | def _obtain_version(self, dist: "Distribution", package_dir: Mapping[str, str]): | |
335 | # Since plugins can set version, let's silently skip if it cannot be obtained | |
336 | if "version" in self.dynamic and "version" in self.dynamic_cfg: | |
337 | return _expand.version(self._obtain(dist, "version", package_dir)) | |
338 | return None | |
339 | ||
340 | def _obtain_readme(self, dist: "Distribution") -> Optional[Dict[str, str]]: | |
341 | if "readme" not in self.dynamic: | |
342 | return None | |
343 | ||
344 | dynamic_cfg = self.dynamic_cfg | |
345 | if "readme" in dynamic_cfg: | |
346 | return { | |
347 | "text": self._obtain(dist, "readme", {}), | |
348 | "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"), | |
349 | } | |
350 | ||
351 | self._ensure_previously_set(dist, "readme") | |
352 | return None | |
353 | ||
354 | def _obtain_entry_points( | |
355 | self, dist: "Distribution", package_dir: Mapping[str, str] | |
356 | ) -> Optional[Dict[str, dict]]: | |
357 | fields = ("entry-points", "scripts", "gui-scripts") | |
358 | if not any(field in self.dynamic for field in fields): | |
359 | return None | |
360 | ||
361 | text = self._obtain(dist, "entry-points", package_dir) | |
362 | if text is None: | |
363 | return None | |
364 | ||
365 | groups = _expand.entry_points(text) | |
366 | expanded = {"entry-points": groups} | |
367 | ||
368 | def _set_scripts(field: str, group: str): | |
369 | if group in groups: | |
370 | value = groups.pop(group) | |
371 | if field not in self.dynamic: | |
372 | msg = _WouldIgnoreField.message(field, value) | |
373 | warnings.warn(msg, _WouldIgnoreField) | |
374 | # TODO: Don't set field when support for pyproject.toml stabilizes | |
375 | # instead raise an error as specified in PEP 621 | |
376 | expanded[field] = value | |
377 | ||
378 | _set_scripts("scripts", "console_scripts") | |
379 | _set_scripts("gui-scripts", "gui_scripts") | |
380 | ||
381 | return expanded | |
382 | ||
383 | def _obtain_classifiers(self, dist: "Distribution"): | |
384 | if "classifiers" in self.dynamic: | |
385 | value = self._obtain(dist, "classifiers", {}) | |
386 | if value: | |
387 | return value.splitlines() | |
388 | return None | |
389 | ||
390 | def _obtain_dependencies(self, dist: "Distribution"): | |
391 | if "dependencies" in self.dynamic: | |
392 | value = self._obtain(dist, "dependencies", {}) | |
393 | if value: | |
394 | return _parse_requirements_list(value) | |
395 | return None | |
396 | ||
397 | def _obtain_optional_dependencies(self, dist: "Distribution"): | |
398 | if "optional-dependencies" not in self.dynamic: | |
399 | return None | |
400 | if "optional-dependencies" in self.dynamic_cfg: | |
401 | optional_dependencies_map = self.dynamic_cfg["optional-dependencies"] | |
402 | assert isinstance(optional_dependencies_map, dict) | |
403 | return { | |
404 | group: _parse_requirements_list(self._expand_directive( | |
405 | f"tool.setuptools.dynamic.optional-dependencies.{group}", | |
406 | directive, | |
407 | {}, | |
408 | )) | |
409 | for group, directive in optional_dependencies_map.items() | |
410 | } | |
411 | self._ensure_previously_set(dist, "optional-dependencies") | |
412 | return None | |
413 | ||
414 | ||
415 | def _parse_requirements_list(value): | |
416 | return [ | |
417 | line | |
418 | for line in value.splitlines() | |
419 | if line.strip() and not line.strip().startswith("#") | |
420 | ] | |
421 | ||
422 | ||
423 | @contextmanager | |
424 | def _ignore_errors(ignore_option_errors: bool): | |
425 | if not ignore_option_errors: | |
426 | yield | |
427 | return | |
428 | ||
429 | try: | |
430 | yield | |
431 | except Exception as ex: | |
432 | _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}") | |
433 | ||
434 | ||
435 | class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered): | |
436 | def __init__( | |
437 | self, distribution: "Distribution", project_cfg: dict, setuptools_cfg: dict | |
438 | ): | |
439 | super().__init__(distribution) | |
440 | self._project_cfg = project_cfg | |
441 | self._setuptools_cfg = setuptools_cfg | |
442 | ||
443 | def __enter__(self): | |
444 | """When entering the context, the values of ``packages``, ``py_modules`` and | |
445 | ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``. | |
446 | """ | |
447 | dist, cfg = self._dist, self._setuptools_cfg | |
448 | package_dir: Dict[str, str] = cfg.setdefault("package-dir", {}) | |
449 | package_dir.update(dist.package_dir or {}) | |
450 | dist.package_dir = package_dir # needs to be the same object | |
451 | ||
452 | dist.set_defaults._ignore_ext_modules() # pyproject.toml-specific behaviour | |
453 | ||
454 | # Set `name`, `py_modules` and `packages` in dist to short-circuit | |
455 | # auto-discovery, but avoid overwriting empty lists purposefully set by users. | |
456 | if dist.metadata.name is None: | |
457 | dist.metadata.name = self._project_cfg.get("name") | |
458 | if dist.py_modules is None: | |
459 | dist.py_modules = cfg.get("py-modules") | |
460 | if dist.packages is None: | |
461 | dist.packages = cfg.get("packages") | |
462 | ||
463 | return super().__enter__() | |
464 | ||
465 | def __exit__(self, exc_type, exc_value, traceback): | |
466 | """When exiting the context, if values of ``packages``, ``py_modules`` and | |
467 | ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``. | |
468 | """ | |
469 | # If anything was discovered set them back, so they count in the final config. | |
470 | self._setuptools_cfg.setdefault("packages", self._dist.packages) | |
471 | self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules) | |
472 | return super().__exit__(exc_type, exc_value, traceback) | |
473 | ||
474 | ||
475 | class _BetaConfiguration(UserWarning): | |
476 | """Explicitly inform users that some `pyproject.toml` configuration is *beta*""" | |
477 | ||
478 | ||
479 | class _InvalidFile(UserWarning): | |
480 | """The given `pyproject.toml` file is invalid and would be ignored. | |
481 | !!\n\n | |
482 | ############################ | |
483 | # Invalid `pyproject.toml` # | |
484 | ############################ | |
485 | ||
486 | Any configurations in `pyproject.toml` will be ignored. | |
487 | Please note that future releases of setuptools will halt the build process | |
488 | if an invalid file is given. | |
489 | ||
490 | To prevent setuptools from considering `pyproject.toml` please | |
491 | DO NOT include the `[project]` or `[tool.setuptools]` tables in your file. | |
492 | \n\n!! | |
493 | """ | |
494 | ||
495 | @classmethod | |
496 | def message(cls): | |
497 | from inspect import cleandoc | |
498 | return cleandoc(cls.__doc__) |