]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | from functools import partial |
2 | from glob import glob | |
3 | from distutils.util import convert_path | |
4 | import distutils.command.build_py as orig | |
5 | import os | |
6 | import fnmatch | |
7 | import textwrap | |
8 | import io | |
9 | import distutils.errors | |
10 | import itertools | |
11 | import stat | |
12 | import warnings | |
13 | from pathlib import Path | |
14 | from typing import Dict, Iterable, Iterator, List, Optional, Tuple | |
15 | ||
16 | from setuptools._deprecation_warning import SetuptoolsDeprecationWarning | |
17 | from setuptools.extern.more_itertools import unique_everseen | |
18 | ||
19 | ||
20 | def make_writable(target): | |
21 | os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE) | |
22 | ||
23 | ||
24 | class build_py(orig.build_py): | |
25 | """Enhanced 'build_py' command that includes data files with packages | |
26 | ||
27 | The data files are specified via a 'package_data' argument to 'setup()'. | |
28 | See 'setuptools.dist.Distribution' for more details. | |
29 | ||
30 | Also, this version of the 'build_py' command allows you to specify both | |
31 | 'py_modules' and 'packages' in the same setup operation. | |
32 | """ | |
33 | editable_mode: bool = False | |
34 | existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. | |
35 | ||
36 | def finalize_options(self): | |
37 | orig.build_py.finalize_options(self) | |
38 | self.package_data = self.distribution.package_data | |
39 | self.exclude_package_data = self.distribution.exclude_package_data or {} | |
40 | if 'data_files' in self.__dict__: | |
41 | del self.__dict__['data_files'] | |
42 | self.__updated_files = [] | |
43 | ||
44 | def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, | |
45 | link=None, level=1): | |
46 | # Overwrite base class to allow using links | |
47 | if link: | |
48 | infile = str(Path(infile).resolve()) | |
49 | outfile = str(Path(outfile).resolve()) | |
50 | return super().copy_file(infile, outfile, preserve_mode, preserve_times, | |
51 | link, level) | |
52 | ||
53 | def run(self): | |
54 | """Build modules, packages, and copy data files to build directory""" | |
55 | if not (self.py_modules or self.packages) or self.editable_mode: | |
56 | return | |
57 | ||
58 | if self.py_modules: | |
59 | self.build_modules() | |
60 | ||
61 | if self.packages: | |
62 | self.build_packages() | |
63 | self.build_package_data() | |
64 | ||
65 | # Only compile actual .py files, using our base class' idea of what our | |
66 | # output files are. | |
67 | self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0)) | |
68 | ||
69 | def __getattr__(self, attr): | |
70 | "lazily compute data files" | |
71 | if attr == 'data_files': | |
72 | self.data_files = self._get_data_files() | |
73 | return self.data_files | |
74 | return orig.build_py.__getattr__(self, attr) | |
75 | ||
76 | def build_module(self, module, module_file, package): | |
77 | outfile, copied = orig.build_py.build_module(self, module, module_file, package) | |
78 | if copied: | |
79 | self.__updated_files.append(outfile) | |
80 | return outfile, copied | |
81 | ||
82 | def _get_data_files(self): | |
83 | """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" | |
84 | self.analyze_manifest() | |
85 | return list(map(self._get_pkg_data_files, self.packages or ())) | |
86 | ||
87 | def get_data_files_without_manifest(self): | |
88 | """ | |
89 | Generate list of ``(package,src_dir,build_dir,filenames)`` tuples, | |
90 | but without triggering any attempt to analyze or build the manifest. | |
91 | """ | |
92 | # Prevent eventual errors from unset `manifest_files` | |
93 | # (that would otherwise be set by `analyze_manifest`) | |
94 | self.__dict__.setdefault('manifest_files', {}) | |
95 | return list(map(self._get_pkg_data_files, self.packages or ())) | |
96 | ||
97 | def _get_pkg_data_files(self, package): | |
98 | # Locate package source directory | |
99 | src_dir = self.get_package_dir(package) | |
100 | ||
101 | # Compute package build directory | |
102 | build_dir = os.path.join(*([self.build_lib] + package.split('.'))) | |
103 | ||
104 | # Strip directory from globbed filenames | |
105 | filenames = [ | |
106 | os.path.relpath(file, src_dir) | |
107 | for file in self.find_data_files(package, src_dir) | |
108 | ] | |
109 | return package, src_dir, build_dir, filenames | |
110 | ||
111 | def find_data_files(self, package, src_dir): | |
112 | """Return filenames for package's data files in 'src_dir'""" | |
113 | patterns = self._get_platform_patterns( | |
114 | self.package_data, | |
115 | package, | |
116 | src_dir, | |
117 | ) | |
118 | globs_expanded = map(partial(glob, recursive=True), patterns) | |
119 | # flatten the expanded globs into an iterable of matches | |
120 | globs_matches = itertools.chain.from_iterable(globs_expanded) | |
121 | glob_files = filter(os.path.isfile, globs_matches) | |
122 | files = itertools.chain( | |
123 | self.manifest_files.get(package, []), | |
124 | glob_files, | |
125 | ) | |
126 | return self.exclude_data_files(package, src_dir, files) | |
127 | ||
128 | def get_outputs(self, include_bytecode=1) -> List[str]: | |
129 | """See :class:`setuptools.commands.build.SubCommand`""" | |
130 | if self.editable_mode: | |
131 | return list(self.get_output_mapping().keys()) | |
132 | return super().get_outputs(include_bytecode) | |
133 | ||
134 | def get_output_mapping(self) -> Dict[str, str]: | |
135 | """See :class:`setuptools.commands.build.SubCommand`""" | |
136 | mapping = itertools.chain( | |
137 | self._get_package_data_output_mapping(), | |
138 | self._get_module_mapping(), | |
139 | ) | |
140 | return dict(sorted(mapping, key=lambda x: x[0])) | |
141 | ||
142 | def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: | |
143 | """Iterate over all modules producing (dest, src) pairs.""" | |
144 | for (package, module, module_file) in self.find_all_modules(): | |
145 | package = package.split('.') | |
146 | filename = self.get_module_outfile(self.build_lib, package, module) | |
147 | yield (filename, module_file) | |
148 | ||
149 | def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: | |
150 | """Iterate over package data producing (dest, src) pairs.""" | |
151 | for package, src_dir, build_dir, filenames in self.data_files: | |
152 | for filename in filenames: | |
153 | target = os.path.join(build_dir, filename) | |
154 | srcfile = os.path.join(src_dir, filename) | |
155 | yield (target, srcfile) | |
156 | ||
157 | def build_package_data(self): | |
158 | """Copy data files into build directory""" | |
159 | for target, srcfile in self._get_package_data_output_mapping(): | |
160 | self.mkpath(os.path.dirname(target)) | |
161 | _outf, _copied = self.copy_file(srcfile, target) | |
162 | make_writable(target) | |
163 | ||
164 | def analyze_manifest(self): | |
165 | self.manifest_files = mf = {} | |
166 | if not self.distribution.include_package_data: | |
167 | return | |
168 | src_dirs = {} | |
169 | for package in self.packages or (): | |
170 | # Locate package source directory | |
171 | src_dirs[assert_relative(self.get_package_dir(package))] = package | |
172 | ||
173 | if ( | |
174 | getattr(self, 'existing_egg_info_dir', None) | |
175 | and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() | |
176 | ): | |
177 | egg_info_dir = self.existing_egg_info_dir | |
178 | manifest = Path(egg_info_dir, "SOURCES.txt") | |
179 | files = manifest.read_text(encoding="utf-8").splitlines() | |
180 | else: | |
181 | self.run_command('egg_info') | |
182 | ei_cmd = self.get_finalized_command('egg_info') | |
183 | egg_info_dir = ei_cmd.egg_info | |
184 | files = ei_cmd.filelist.files | |
185 | ||
186 | check = _IncludePackageDataAbuse() | |
187 | for path in self._filter_build_files(files, egg_info_dir): | |
188 | d, f = os.path.split(assert_relative(path)) | |
189 | prev = None | |
190 | oldf = f | |
191 | while d and d != prev and d not in src_dirs: | |
192 | prev = d | |
193 | d, df = os.path.split(d) | |
194 | f = os.path.join(df, f) | |
195 | if d in src_dirs: | |
196 | if f == oldf: | |
197 | if check.is_module(f): | |
198 | continue # it's a module, not data | |
199 | else: | |
200 | importable = check.importable_subpackage(src_dirs[d], f) | |
201 | if importable: | |
202 | check.warn(importable) | |
203 | mf.setdefault(src_dirs[d], []).append(path) | |
204 | ||
205 | def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]: | |
206 | """ | |
207 | ``build_meta`` may try to create egg_info outside of the project directory, | |
208 | and this can be problematic for certain plugins (reported in issue #3500). | |
209 | ||
210 | Extensions might also include between their sources files created on the | |
211 | ``build_lib`` and ``build_temp`` directories. | |
212 | ||
213 | This function should filter this case of invalid files out. | |
214 | """ | |
215 | build = self.get_finalized_command("build") | |
216 | build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base) | |
217 | norm_dirs = [os.path.normpath(p) for p in build_dirs if p] | |
218 | ||
219 | for file in files: | |
220 | norm_path = os.path.normpath(file) | |
221 | if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs): | |
222 | yield file | |
223 | ||
224 | def get_data_files(self): | |
225 | pass # Lazily compute data files in _get_data_files() function. | |
226 | ||
227 | def check_package(self, package, package_dir): | |
228 | """Check namespace packages' __init__ for declare_namespace""" | |
229 | try: | |
230 | return self.packages_checked[package] | |
231 | except KeyError: | |
232 | pass | |
233 | ||
234 | init_py = orig.build_py.check_package(self, package, package_dir) | |
235 | self.packages_checked[package] = init_py | |
236 | ||
237 | if not init_py or not self.distribution.namespace_packages: | |
238 | return init_py | |
239 | ||
240 | for pkg in self.distribution.namespace_packages: | |
241 | if pkg == package or pkg.startswith(package + '.'): | |
242 | break | |
243 | else: | |
244 | return init_py | |
245 | ||
246 | with io.open(init_py, 'rb') as f: | |
247 | contents = f.read() | |
248 | if b'declare_namespace' not in contents: | |
249 | raise distutils.errors.DistutilsError( | |
250 | "Namespace package problem: %s is a namespace package, but " | |
251 | "its\n__init__.py does not call declare_namespace()! Please " | |
252 | 'fix it.\n(See the setuptools manual under ' | |
253 | '"Namespace Packages" for details.)\n"' % (package,) | |
254 | ) | |
255 | return init_py | |
256 | ||
257 | def initialize_options(self): | |
258 | self.packages_checked = {} | |
259 | orig.build_py.initialize_options(self) | |
260 | self.editable_mode = False | |
261 | self.existing_egg_info_dir = None | |
262 | ||
263 | def get_package_dir(self, package): | |
264 | res = orig.build_py.get_package_dir(self, package) | |
265 | if self.distribution.src_root is not None: | |
266 | return os.path.join(self.distribution.src_root, res) | |
267 | return res | |
268 | ||
269 | def exclude_data_files(self, package, src_dir, files): | |
270 | """Filter filenames for package's data files in 'src_dir'""" | |
271 | files = list(files) | |
272 | patterns = self._get_platform_patterns( | |
273 | self.exclude_package_data, | |
274 | package, | |
275 | src_dir, | |
276 | ) | |
277 | match_groups = (fnmatch.filter(files, pattern) for pattern in patterns) | |
278 | # flatten the groups of matches into an iterable of matches | |
279 | matches = itertools.chain.from_iterable(match_groups) | |
280 | bad = set(matches) | |
281 | keepers = (fn for fn in files if fn not in bad) | |
282 | # ditch dupes | |
283 | return list(unique_everseen(keepers)) | |
284 | ||
285 | @staticmethod | |
286 | def _get_platform_patterns(spec, package, src_dir): | |
287 | """ | |
288 | yield platform-specific path patterns (suitable for glob | |
289 | or fn_match) from a glob-based spec (such as | |
290 | self.package_data or self.exclude_package_data) | |
291 | matching package in src_dir. | |
292 | """ | |
293 | raw_patterns = itertools.chain( | |
294 | spec.get('', []), | |
295 | spec.get(package, []), | |
296 | ) | |
297 | return ( | |
298 | # Each pattern has to be converted to a platform-specific path | |
299 | os.path.join(src_dir, convert_path(pattern)) | |
300 | for pattern in raw_patterns | |
301 | ) | |
302 | ||
303 | ||
304 | def assert_relative(path): | |
305 | if not os.path.isabs(path): | |
306 | return path | |
307 | from distutils.errors import DistutilsSetupError | |
308 | ||
309 | msg = ( | |
310 | textwrap.dedent( | |
311 | """ | |
312 | Error: setup script specifies an absolute path: | |
313 | ||
314 | %s | |
315 | ||
316 | setup() arguments must *always* be /-separated paths relative to the | |
317 | setup.py directory, *never* absolute paths. | |
318 | """ | |
319 | ).lstrip() | |
320 | % path | |
321 | ) | |
322 | raise DistutilsSetupError(msg) | |
323 | ||
324 | ||
325 | class _IncludePackageDataAbuse: | |
326 | """Inform users that package or module is included as 'data file'""" | |
327 | ||
328 | MESSAGE = """\ | |
329 | Installing {importable!r} as data is deprecated, please list it in `packages`. | |
330 | !!\n\n | |
331 | ############################ | |
332 | # Package would be ignored # | |
333 | ############################ | |
334 | Python recognizes {importable!r} as an importable package, | |
335 | but it is not listed in the `packages` configuration of setuptools. | |
336 | ||
337 | {importable!r} has been automatically added to the distribution only | |
338 | because it may contain data files, but this behavior is likely to change | |
339 | in future versions of setuptools (and therefore is considered deprecated). | |
340 | ||
341 | Please make sure that {importable!r} is included as a package by using | |
342 | the `packages` configuration field or the proper discovery methods | |
343 | (for example by using `find_namespace_packages(...)`/`find_namespace:` | |
344 | instead of `find_packages(...)`/`find:`). | |
345 | ||
346 | You can read more about "package discovery" and "data files" on setuptools | |
347 | documentation page. | |
348 | \n\n!! | |
349 | """ | |
350 | ||
351 | def __init__(self): | |
352 | self._already_warned = set() | |
353 | ||
354 | def is_module(self, file): | |
355 | return file.endswith(".py") and file[:-len(".py")].isidentifier() | |
356 | ||
357 | def importable_subpackage(self, parent, file): | |
358 | pkg = Path(file).parent | |
359 | parts = list(itertools.takewhile(str.isidentifier, pkg.parts)) | |
360 | if parts: | |
361 | return ".".join([parent, *parts]) | |
362 | return None | |
363 | ||
364 | def warn(self, importable): | |
365 | if importable not in self._already_warned: | |
366 | msg = textwrap.dedent(self.MESSAGE).format(importable=importable) | |
367 | warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) | |
368 | self._already_warned.add(importable) |