djavu-asus/elpy/rpc-venv/lib/python3.11/site-packages/setuptools/discovery.py

   1 """Automatic discovery of Python modules and packages (for inclusion in the
   2 distribution) and other config values.
   3
   4 For the purposes of this module, the following nomenclature is used:
   5
   6 - "src-layout": a directory representing a Python project that contains a "src"
   7   folder. Everything under the "src" folder is meant to be included in the
   8   distribution when packaging the project. Example::
   9
  10     .
  11     ├── tox.ini
  12     ├── pyproject.toml
  13     └── src/
  14         └── mypkg/
  15             ├── __init__.py
  16             ├── mymodule.py
  17             └── my_data_file.txt
  18
  19 - "flat-layout": a Python project that does not use "src-layout" but instead
  20   have a directory under the project root for each package::
  21
  22     .
  23     ├── tox.ini
  24     ├── pyproject.toml
  25     └── mypkg/
  26         ├── __init__.py
  27         ├── mymodule.py
  28         └── my_data_file.txt
  29
  30 - "single-module": a project that contains a single Python script direct under
  31   the project root (no directory used)::
  32
  33     .
  34     ├── tox.ini
  35     ├── pyproject.toml
  36     └── mymodule.py
  37
  38 """
  39
  40 import itertools
  41 import os
  42 from fnmatch import fnmatchcase
  43 from glob import glob
  44 from pathlib import Path
  45 from typing import (
  46     TYPE_CHECKING,
  47     Callable,
  48     Dict,
  49     Iterable,
  50     Iterator,
  51     List,
  52     Mapping,
  53     Optional,
  54     Tuple,
  55     Union
  56 )
  57
  58 import _distutils_hack.override  # noqa: F401
  59
  60 from distutils import log
  61 from distutils.util import convert_path
  62
  63 _Path = Union[str, os.PathLike]
  64 _Filter = Callable[[str], bool]
  65 StrIter = Iterator[str]
  66
  67 chain_iter = itertools.chain.from_iterable
  68
  69 if TYPE_CHECKING:
  70     from setuptools import Distribution  # noqa
  71
  72
  73 def _valid_name(path: _Path) -> bool:
  74     # Ignore invalid names that cannot be imported directly
  75     return os.path.basename(path).isidentifier()
  76
  77
  78 class _Finder:
  79     """Base class that exposes functionality for module/package finders"""
  80
  81     ALWAYS_EXCLUDE: Tuple[str, ...] = ()
  82     DEFAULT_EXCLUDE: Tuple[str, ...] = ()
  83
  84     @classmethod
  85     def find(
  86         cls,
  87         where: _Path = '.',
  88         exclude: Iterable[str] = (),
  89         include: Iterable[str] = ('*',)
  90     ) -> List[str]:
  91         """Return a list of all Python items (packages or modules, depending on
  92         the finder implementation) found within directory 'where'.
  93
  94         'where' is the root directory which will be searched.
  95         It should be supplied as a "cross-platform" (i.e. URL-style) path;
  96         it will be converted to the appropriate local path syntax.
  97
  98         'exclude' is a sequence of names to exclude; '*' can be used
  99         as a wildcard in the names.
 100         When finding packages, 'foo.*' will exclude all subpackages of 'foo'
 101         (but not 'foo' itself).
 102
 103         'include' is a sequence of names to include.
 104         If it's specified, only the named items will be included.
 105         If it's not specified, all found items will be included.
 106         'include' can contain shell style wildcard patterns just like
 107         'exclude'.
 108         """
 109
 110         exclude = exclude or cls.DEFAULT_EXCLUDE
 111         return list(
 112             cls._find_iter(
 113                 convert_path(str(where)),
 114                 cls._build_filter(*cls.ALWAYS_EXCLUDE, *exclude),
 115                 cls._build_filter(*include),
 116             )
 117         )
 118
 119     @classmethod
 120     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 121         raise NotImplementedError
 122
 123     @staticmethod
 124     def _build_filter(*patterns: str) -> _Filter:
 125         """
 126         Given a list of patterns, return a callable that will be true only if
 127         the input matches at least one of the patterns.
 128         """
 129         return lambda name: any(fnmatchcase(name, pat) for pat in patterns)
 130
 131
 132 class PackageFinder(_Finder):
 133     """
 134     Generate a list of all Python packages found within a directory
 135     """
 136
 137     ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__")
 138
 139     @classmethod
 140     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 141         """
 142         All the packages found in 'where' that pass the 'include' filter, but
 143         not the 'exclude' filter.
 144         """
 145         for root, dirs, files in os.walk(str(where), followlinks=True):
 146             # Copy dirs to iterate over it, then empty dirs.
 147             all_dirs = dirs[:]
 148             dirs[:] = []
 149
 150             for dir in all_dirs:
 151                 full_path = os.path.join(root, dir)
 152                 rel_path = os.path.relpath(full_path, where)
 153                 package = rel_path.replace(os.path.sep, '.')
 154
 155                 # Skip directory trees that are not valid packages
 156                 if '.' in dir or not cls._looks_like_package(full_path, package):
 157                     continue
 158
 159                 # Should this package be included?
 160                 if include(package) and not exclude(package):
 161                     yield package
 162
 163                 # Keep searching subdirectories, as there may be more packages
 164                 # down there, even if the parent was excluded.
 165                 dirs.append(dir)
 166
 167     @staticmethod
 168     def _looks_like_package(path: _Path, _package_name: str) -> bool:
 169         """Does a directory look like a package?"""
 170         return os.path.isfile(os.path.join(path, '__init__.py'))
 171
 172
 173 class PEP420PackageFinder(PackageFinder):
 174     @staticmethod
 175     def _looks_like_package(_path: _Path, _package_name: str) -> bool:
 176         return True
 177
 178
 179 class ModuleFinder(_Finder):
 180     """Find isolated Python modules.
 181     This function will **not** recurse subdirectories.
 182     """
 183
 184     @classmethod
 185     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 186         for file in glob(os.path.join(where, "*.py")):
 187             module, _ext = os.path.splitext(os.path.basename(file))
 188
 189             if not cls._looks_like_module(module):
 190                 continue
 191
 192             if include(module) and not exclude(module):
 193                 yield module
 194
 195     _looks_like_module = staticmethod(_valid_name)
 196
 197
 198 # We have to be extra careful in the case of flat layout to not include files
 199 # and directories not meant for distribution (e.g. tool-related)
 200
 201
 202 class FlatLayoutPackageFinder(PEP420PackageFinder):
 203     _EXCLUDE = (
 204         "ci",
 205         "bin",
 206         "doc",
 207         "docs",
 208         "documentation",
 209         "manpages",
 210         "news",
 211         "changelog",
 212         "test",
 213         "tests",
 214         "unit_test",
 215         "unit_tests",
 216         "example",
 217         "examples",
 218         "scripts",
 219         "tools",
 220         "util",
 221         "utils",
 222         "python",
 223         "build",
 224         "dist",
 225         "venv",
 226         "env",
 227         "requirements",
 228         # ---- Task runners / Build tools ----
 229         "tasks",  # invoke
 230         "fabfile",  # fabric
 231         "site_scons",  # SCons
 232         # ---- Other tools ----
 233         "benchmark",
 234         "benchmarks",
 235         "exercise",
 236         "exercises",
 237         # ---- Hidden directories/Private packages ----
 238         "[._]*",
 239     )
 240
 241     DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE))
 242     """Reserved package names"""
 243
 244     @staticmethod
 245     def _looks_like_package(_path: _Path, package_name: str) -> bool:
 246         names = package_name.split('.')
 247         # Consider PEP 561
 248         root_pkg_is_valid = names[0].isidentifier() or names[0].endswith("-stubs")
 249         return root_pkg_is_valid and all(name.isidentifier() for name in names[1:])
 250
 251
 252 class FlatLayoutModuleFinder(ModuleFinder):
 253     DEFAULT_EXCLUDE = (
 254         "setup",
 255         "conftest",
 256         "test",
 257         "tests",
 258         "example",
 259         "examples",
 260         "build",
 261         # ---- Task runners ----
 262         "toxfile",
 263         "noxfile",
 264         "pavement",
 265         "dodo",
 266         "tasks",
 267         "fabfile",
 268         # ---- Other tools ----
 269         "[Ss][Cc]onstruct",  # SCons
 270         "conanfile",  # Connan: C/C++ build tool
 271         "manage",  # Django
 272         "benchmark",
 273         "benchmarks",
 274         "exercise",
 275         "exercises",
 276         "htmlcov",
 277         # ---- Hidden files/Private modules ----
 278         "[._]*",
 279     )
 280     """Reserved top-level module names"""
 281
 282
 283 def _find_packages_within(root_pkg: str, pkg_dir: _Path) -> List[str]:
 284     nested = PEP420PackageFinder.find(pkg_dir)
 285     return [root_pkg] + [".".join((root_pkg, n)) for n in nested]
 286
 287
 288 class ConfigDiscovery:
 289     """Fill-in metadata and options that can be automatically derived
 290     (from other metadata/options, the file system or conventions)
 291     """
 292
 293     def __init__(self, distribution: "Distribution"):
 294         self.dist = distribution
 295         self._called = False
 296         self._disabled = False
 297         self._skip_ext_modules = False
 298
 299     def _disable(self):
 300         """Internal API to disable automatic discovery"""
 301         self._disabled = True
 302
 303     def _ignore_ext_modules(self):
 304         """Internal API to disregard ext_modules.
 305
 306         Normally auto-discovery would not be triggered if ``ext_modules`` are set
 307         (this is done for backward compatibility with existing packages relying on
 308         ``setup.py`` or ``setup.cfg``). However, ``setuptools`` can call this function
 309         to ignore given ``ext_modules`` and proceed with the auto-discovery if
 310         ``packages`` and ``py_modules`` are not given (e.g. when using pyproject.toml
 311         metadata).
 312         """
 313         self._skip_ext_modules = True
 314
 315     @property
 316     def _root_dir(self) -> _Path:
 317         # The best is to wait until `src_root` is set in dist, before using _root_dir.
 318         return self.dist.src_root or os.curdir
 319
 320     @property
 321     def _package_dir(self) -> Dict[str, str]:
 322         if self.dist.package_dir is None:
 323             return {}
 324         return self.dist.package_dir
 325
 326     def __call__(self, force=False, name=True, ignore_ext_modules=False):
 327         """Automatically discover missing configuration fields
 328         and modifies the given ``distribution`` object in-place.
 329
 330         Note that by default this will only have an effect the first time the
 331         ``ConfigDiscovery`` object is called.
 332
 333         To repeatedly invoke automatic discovery (e.g. when the project
 334         directory changes), please use ``force=True`` (or create a new
 335         ``ConfigDiscovery`` instance).
 336         """
 337         if force is False and (self._called or self._disabled):
 338             # Avoid overhead of multiple calls
 339             return
 340
 341         self._analyse_package_layout(ignore_ext_modules)
 342         if name:
 343             self.analyse_name()  # depends on ``packages`` and ``py_modules``
 344
 345         self._called = True
 346
 347     def _explicitly_specified(self, ignore_ext_modules: bool) -> bool:
 348         """``True`` if the user has specified some form of package/module listing"""
 349         ignore_ext_modules = ignore_ext_modules or self._skip_ext_modules
 350         ext_modules = not (self.dist.ext_modules is None or ignore_ext_modules)
 351         return (
 352             self.dist.packages is not None
 353             or self.dist.py_modules is not None
 354             or ext_modules
 355             or hasattr(self.dist, "configuration") and self.dist.configuration
 356             # ^ Some projects use numpy.distutils.misc_util.Configuration
 357         )
 358
 359     def _analyse_package_layout(self, ignore_ext_modules: bool) -> bool:
 360         if self._explicitly_specified(ignore_ext_modules):
 361             # For backward compatibility, just try to find modules/packages
 362             # when nothing is given
 363             return True
 364
 365         log.debug(
 366             "No `packages` or `py_modules` configuration, performing "
 367             "automatic discovery."
 368         )
 369
 370         return (
 371             self._analyse_explicit_layout()
 372             or self._analyse_src_layout()
 373             # flat-layout is the trickiest for discovery so it should be last
 374             or self._analyse_flat_layout()
 375         )
 376
 377     def _analyse_explicit_layout(self) -> bool:
 378         """The user can explicitly give a package layout via ``package_dir``"""
 379         package_dir = self._package_dir.copy()  # don't modify directly
 380         package_dir.pop("", None)  # This falls under the "src-layout" umbrella
 381         root_dir = self._root_dir
 382
 383         if not package_dir:
 384             return False
 385
 386         log.debug(f"`explicit-layout` detected -- analysing {package_dir}")
 387         pkgs = chain_iter(
 388             _find_packages_within(pkg, os.path.join(root_dir, parent_dir))
 389             for pkg, parent_dir in package_dir.items()
 390         )
 391         self.dist.packages = list(pkgs)
 392         log.debug(f"discovered packages -- {self.dist.packages}")
 393         return True
 394
 395     def _analyse_src_layout(self) -> bool:
 396         """Try to find all packages or modules under the ``src`` directory
 397         (or anything pointed by ``package_dir[""]``).
 398
 399         The "src-layout" is relatively safe for automatic discovery.
 400         We assume that everything within is meant to be included in the
 401         distribution.
 402
 403         If ``package_dir[""]`` is not given, but the ``src`` directory exists,
 404         this function will set ``package_dir[""] = "src"``.
 405         """
 406         package_dir = self._package_dir
 407         src_dir = os.path.join(self._root_dir, package_dir.get("", "src"))
 408         if not os.path.isdir(src_dir):
 409             return False
 410
 411         log.debug(f"`src-layout` detected -- analysing {src_dir}")
 412         package_dir.setdefault("", os.path.basename(src_dir))
 413         self.dist.package_dir = package_dir  # persist eventual modifications
 414         self.dist.packages = PEP420PackageFinder.find(src_dir)
 415         self.dist.py_modules = ModuleFinder.find(src_dir)
 416         log.debug(f"discovered packages -- {self.dist.packages}")
 417         log.debug(f"discovered py_modules -- {self.dist.py_modules}")
 418         return True
 419
 420     def _analyse_flat_layout(self) -> bool:
 421         """Try to find all packages and modules under the project root.
 422
 423         Since the ``flat-layout`` is more dangerous in terms of accidentally including
 424         extra files/directories, this function is more conservative and will raise an
 425         error if multiple packages or modules are found.
 426
 427         This assumes that multi-package dists are uncommon and refuse to support that
 428         use case in order to be able to prevent unintended errors.
 429         """
 430         log.debug(f"`flat-layout` detected -- analysing {self._root_dir}")
 431         return self._analyse_flat_packages() or self._analyse_flat_modules()
 432
 433     def _analyse_flat_packages(self) -> bool:
 434         self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir)
 435         top_level = remove_nested_packages(remove_stubs(self.dist.packages))
 436         log.debug(f"discovered packages -- {self.dist.packages}")
 437         self._ensure_no_accidental_inclusion(top_level, "packages")
 438         return bool(top_level)
 439
 440     def _analyse_flat_modules(self) -> bool:
 441         self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir)
 442         log.debug(f"discovered py_modules -- {self.dist.py_modules}")
 443         self._ensure_no_accidental_inclusion(self.dist.py_modules, "modules")
 444         return bool(self.dist.py_modules)
 445
 446     def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str):
 447         if len(detected) > 1:
 448             from inspect import cleandoc
 449
 450             from setuptools.errors import PackageDiscoveryError
 451
 452             msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.
 453
 454             To avoid accidental inclusion of unwanted files or directories,
 455             setuptools will not proceed with this build.
 456
 457             If you are trying to create a single distribution with multiple {kind}
 458             on purpose, you should not rely on automatic discovery.
 459             Instead, consider the following options:
 460
 461             1. set up custom discovery (`find` directive with `include` or `exclude`)
 462             2. use a `src-layout`
 463             3. explicitly set `py_modules` or `packages` with a list of names
 464
 465             To find more information, look for "package discovery" on setuptools docs.
 466             """
 467             raise PackageDiscoveryError(cleandoc(msg))
 468
 469     def analyse_name(self):
 470         """The packages/modules are the essential contribution of the author.
 471         Therefore the name of the distribution can be derived from them.
 472         """
 473         if self.dist.metadata.name or self.dist.name:
 474             # get_name() is not reliable (can return "UNKNOWN")
 475             return None
 476
 477         log.debug("No `name` configuration, performing automatic discovery")
 478
 479         name = (
 480             self._find_name_single_package_or_module()
 481             or self._find_name_from_packages()
 482         )
 483         if name:
 484             self.dist.metadata.name = name
 485
 486     def _find_name_single_package_or_module(self) -> Optional[str]:
 487         """Exactly one module or package"""
 488         for field in ('packages', 'py_modules'):
 489             items = getattr(self.dist, field, None) or []
 490             if items and len(items) == 1:
 491                 log.debug(f"Single module/package detected, name: {items[0]}")
 492                 return items[0]
 493
 494         return None
 495
 496     def _find_name_from_packages(self) -> Optional[str]:
 497         """Try to find the root package that is not a PEP 420 namespace"""
 498         if not self.dist.packages:
 499             return None
 500
 501         packages = remove_stubs(sorted(self.dist.packages, key=len))
 502         package_dir = self.dist.package_dir or {}
 503
 504         parent_pkg = find_parent_package(packages, package_dir, self._root_dir)
 505         if parent_pkg:
 506             log.debug(f"Common parent package detected, name: {parent_pkg}")
 507             return parent_pkg
 508
 509         log.warn("No parent package detected, impossible to derive `name`")
 510         return None
 511
 512
 513 def remove_nested_packages(packages: List[str]) -> List[str]:
 514     """Remove nested packages from a list of packages.
 515
 516     >>> remove_nested_packages(["a", "a.b1", "a.b2", "a.b1.c1"])
 517     ['a']
 518     >>> remove_nested_packages(["a", "b", "c.d", "c.d.e.f", "g.h", "a.a1"])
 519     ['a', 'b', 'c.d', 'g.h']
 520     """
 521     pkgs = sorted(packages, key=len)
 522     top_level = pkgs[:]
 523     size = len(pkgs)
 524     for i, name in enumerate(reversed(pkgs)):
 525         if any(name.startswith(f"{other}.") for other in top_level):
 526             top_level.pop(size - i - 1)
 527
 528     return top_level
 529
 530
 531 def remove_stubs(packages: List[str]) -> List[str]:
 532     """Remove type stubs (:pep:`561`) from a list of packages.
 533
 534     >>> remove_stubs(["a", "a.b", "a-stubs", "a-stubs.b.c", "b", "c-stubs"])
 535     ['a', 'a.b', 'b']
 536     """
 537     return [pkg for pkg in packages if not pkg.split(".")[0].endswith("-stubs")]
 538
 539
 540 def find_parent_package(
 541     packages: List[str], package_dir: Mapping[str, str], root_dir: _Path
 542 ) -> Optional[str]:
 543     """Find the parent package that is not a namespace."""
 544     packages = sorted(packages, key=len)
 545     common_ancestors = []
 546     for i, name in enumerate(packages):
 547         if not all(n.startswith(f"{name}.") for n in packages[i+1:]):
 548             # Since packages are sorted by length, this condition is able
 549             # to find a list of all common ancestors.
 550             # When there is divergence (e.g. multiple root packages)
 551             # the list will be empty
 552             break
 553         common_ancestors.append(name)
 554
 555     for name in common_ancestors:
 556         pkg_path = find_package_path(name, package_dir, root_dir)
 557         init = os.path.join(pkg_path, "__init__.py")
 558         if os.path.isfile(init):
 559             return name
 560
 561     return None
 562
 563
 564 def find_package_path(
 565     name: str, package_dir: Mapping[str, str], root_dir: _Path
 566 ) -> str:
 567     """Given a package name, return the path where it should be found on
 568     disk, considering the ``package_dir`` option.
 569
 570     >>> path = find_package_path("my.pkg", {"": "root/is/nested"}, ".")
 571     >>> path.replace(os.sep, "/")
 572     './root/is/nested/my/pkg'
 573
 574     >>> path = find_package_path("my.pkg", {"my": "root/is/nested"}, ".")
 575     >>> path.replace(os.sep, "/")
 576     './root/is/nested/pkg'
 577
 578     >>> path = find_package_path("my.pkg", {"my.pkg": "root/is/nested"}, ".")
 579     >>> path.replace(os.sep, "/")
 580     './root/is/nested'
 581
 582     >>> path = find_package_path("other.pkg", {"my.pkg": "root/is/nested"}, ".")
 583     >>> path.replace(os.sep, "/")
 584     './other/pkg'
 585     """
 586     parts = name.split(".")
 587     for i in range(len(parts), 0, -1):
 588         # Look backwards, the most specific package_dir first
 589         partial_name = ".".join(parts[:i])
 590         if partial_name in package_dir:
 591             parent = package_dir[partial_name]
 592             return os.path.join(root_dir, parent, *parts[i:])
 593
 594     parent = package_dir.get("") or ""
 595     return os.path.join(root_dir, *parent.split("/"), *parts)
 596
 597
 598 def construct_package_dir(packages: List[str], package_path: _Path) -> Dict[str, str]:
 599     parent_pkgs = remove_nested_packages(packages)
 600     prefix = Path(package_path).parts
 601     return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}