You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

877 lines
31KB

  1. """Core implementation of the testing process: init, session, runtest loop."""
  2. import argparse
  3. import fnmatch
  4. import functools
  5. import importlib
  6. import os
  7. import sys
  8. from pathlib import Path
  9. from typing import Callable
  10. from typing import Dict
  11. from typing import FrozenSet
  12. from typing import Iterator
  13. from typing import List
  14. from typing import Optional
  15. from typing import overload
  16. from typing import Sequence
  17. from typing import Set
  18. from typing import Tuple
  19. from typing import Type
  20. from typing import TYPE_CHECKING
  21. from typing import Union
  22. import attr
  23. import py
  24. import _pytest._code
  25. from _pytest import nodes
  26. from _pytest.compat import final
  27. from _pytest.config import Config
  28. from _pytest.config import directory_arg
  29. from _pytest.config import ExitCode
  30. from _pytest.config import hookimpl
  31. from _pytest.config import PytestPluginManager
  32. from _pytest.config import UsageError
  33. from _pytest.config.argparsing import Parser
  34. from _pytest.fixtures import FixtureManager
  35. from _pytest.outcomes import exit
  36. from _pytest.pathlib import absolutepath
  37. from _pytest.pathlib import bestrelpath
  38. from _pytest.pathlib import visit
  39. from _pytest.reports import CollectReport
  40. from _pytest.reports import TestReport
  41. from _pytest.runner import collect_one_node
  42. from _pytest.runner import SetupState
  43. if TYPE_CHECKING:
  44. from typing_extensions import Literal
  45. def pytest_addoption(parser: Parser) -> None:
  46. parser.addini(
  47. "norecursedirs",
  48. "directory patterns to avoid for recursion",
  49. type="args",
  50. default=[
  51. "*.egg",
  52. ".*",
  53. "_darcs",
  54. "build",
  55. "CVS",
  56. "dist",
  57. "node_modules",
  58. "venv",
  59. "{arch}",
  60. ],
  61. )
  62. parser.addini(
  63. "testpaths",
  64. "directories to search for tests when no files or directories are given in the "
  65. "command line.",
  66. type="args",
  67. default=[],
  68. )
  69. group = parser.getgroup("general", "running and selection options")
  70. group._addoption(
  71. "-x",
  72. "--exitfirst",
  73. action="store_const",
  74. dest="maxfail",
  75. const=1,
  76. help="exit instantly on first error or failed test.",
  77. )
  78. group = parser.getgroup("pytest-warnings")
  79. group.addoption(
  80. "-W",
  81. "--pythonwarnings",
  82. action="append",
  83. help="set which warnings to report, see -W option of python itself.",
  84. )
  85. parser.addini(
  86. "filterwarnings",
  87. type="linelist",
  88. help="Each line specifies a pattern for "
  89. "warnings.filterwarnings. "
  90. "Processed after -W/--pythonwarnings.",
  91. )
  92. group._addoption(
  93. "--maxfail",
  94. metavar="num",
  95. action="store",
  96. type=int,
  97. dest="maxfail",
  98. default=0,
  99. help="exit after first num failures or errors.",
  100. )
  101. group._addoption(
  102. "--strict-config",
  103. action="store_true",
  104. help="any warnings encountered while parsing the `pytest` section of the configuration file raise errors.",
  105. )
  106. group._addoption(
  107. "--strict-markers",
  108. action="store_true",
  109. help="markers not registered in the `markers` section of the configuration file raise errors.",
  110. )
  111. group._addoption(
  112. "--strict", action="store_true", help="(deprecated) alias to --strict-markers.",
  113. )
  114. group._addoption(
  115. "-c",
  116. metavar="file",
  117. type=str,
  118. dest="inifilename",
  119. help="load configuration from `file` instead of trying to locate one of the implicit "
  120. "configuration files.",
  121. )
  122. group._addoption(
  123. "--continue-on-collection-errors",
  124. action="store_true",
  125. default=False,
  126. dest="continue_on_collection_errors",
  127. help="Force test execution even if collection errors occur.",
  128. )
  129. group._addoption(
  130. "--rootdir",
  131. action="store",
  132. dest="rootdir",
  133. help="Define root directory for tests. Can be relative path: 'root_dir', './root_dir', "
  134. "'root_dir/another_dir/'; absolute path: '/home/user/root_dir'; path with variables: "
  135. "'$HOME/root_dir'.",
  136. )
  137. group = parser.getgroup("collect", "collection")
  138. group.addoption(
  139. "--collectonly",
  140. "--collect-only",
  141. "--co",
  142. action="store_true",
  143. help="only collect tests, don't execute them.",
  144. )
  145. group.addoption(
  146. "--pyargs",
  147. action="store_true",
  148. help="try to interpret all arguments as python packages.",
  149. )
  150. group.addoption(
  151. "--ignore",
  152. action="append",
  153. metavar="path",
  154. help="ignore path during collection (multi-allowed).",
  155. )
  156. group.addoption(
  157. "--ignore-glob",
  158. action="append",
  159. metavar="path",
  160. help="ignore path pattern during collection (multi-allowed).",
  161. )
  162. group.addoption(
  163. "--deselect",
  164. action="append",
  165. metavar="nodeid_prefix",
  166. help="deselect item (via node id prefix) during collection (multi-allowed).",
  167. )
  168. group.addoption(
  169. "--confcutdir",
  170. dest="confcutdir",
  171. default=None,
  172. metavar="dir",
  173. type=functools.partial(directory_arg, optname="--confcutdir"),
  174. help="only load conftest.py's relative to specified dir.",
  175. )
  176. group.addoption(
  177. "--noconftest",
  178. action="store_true",
  179. dest="noconftest",
  180. default=False,
  181. help="Don't load any conftest.py files.",
  182. )
  183. group.addoption(
  184. "--keepduplicates",
  185. "--keep-duplicates",
  186. action="store_true",
  187. dest="keepduplicates",
  188. default=False,
  189. help="Keep duplicate tests.",
  190. )
  191. group.addoption(
  192. "--collect-in-virtualenv",
  193. action="store_true",
  194. dest="collect_in_virtualenv",
  195. default=False,
  196. help="Don't ignore tests in a local virtualenv directory",
  197. )
  198. group.addoption(
  199. "--import-mode",
  200. default="prepend",
  201. choices=["prepend", "append", "importlib"],
  202. dest="importmode",
  203. help="prepend/append to sys.path when importing test modules and conftest files, "
  204. "default is to prepend.",
  205. )
  206. group = parser.getgroup("debugconfig", "test session debugging and configuration")
  207. group.addoption(
  208. "--basetemp",
  209. dest="basetemp",
  210. default=None,
  211. type=validate_basetemp,
  212. metavar="dir",
  213. help=(
  214. "base temporary directory for this test run."
  215. "(warning: this directory is removed if it exists)"
  216. ),
  217. )
  218. def validate_basetemp(path: str) -> str:
  219. # GH 7119
  220. msg = "basetemp must not be empty, the current working directory or any parent directory of it"
  221. # empty path
  222. if not path:
  223. raise argparse.ArgumentTypeError(msg)
  224. def is_ancestor(base: Path, query: Path) -> bool:
  225. """Return whether query is an ancestor of base."""
  226. if base == query:
  227. return True
  228. for parent in base.parents:
  229. if parent == query:
  230. return True
  231. return False
  232. # check if path is an ancestor of cwd
  233. if is_ancestor(Path.cwd(), Path(path).absolute()):
  234. raise argparse.ArgumentTypeError(msg)
  235. # check symlinks for ancestors
  236. if is_ancestor(Path.cwd().resolve(), Path(path).resolve()):
  237. raise argparse.ArgumentTypeError(msg)
  238. return path
  239. def wrap_session(
  240. config: Config, doit: Callable[[Config, "Session"], Optional[Union[int, ExitCode]]]
  241. ) -> Union[int, ExitCode]:
  242. """Skeleton command line program."""
  243. session = Session.from_config(config)
  244. session.exitstatus = ExitCode.OK
  245. initstate = 0
  246. try:
  247. try:
  248. config._do_configure()
  249. initstate = 1
  250. config.hook.pytest_sessionstart(session=session)
  251. initstate = 2
  252. session.exitstatus = doit(config, session) or 0
  253. except UsageError:
  254. session.exitstatus = ExitCode.USAGE_ERROR
  255. raise
  256. except Failed:
  257. session.exitstatus = ExitCode.TESTS_FAILED
  258. except (KeyboardInterrupt, exit.Exception):
  259. excinfo = _pytest._code.ExceptionInfo.from_current()
  260. exitstatus: Union[int, ExitCode] = ExitCode.INTERRUPTED
  261. if isinstance(excinfo.value, exit.Exception):
  262. if excinfo.value.returncode is not None:
  263. exitstatus = excinfo.value.returncode
  264. if initstate < 2:
  265. sys.stderr.write(f"{excinfo.typename}: {excinfo.value.msg}\n")
  266. config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
  267. session.exitstatus = exitstatus
  268. except BaseException:
  269. session.exitstatus = ExitCode.INTERNAL_ERROR
  270. excinfo = _pytest._code.ExceptionInfo.from_current()
  271. try:
  272. config.notify_exception(excinfo, config.option)
  273. except exit.Exception as exc:
  274. if exc.returncode is not None:
  275. session.exitstatus = exc.returncode
  276. sys.stderr.write("{}: {}\n".format(type(exc).__name__, exc))
  277. else:
  278. if isinstance(excinfo.value, SystemExit):
  279. sys.stderr.write("mainloop: caught unexpected SystemExit!\n")
  280. finally:
  281. # Explicitly break reference cycle.
  282. excinfo = None # type: ignore
  283. session.startdir.chdir()
  284. if initstate >= 2:
  285. try:
  286. config.hook.pytest_sessionfinish(
  287. session=session, exitstatus=session.exitstatus
  288. )
  289. except exit.Exception as exc:
  290. if exc.returncode is not None:
  291. session.exitstatus = exc.returncode
  292. sys.stderr.write("{}: {}\n".format(type(exc).__name__, exc))
  293. config._ensure_unconfigure()
  294. return session.exitstatus
  295. def pytest_cmdline_main(config: Config) -> Union[int, ExitCode]:
  296. return wrap_session(config, _main)
  297. def _main(config: Config, session: "Session") -> Optional[Union[int, ExitCode]]:
  298. """Default command line protocol for initialization, session,
  299. running tests and reporting."""
  300. config.hook.pytest_collection(session=session)
  301. config.hook.pytest_runtestloop(session=session)
  302. if session.testsfailed:
  303. return ExitCode.TESTS_FAILED
  304. elif session.testscollected == 0:
  305. return ExitCode.NO_TESTS_COLLECTED
  306. return None
  307. def pytest_collection(session: "Session") -> None:
  308. session.perform_collect()
  309. def pytest_runtestloop(session: "Session") -> bool:
  310. if session.testsfailed and not session.config.option.continue_on_collection_errors:
  311. raise session.Interrupted(
  312. "%d error%s during collection"
  313. % (session.testsfailed, "s" if session.testsfailed != 1 else "")
  314. )
  315. if session.config.option.collectonly:
  316. return True
  317. for i, item in enumerate(session.items):
  318. nextitem = session.items[i + 1] if i + 1 < len(session.items) else None
  319. item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
  320. if session.shouldfail:
  321. raise session.Failed(session.shouldfail)
  322. if session.shouldstop:
  323. raise session.Interrupted(session.shouldstop)
  324. return True
  325. def _in_venv(path: py.path.local) -> bool:
  326. """Attempt to detect if ``path`` is the root of a Virtual Environment by
  327. checking for the existence of the appropriate activate script."""
  328. bindir = path.join("Scripts" if sys.platform.startswith("win") else "bin")
  329. if not bindir.isdir():
  330. return False
  331. activates = (
  332. "activate",
  333. "activate.csh",
  334. "activate.fish",
  335. "Activate",
  336. "Activate.bat",
  337. "Activate.ps1",
  338. )
  339. return any([fname.basename in activates for fname in bindir.listdir()])
  340. def pytest_ignore_collect(path: py.path.local, config: Config) -> Optional[bool]:
  341. ignore_paths = config._getconftest_pathlist("collect_ignore", path=path.dirpath())
  342. ignore_paths = ignore_paths or []
  343. excludeopt = config.getoption("ignore")
  344. if excludeopt:
  345. ignore_paths.extend([py.path.local(x) for x in excludeopt])
  346. if py.path.local(path) in ignore_paths:
  347. return True
  348. ignore_globs = config._getconftest_pathlist(
  349. "collect_ignore_glob", path=path.dirpath()
  350. )
  351. ignore_globs = ignore_globs or []
  352. excludeglobopt = config.getoption("ignore_glob")
  353. if excludeglobopt:
  354. ignore_globs.extend([py.path.local(x) for x in excludeglobopt])
  355. if any(fnmatch.fnmatch(str(path), str(glob)) for glob in ignore_globs):
  356. return True
  357. allow_in_venv = config.getoption("collect_in_virtualenv")
  358. if not allow_in_venv and _in_venv(path):
  359. return True
  360. return None
  361. def pytest_collection_modifyitems(items: List[nodes.Item], config: Config) -> None:
  362. deselect_prefixes = tuple(config.getoption("deselect") or [])
  363. if not deselect_prefixes:
  364. return
  365. remaining = []
  366. deselected = []
  367. for colitem in items:
  368. if colitem.nodeid.startswith(deselect_prefixes):
  369. deselected.append(colitem)
  370. else:
  371. remaining.append(colitem)
  372. if deselected:
  373. config.hook.pytest_deselected(items=deselected)
  374. items[:] = remaining
  375. class FSHookProxy:
  376. def __init__(self, pm: PytestPluginManager, remove_mods) -> None:
  377. self.pm = pm
  378. self.remove_mods = remove_mods
  379. def __getattr__(self, name: str):
  380. x = self.pm.subset_hook_caller(name, remove_plugins=self.remove_mods)
  381. self.__dict__[name] = x
  382. return x
  383. class Interrupted(KeyboardInterrupt):
  384. """Signals that the test run was interrupted."""
  385. __module__ = "builtins" # For py3.
  386. class Failed(Exception):
  387. """Signals a stop as failed test run."""
  388. @attr.s
  389. class _bestrelpath_cache(Dict[Path, str]):
  390. path = attr.ib(type=Path)
  391. def __missing__(self, path: Path) -> str:
  392. r = bestrelpath(self.path, path)
  393. self[path] = r
  394. return r
  395. @final
  396. class Session(nodes.FSCollector):
  397. Interrupted = Interrupted
  398. Failed = Failed
  399. # Set on the session by runner.pytest_sessionstart.
  400. _setupstate: SetupState
  401. # Set on the session by fixtures.pytest_sessionstart.
  402. _fixturemanager: FixtureManager
  403. exitstatus: Union[int, ExitCode]
  404. def __init__(self, config: Config) -> None:
  405. super().__init__(
  406. config.rootdir, parent=None, config=config, session=self, nodeid=""
  407. )
  408. self.testsfailed = 0
  409. self.testscollected = 0
  410. self.shouldstop: Union[bool, str] = False
  411. self.shouldfail: Union[bool, str] = False
  412. self.trace = config.trace.root.get("collection")
  413. self.startdir = config.invocation_dir
  414. self._initialpaths: FrozenSet[py.path.local] = frozenset()
  415. self._bestrelpathcache: Dict[Path, str] = _bestrelpath_cache(config.rootpath)
  416. self.config.pluginmanager.register(self, name="session")
  417. @classmethod
  418. def from_config(cls, config: Config) -> "Session":
  419. session: Session = cls._create(config)
  420. return session
  421. def __repr__(self) -> str:
  422. return "<%s %s exitstatus=%r testsfailed=%d testscollected=%d>" % (
  423. self.__class__.__name__,
  424. self.name,
  425. getattr(self, "exitstatus", "<UNSET>"),
  426. self.testsfailed,
  427. self.testscollected,
  428. )
  429. def _node_location_to_relpath(self, node_path: Path) -> str:
  430. # bestrelpath is a quite slow function.
  431. return self._bestrelpathcache[node_path]
  432. @hookimpl(tryfirst=True)
  433. def pytest_collectstart(self) -> None:
  434. if self.shouldfail:
  435. raise self.Failed(self.shouldfail)
  436. if self.shouldstop:
  437. raise self.Interrupted(self.shouldstop)
  438. @hookimpl(tryfirst=True)
  439. def pytest_runtest_logreport(
  440. self, report: Union[TestReport, CollectReport]
  441. ) -> None:
  442. if report.failed and not hasattr(report, "wasxfail"):
  443. self.testsfailed += 1
  444. maxfail = self.config.getvalue("maxfail")
  445. if maxfail and self.testsfailed >= maxfail:
  446. self.shouldfail = "stopping after %d failures" % (self.testsfailed)
  447. pytest_collectreport = pytest_runtest_logreport
  448. def isinitpath(self, path: py.path.local) -> bool:
  449. return path in self._initialpaths
  450. def gethookproxy(self, fspath: py.path.local):
  451. # Check if we have the common case of running
  452. # hooks with all conftest.py files.
  453. pm = self.config.pluginmanager
  454. my_conftestmodules = pm._getconftestmodules(
  455. fspath, self.config.getoption("importmode")
  456. )
  457. remove_mods = pm._conftest_plugins.difference(my_conftestmodules)
  458. if remove_mods:
  459. # One or more conftests are not in use at this fspath.
  460. proxy = FSHookProxy(pm, remove_mods)
  461. else:
  462. # All plugins are active for this fspath.
  463. proxy = self.config.hook
  464. return proxy
  465. def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
  466. if direntry.name == "__pycache__":
  467. return False
  468. path = py.path.local(direntry.path)
  469. ihook = self.gethookproxy(path.dirpath())
  470. if ihook.pytest_ignore_collect(path=path, config=self.config):
  471. return False
  472. norecursepatterns = self.config.getini("norecursedirs")
  473. if any(path.check(fnmatch=pat) for pat in norecursepatterns):
  474. return False
  475. return True
  476. def _collectfile(
  477. self, path: py.path.local, handle_dupes: bool = True
  478. ) -> Sequence[nodes.Collector]:
  479. assert (
  480. path.isfile()
  481. ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format(
  482. path, path.isdir(), path.exists(), path.islink()
  483. )
  484. ihook = self.gethookproxy(path)
  485. if not self.isinitpath(path):
  486. if ihook.pytest_ignore_collect(path=path, config=self.config):
  487. return ()
  488. if handle_dupes:
  489. keepduplicates = self.config.getoption("keepduplicates")
  490. if not keepduplicates:
  491. duplicate_paths = self.config.pluginmanager._duplicatepaths
  492. if path in duplicate_paths:
  493. return ()
  494. else:
  495. duplicate_paths.add(path)
  496. return ihook.pytest_collect_file(path=path, parent=self) # type: ignore[no-any-return]
  497. @overload
  498. def perform_collect(
  499. self, args: Optional[Sequence[str]] = ..., genitems: "Literal[True]" = ...
  500. ) -> Sequence[nodes.Item]:
  501. ...
  502. @overload
  503. def perform_collect(
  504. self, args: Optional[Sequence[str]] = ..., genitems: bool = ...
  505. ) -> Sequence[Union[nodes.Item, nodes.Collector]]:
  506. ...
  507. def perform_collect(
  508. self, args: Optional[Sequence[str]] = None, genitems: bool = True
  509. ) -> Sequence[Union[nodes.Item, nodes.Collector]]:
  510. """Perform the collection phase for this session.
  511. This is called by the default
  512. :func:`pytest_collection <_pytest.hookspec.pytest_collection>` hook
  513. implementation; see the documentation of this hook for more details.
  514. For testing purposes, it may also be called directly on a fresh
  515. ``Session``.
  516. This function normally recursively expands any collectors collected
  517. from the session to their items, and only items are returned. For
  518. testing purposes, this may be suppressed by passing ``genitems=False``,
  519. in which case the return value contains these collectors unexpanded,
  520. and ``session.items`` is empty.
  521. """
  522. if args is None:
  523. args = self.config.args
  524. self.trace("perform_collect", self, args)
  525. self.trace.root.indent += 1
  526. self._notfound: List[Tuple[str, Sequence[nodes.Collector]]] = []
  527. self._initial_parts: List[Tuple[py.path.local, List[str]]] = []
  528. self.items: List[nodes.Item] = []
  529. hook = self.config.hook
  530. items: Sequence[Union[nodes.Item, nodes.Collector]] = self.items
  531. try:
  532. initialpaths: List[py.path.local] = []
  533. for arg in args:
  534. fspath, parts = resolve_collection_argument(
  535. self.config.invocation_params.dir,
  536. arg,
  537. as_pypath=self.config.option.pyargs,
  538. )
  539. self._initial_parts.append((fspath, parts))
  540. initialpaths.append(fspath)
  541. self._initialpaths = frozenset(initialpaths)
  542. rep = collect_one_node(self)
  543. self.ihook.pytest_collectreport(report=rep)
  544. self.trace.root.indent -= 1
  545. if self._notfound:
  546. errors = []
  547. for arg, cols in self._notfound:
  548. line = f"(no name {arg!r} in any of {cols!r})"
  549. errors.append(f"not found: {arg}\n{line}")
  550. raise UsageError(*errors)
  551. if not genitems:
  552. items = rep.result
  553. else:
  554. if rep.passed:
  555. for node in rep.result:
  556. self.items.extend(self.genitems(node))
  557. self.config.pluginmanager.check_pending()
  558. hook.pytest_collection_modifyitems(
  559. session=self, config=self.config, items=items
  560. )
  561. finally:
  562. hook.pytest_collection_finish(session=self)
  563. self.testscollected = len(items)
  564. return items
  565. def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]:
  566. from _pytest.python import Package
  567. # Keep track of any collected nodes in here, so we don't duplicate fixtures.
  568. node_cache1: Dict[py.path.local, Sequence[nodes.Collector]] = {}
  569. node_cache2: Dict[
  570. Tuple[Type[nodes.Collector], py.path.local], nodes.Collector
  571. ] = ({})
  572. # Keep track of any collected collectors in matchnodes paths, so they
  573. # are not collected more than once.
  574. matchnodes_cache: Dict[Tuple[Type[nodes.Collector], str], CollectReport] = ({})
  575. # Dirnames of pkgs with dunder-init files.
  576. pkg_roots: Dict[str, Package] = {}
  577. for argpath, names in self._initial_parts:
  578. self.trace("processing argument", (argpath, names))
  579. self.trace.root.indent += 1
  580. # Start with a Session root, and delve to argpath item (dir or file)
  581. # and stack all Packages found on the way.
  582. # No point in finding packages when collecting doctests.
  583. if not self.config.getoption("doctestmodules", False):
  584. pm = self.config.pluginmanager
  585. for parent in reversed(argpath.parts()):
  586. if pm._confcutdir and pm._confcutdir.relto(parent):
  587. break
  588. if parent.isdir():
  589. pkginit = parent.join("__init__.py")
  590. if pkginit.isfile() and pkginit not in node_cache1:
  591. col = self._collectfile(pkginit, handle_dupes=False)
  592. if col:
  593. if isinstance(col[0], Package):
  594. pkg_roots[str(parent)] = col[0]
  595. node_cache1[col[0].fspath] = [col[0]]
  596. # If it's a directory argument, recurse and look for any Subpackages.
  597. # Let the Package collector deal with subnodes, don't collect here.
  598. if argpath.check(dir=1):
  599. assert not names, "invalid arg {!r}".format((argpath, names))
  600. seen_dirs: Set[py.path.local] = set()
  601. for direntry in visit(str(argpath), self._recurse):
  602. if not direntry.is_file():
  603. continue
  604. path = py.path.local(direntry.path)
  605. dirpath = path.dirpath()
  606. if dirpath not in seen_dirs:
  607. # Collect packages first.
  608. seen_dirs.add(dirpath)
  609. pkginit = dirpath.join("__init__.py")
  610. if pkginit.exists():
  611. for x in self._collectfile(pkginit):
  612. yield x
  613. if isinstance(x, Package):
  614. pkg_roots[str(dirpath)] = x
  615. if str(dirpath) in pkg_roots:
  616. # Do not collect packages here.
  617. continue
  618. for x in self._collectfile(path):
  619. key = (type(x), x.fspath)
  620. if key in node_cache2:
  621. yield node_cache2[key]
  622. else:
  623. node_cache2[key] = x
  624. yield x
  625. else:
  626. assert argpath.check(file=1)
  627. if argpath in node_cache1:
  628. col = node_cache1[argpath]
  629. else:
  630. collect_root = pkg_roots.get(argpath.dirname, self)
  631. col = collect_root._collectfile(argpath, handle_dupes=False)
  632. if col:
  633. node_cache1[argpath] = col
  634. matching = []
  635. work: List[
  636. Tuple[Sequence[Union[nodes.Item, nodes.Collector]], Sequence[str]]
  637. ] = [(col, names)]
  638. while work:
  639. self.trace("matchnodes", col, names)
  640. self.trace.root.indent += 1
  641. matchnodes, matchnames = work.pop()
  642. for node in matchnodes:
  643. if not matchnames:
  644. matching.append(node)
  645. continue
  646. if not isinstance(node, nodes.Collector):
  647. continue
  648. key = (type(node), node.nodeid)
  649. if key in matchnodes_cache:
  650. rep = matchnodes_cache[key]
  651. else:
  652. rep = collect_one_node(node)
  653. matchnodes_cache[key] = rep
  654. if rep.passed:
  655. submatchnodes = []
  656. for r in rep.result:
  657. # TODO: Remove parametrized workaround once collection structure contains
  658. # parametrization.
  659. if (
  660. r.name == matchnames[0]
  661. or r.name.split("[")[0] == matchnames[0]
  662. ):
  663. submatchnodes.append(r)
  664. if submatchnodes:
  665. work.append((submatchnodes, matchnames[1:]))
  666. # XXX Accept IDs that don't have "()" for class instances.
  667. elif len(rep.result) == 1 and rep.result[0].name == "()":
  668. work.append((rep.result, matchnames))
  669. else:
  670. # Report collection failures here to avoid failing to run some test
  671. # specified in the command line because the module could not be
  672. # imported (#134).
  673. node.ihook.pytest_collectreport(report=rep)
  674. self.trace("matchnodes finished -> ", len(matching), "nodes")
  675. self.trace.root.indent -= 1
  676. if not matching:
  677. report_arg = "::".join((str(argpath), *names))
  678. self._notfound.append((report_arg, col))
  679. continue
  680. # If __init__.py was the only file requested, then the matched
  681. # node will be the corresponding Package (by default), and the
  682. # first yielded item will be the __init__ Module itself, so
  683. # just use that. If this special case isn't taken, then all the
  684. # files in the package will be yielded.
  685. if argpath.basename == "__init__.py" and isinstance(
  686. matching[0], Package
  687. ):
  688. try:
  689. yield next(iter(matching[0].collect()))
  690. except StopIteration:
  691. # The package collects nothing with only an __init__.py
  692. # file in it, which gets ignored by the default
  693. # "python_files" option.
  694. pass
  695. continue
  696. yield from matching
  697. self.trace.root.indent -= 1
  698. def genitems(
  699. self, node: Union[nodes.Item, nodes.Collector]
  700. ) -> Iterator[nodes.Item]:
  701. self.trace("genitems", node)
  702. if isinstance(node, nodes.Item):
  703. node.ihook.pytest_itemcollected(item=node)
  704. yield node
  705. else:
  706. assert isinstance(node, nodes.Collector)
  707. rep = collect_one_node(node)
  708. if rep.passed:
  709. for subnode in rep.result:
  710. yield from self.genitems(subnode)
  711. node.ihook.pytest_collectreport(report=rep)
  712. def search_pypath(module_name: str) -> str:
  713. """Search sys.path for the given a dotted module name, and return its file system path."""
  714. try:
  715. spec = importlib.util.find_spec(module_name)
  716. # AttributeError: looks like package module, but actually filename
  717. # ImportError: module does not exist
  718. # ValueError: not a module name
  719. except (AttributeError, ImportError, ValueError):
  720. return module_name
  721. if spec is None or spec.origin is None or spec.origin == "namespace":
  722. return module_name
  723. elif spec.submodule_search_locations:
  724. return os.path.dirname(spec.origin)
  725. else:
  726. return spec.origin
  727. def resolve_collection_argument(
  728. invocation_path: Path, arg: str, *, as_pypath: bool = False
  729. ) -> Tuple[py.path.local, List[str]]:
  730. """Parse path arguments optionally containing selection parts and return (fspath, names).
  731. Command-line arguments can point to files and/or directories, and optionally contain
  732. parts for specific tests selection, for example:
  733. "pkg/tests/test_foo.py::TestClass::test_foo"
  734. This function ensures the path exists, and returns a tuple:
  735. (py.path.path("/full/path/to/pkg/tests/test_foo.py"), ["TestClass", "test_foo"])
  736. When as_pypath is True, expects that the command-line argument actually contains
  737. module paths instead of file-system paths:
  738. "pkg.tests.test_foo::TestClass::test_foo"
  739. In which case we search sys.path for a matching module, and then return the *path* to the
  740. found module.
  741. If the path doesn't exist, raise UsageError.
  742. If the path is a directory and selection parts are present, raise UsageError.
  743. """
  744. strpath, *parts = str(arg).split("::")
  745. if as_pypath:
  746. strpath = search_pypath(strpath)
  747. fspath = invocation_path / strpath
  748. fspath = absolutepath(fspath)
  749. if not fspath.exists():
  750. msg = (
  751. "module or package not found: {arg} (missing __init__.py?)"
  752. if as_pypath
  753. else "file or directory not found: {arg}"
  754. )
  755. raise UsageError(msg.format(arg=arg))
  756. if parts and fspath.is_dir():
  757. msg = (
  758. "package argument cannot contain :: selection parts: {arg}"
  759. if as_pypath
  760. else "directory argument cannot contain :: selection parts: {arg}"
  761. )
  762. raise UsageError(msg.format(arg=arg))
  763. return py.path.local(str(fspath)), parts