Coverage for python/lsst/ctrl/mpexec/cli/opt/options.py: 83%
86 statements
« prev ^ index » next coverage.py v7.14.3, created at 2026-07-09 02:05 -0700
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30from collections.abc import Iterable, Mapping
31from typing import TYPE_CHECKING
33import click
35from lsst.daf.butler.cli.utils import MWOptionDecorator, MWPath, split_commas, unwrap
36from lsst.utils.doImport import doImportType
38if TYPE_CHECKING:
39 # Avoid regular module-scope import of test-only code that tinkers with the
40 # storage class singleton.
41 from lsst.pipe.base.tests.mocks import ForcedFailure
44butler_config_option = MWOptionDecorator(
45 "-b", "--butler-config", help="Location of the gen3 butler/registry config file."
46)
49data_query_option = MWOptionDecorator(
50 "-d", "--data-query", help="User data selection expression.", metavar="QUERY"
51)
54data_id_table_option = MWOptionDecorator(
55 "--data-id-table",
56 multiple=True,
57 default=(),
58 help=(
59 "URI to table of data IDs to join as a constraint; may be any format accepted by astropy.table. "
60 "May be passed multiple times."
61 ),
62 metavar="URI",
63)
66debug_option = MWOptionDecorator(
67 "--debug", help="Enable debugging output using lsstDebug facility (imports debug.py).", is_flag=True
68)
70coverage_option = MWOptionDecorator(
71 "--coverage", help="Enable coverage output (requires coverage package).", is_flag=True
72)
74coverage_report_option = MWOptionDecorator(
75 "--cov-report/--no-cov-report",
76 help="If coverage is enabled, controls whether to produce an HTML coverage report.",
77 default=True,
78)
80coverage_packages_option = MWOptionDecorator(
81 "--cov-packages",
82 help=unwrap(
83 """Python packages to restrict coverage to. If none are provided, runs coverage on all packages."""
84 ),
85 multiple=True,
86 callback=split_commas,
87)
89delete_option = MWOptionDecorator(
90 "--delete", callback=split_commas, help="Delete task with given label from pipeline.", multiple=True
91)
94pdb_option = MWOptionDecorator(
95 "--pdb",
96 help="Post-mortem debugger to launch for exceptions (defaults to pdb if unspecified; requires a tty).",
97 is_flag=False,
98 flag_value="pdb",
99 default=None,
100)
103extend_run_option = MWOptionDecorator(
104 "--extend-run",
105 help=(
106 "Instead of creating a new RUN collection, insert datasets into either the one given by "
107 "--output-run (if provided) or the first child collection of --output (which must be of type RUN). "
108 "This also enables --skip-existing option when building a graph. "
109 "When executing a graph this option skips quanta with all existing outputs."
110 ),
111 is_flag=True,
112)
115graph_fixup_option = MWOptionDecorator(
116 "--graph-fixup",
117 help="Name of the class or factory method which makes an instance used for execution graph fixup.",
118)
121init_only_option = MWOptionDecorator(
122 "--init-only",
123 help="Do not actually run; just register dataset types and/or save init outputs.",
124 is_flag=True,
125)
128input_option = MWOptionDecorator(
129 "-i",
130 "--input",
131 callback=split_commas,
132 default=[],
133 help="Comma-separated names of the input collection(s).",
134 metavar="COLLECTION",
135 multiple=True,
136)
139rebase_option = MWOptionDecorator(
140 "--rebase",
141 help=unwrap("""Reset output collection chain if it is inconsistent with --inputs"""),
142 is_flag=True,
143)
146no_versions_option = MWOptionDecorator(
147 "--no-versions", help="Do not save or check package versions.", is_flag=True
148)
151output_option = MWOptionDecorator(
152 "-o",
153 "--output",
154 help=unwrap(
155 """Name of the output CHAINED collection. This may either be an
156 existing CHAINED collection to use as both input and output
157 (incompatible with --input), or a new CHAINED collection created
158 to include all inputs (requires --input). In both cases, the
159 collection's children will start with an output RUN collection
160 that directly holds all new datasets (see --output-run)."""
161 ),
162 metavar="COLL",
163)
166output_run_option = MWOptionDecorator(
167 "--output-run",
168 help=unwrap(
169 """Name of the new output RUN collection. If not provided
170 then --output must be provided and a new RUN collection will
171 be created by appending a timestamp to the value passed with
172 --output. If this collection already exists then
173 --extend-run must be passed."""
174 ),
175 metavar="COLL",
176)
179pipeline_option = MWOptionDecorator(
180 "-p",
181 "--pipeline",
182 help="Location of a pipeline definition file in YAML format.",
183 type=MWPath(file_okay=True, dir_okay=False, readable=True),
184)
187pipeline_dot_option = MWOptionDecorator(
188 "--pipeline-dot",
189 help="Location for storing GraphViz DOT representation of a pipeline.",
190 type=MWPath(writable=True, file_okay=True, dir_okay=False),
191)
194pipeline_mermaid_option = MWOptionDecorator(
195 "--pipeline-mermaid",
196 help="Location for storing Mermaid representation of a pipeline.",
197 type=MWPath(writable=True, file_okay=True, dir_okay=False),
198)
201profile_option = MWOptionDecorator(
202 "--profile", help="Dump cProfile statistics to file name.", type=MWPath(file_okay=True, dir_okay=False)
203)
206prune_replaced_option = MWOptionDecorator(
207 "--prune-replaced",
208 help=unwrap(
209 """Delete the datasets in the collection replaced by
210 --replace-run, either just from the datastore
211 ('unstore') or by removing them and the RUN completely
212 ('purge'). Requires --replace-run."""
213 ),
214 type=click.Choice(choices=("unstore", "purge"), case_sensitive=False),
215)
218qgraph_option = MWOptionDecorator(
219 "-g",
220 "--qgraph",
221 help=unwrap(
222 """Location for a serialized quantum graph definition (pickle
223 file). If this option is given then all input data options and
224 pipeline-building options cannot be used. Can be a URI."""
225 ),
226)
229qgraph_id_option = MWOptionDecorator(
230 "--qgraph-id",
231 help=unwrap(
232 """Quantum graph identifier, if specified must match the
233 identifier of the graph loaded from a file. Ignored if graph
234 is not loaded from a file."""
235 ),
236)
239qgraph_datastore_records_option = MWOptionDecorator(
240 "--qgraph-datastore-records",
241 help=unwrap(
242 """Include datastore records into generated quantum graph, these records are used by a
243 quantum-backed butler.
244 """
245 ),
246 is_flag=True,
247)
250# I wanted to use default=None here to match Python API but click silently
251# replaces None with an empty tuple when multiple=True.
252qgraph_node_id_option = MWOptionDecorator(
253 "--qgraph-node-id",
254 callback=split_commas,
255 multiple=True,
256 help=unwrap(
257 """Only load a specified set of nodes when graph is
258 loaded from a file, nodes are identified by UUID
259 values. One or more comma-separated integers are
260 accepted. By default all nodes are loaded. Ignored if
261 graph is not loaded from a file."""
262 ),
263)
265qgraph_dot_option = MWOptionDecorator(
266 "--qgraph-dot",
267 help="Location for storing GraphViz DOT representation of a quantum graph.",
268 type=MWPath(writable=True, file_okay=True, dir_okay=False),
269)
271qgraph_mermaid_option = MWOptionDecorator(
272 "--qgraph-mermaid",
273 help="Location for storing Mermaid representation of a quantum graph.",
274 type=MWPath(writable=True, file_okay=True, dir_okay=False),
275)
278replace_run_option = MWOptionDecorator(
279 "--replace-run",
280 help=unwrap(
281 """Before creating a new RUN collection in an existing
282 CHAINED collection, remove the first child collection
283 (which must be of type RUN). This can be used to repeatedly
284 write to the same (parent) collection during development,
285 but it does not delete the datasets associated with the
286 replaced run unless --prune-replaced is also passed.
287 Requires --output, and incompatible with --extend-run."""
288 ),
289 is_flag=True,
290)
293save_pipeline_option = MWOptionDecorator(
294 "-s",
295 "--save-pipeline",
296 help="Location for storing resulting pipeline definition in YAML format.",
297 type=MWPath(dir_okay=False, file_okay=True, writable=True),
298)
300save_qgraph_option = MWOptionDecorator(
301 "-q",
302 "--save-qgraph",
303 help="URI location for storing a serialized quantum graph definition (pickle file).",
304)
307show_option = MWOptionDecorator(
308 "--show",
309 callback=split_commas,
310 help=unwrap(
311 """Dump various info to standard output. Possible items are:
312 ``config``, ``config=[Task::]<PATTERN>`` or
313 ``config=[Task::]<PATTERN>:NOIGNORECASE`` to dump configuration
314 fields possibly matching given pattern and/or task label;
315 ``history=<FIELD>`` to dump configuration history for a field,
316 field name is specified as ``[Task::]<PATTERN>``; ``dump-config``,
317 ``dump-config=Task`` to dump complete configuration for a task
318 given its label or all tasks; ``pipeline`` to show pipeline
319 composition; ``graph`` to show information about quanta;
320 ``workflow`` to show information about quanta and their
321 dependency; ``tasks`` to show task composition; ``subsets`` to
322 show subset labels and associated tasks; ``uri`` to show
323 predicted dataset URIs of quanta; ``pipeline-graph`` for a
324 text-based visualization of the pipeline (tasks and dataset types);
325 ``task-graph`` for a text-based visualization of just the tasks;
326 ``inputs`` for a list of overall-input dataset types.
327 With -b, pipeline-graph and task-graph include additional information.
328 """
329 ),
330 metavar="ITEM|ITEM=VALUE",
331 multiple=True,
332)
335skip_existing_in_option = MWOptionDecorator(
336 "--skip-existing-in",
337 callback=split_commas,
338 default=None,
339 metavar="COLLECTION",
340 multiple=True,
341 help=unwrap(
342 """If all Quantum outputs already exist in the specified list of
343 collections then that Quantum will be excluded from the QuantumGraph.
344 """
345 ),
346)
349skip_existing_option = MWOptionDecorator(
350 "--skip-existing",
351 is_flag=True,
352 help=unwrap(
353 """This option is equivalent to --skip-existing-in with the name of
354 the output RUN collection. If both --skip-existing-in and
355 --skip-existing are given then output RUN collection is appended to
356 the list of collections."""
357 ),
358)
361retained_dataset_types_option = MWOptionDecorator(
362 "--retained-dataset-types",
363 default=None,
364 metavar="PATH",
365 type=MWPath(file_okay=True, dir_okay=False, readable=True),
366 help=unwrap(
367 """Path to a YAML file listing dataset type names or glob-style wildcard
368 patterns that should exist in --skip-existing-in when the producing
369 task ran successfully. When a quantum should run, the builder propagates the
370 must-run signal backward through non-retained input datasets, forcing
371 the upstream quanta that need to regenerate those intermediates to also
372 run. Has no effect without ``skip_existing_in``.
373 """
374 ),
375)
378def parse_prune_unanchored_quanta(
379 ctx: click.Context, param: click.Option, value: str | None
380) -> tuple[str, str] | None:
381 """Parse the --prune-unanchored-quanta option value into a tuple.
383 Parameters
384 ----------
385 ctx : `click.Context`
386 Context provided by Click.
387 param : `click.Option`
388 Click option.
389 value : `str` or `None`
390 Value from option, expected to be ``SOURCE:ANCHOR`` or `None` if the
391 option was not provided.
393 Returns
394 -------
395 result : `tuple` [`str`, `str`] or `None`
396 A ``(source_label, anchor_label)`` tuple, or `None` if ``value`` is
397 `None`.
399 Raises
400 ------
401 click.UsageError
402 Raised if ``value`` is not `None` and does not match the
403 ``SOURCE:ANCHOR`` format.
404 """
405 if value is None:
406 return None
407 parts = value.split(":", 1)
408 if len(parts) != 2 or not parts[0] or not parts[1]: 408 ↛ 409line 408 didn't jump to line 409 because the condition on line 408 was never true
409 raise click.UsageError(
410 f"Invalid value for --prune-unanchored-quanta: {value!r}; expected SOURCE:ANCHOR."
411 )
412 return (parts[0], parts[1])
415prune_unanchored_quanta_option = MWOptionDecorator(
416 "--prune-unanchored-quanta",
417 callback=parse_prune_unanchored_quanta,
418 default=None,
419 metavar="SOURCE:ANCHOR",
420 help=unwrap(
421 """Remove source quanta that have no reachable anchor quantum downstream,
422 along with their entire downstream chain. Specify as a colon-separated
423 pair of task labels SOURCE:ANCHOR."""
424 ),
425)
428clobber_outputs_option = MWOptionDecorator(
429 "--clobber-outputs",
430 help=(
431 "Remove outputs of failed quanta from the output run when they would block the execution of new "
432 "quanta with the same data ID (or assume that this will be done, if just building a QuantumGraph). "
433 "Does nothing if --extend-run is not passed."
434 ),
435 is_flag=True,
436)
439no_existing_outputs_option = MWOptionDecorator(
440 "--no-existing-outputs",
441 help=(
442 "Assume that no predicted outputs already exist in the output run collection. "
443 "This will eliminate existence checks that otherwise run before each quantum is executed."
444 ),
445 is_flag=True,
446)
449skip_init_writes_option = MWOptionDecorator(
450 "--skip-init-writes",
451 help=unwrap(
452 """Do not write collection-wide 'init output' datasets
453 (e.g.schemas)."""
454 ),
455 is_flag=True,
456)
459enable_implicit_threading_option = MWOptionDecorator(
460 "--enable-implicit-threading",
461 help=unwrap(
462 """Do not disable implicit threading use by third-party libraries (e.g. OpenBLAS).
463 Implicit threading is always disabled during execution with multiprocessing."""
464 ),
465 is_flag=True,
466)
468cores_per_quantum_option = MWOptionDecorator(
469 "-n",
470 "--cores-per-quantum",
471 default=1,
472 help=unwrap(
473 """Number of cores available to each quantum when executing.
474 If '-j' is used each subprocess will be allowed to use this number of cores."""
475 ),
476 type=click.IntRange(min=1),
477)
479memory_per_quantum_option = MWOptionDecorator(
480 "--memory-per-quantum",
481 default="",
482 help=unwrap(
483 """Memory allocated for each quantum to use when executing.
484 This memory allocation is not enforced by the execution system and is purely advisory.
485 If '-j' used each subprocess will be allowed to use this amount of memory.
486 Units are allowed and the default units for a plain integer are MB.
487 For example: '3GB', '3000MB' and '3000' would all result in the same
488 memory limit. Default is for no limit."""
489 ),
490 type=str,
491)
493task_option = MWOptionDecorator(
494 "-t",
495 "--task",
496 callback=split_commas,
497 help=unwrap(
498 """Task name to add to pipeline, must be a fully qualified task
499 name. Task name can be followed by colon and label name, if label
500 is not given then task base name (class name) is used as
501 label."""
502 ),
503 metavar="TASK[:LABEL]",
504 multiple=True,
505)
507select_tasks_option = MWOptionDecorator(
508 "--select-tasks",
509 metavar="EXPR",
510 default="",
511 help=(
512 "A string expression that filters the tasks to run from the pipeline. "
513 "See https://pipelines.lsst.io/v/weekly/modules/lsst.pipe.base/working-with-pipeline-graphs.html"
514 "#pipeline-graph-subset-expressions for details."
515 ),
516)
519timeout_option = MWOptionDecorator(
520 "--timeout", type=click.IntRange(min=0), help="Timeout for multiprocessing; maximum wall time (sec)."
521)
524start_method_option = MWOptionDecorator(
525 "--start-method",
526 default=None,
527 type=click.Choice(choices=["spawn", "fork", "forkserver"]),
528 help=(
529 "Multiprocessing start method, default is platform-specific. "
530 "Fork method is no longer supported, spawn is used instead if fork is selected."
531 ),
532)
535fail_fast_option = MWOptionDecorator(
536 "--fail-fast",
537 help="Stop processing at first error, default is to process as many tasks as possible.",
538 is_flag=True,
539)
541raise_on_partial_outputs_option = MWOptionDecorator(
542 "--raise-on-partial-outputs/--no-raise-on-partial-outputs",
543 help="Consider partial outputs from a task an error instead of a qualified success.",
544 is_flag=True,
545 default=True,
546)
548mock_option = MWOptionDecorator(
549 "--mock",
550 help="Mock pipeline execution.",
551 is_flag=True,
552)
554unmocked_dataset_types_option = MWOptionDecorator(
555 "--unmocked-dataset-types",
556 callback=split_commas,
557 default=None,
558 metavar="COLLECTION",
559 multiple=True,
560 help="Names of input dataset types that should not be mocked.",
561)
564def parse_mock_failure(
565 ctx: click.Context, param: click.Option, value: Iterable[str] | None
566) -> Mapping[str, ForcedFailure]:
567 """Parse the --mock-failure option values into the mapping accepted by
568 `~lsst.pipe.base.tests.mocks.mock_task_defs`.
570 Parameters
571 ----------
572 ctx : `click.Context`
573 Context provided by Click.
574 param : `click.Option`
575 Click option.
576 value : `~collections.abc.Iterable` [`str`] or `None`
577 Value from option.
578 """
579 # Avoid regular module-scope import of test-only code that tinkers with the
580 # storage class singleton.
581 from lsst.pipe.base.tests.mocks import ForcedFailure
583 result: dict[str, ForcedFailure] = {}
584 if value is None: 584 ↛ 585line 584 didn't jump to line 585 because the condition on line 584 was never true
585 return result
586 for entry in value: 586 ↛ 587line 586 didn't jump to line 587 because the loop on line 586 never started
587 try:
588 task_label, error_type_name, where, *rest = entry.split(":")
589 if rest:
590 (memory_required,) = rest
591 else:
592 memory_required = None
593 except ValueError:
594 raise click.UsageError(
595 f"Invalid value for --mock-failure option: {entry!r}; "
596 "expected a string of the form 'task:error:where[:mem]'."
597 ) from None
598 error_type = doImportType(error_type_name) if error_type_name else None
599 result[task_label] = ForcedFailure(where, error_type, memory_required)
600 return result
603mock_failure_option = MWOptionDecorator(
604 "--mock-failure",
605 callback=parse_mock_failure,
606 metavar="LABEL:EXCEPTION:WHERE",
607 default=None,
608 multiple=True,
609 help=unwrap(
610 """Specifications for tasks that should be configured to fail
611 when mocking execution. This is a colon-separated 3-tuple or 4-tuple,
612 where the first entry the task label, the second the fully-qualified
613 exception type (empty for ValueError, and the third a string (which
614 typically needs to be quoted to be passed as one argument value by the
615 shell) of the form passed to --where, indicating which data IDs should
616 fail. The final optional term is the memory "required" by the task
617 (with units recognized by astropy), which will cause the error to only
618 occur if the "available" memory (according to
619 ExecutionResources.max_mem) is less than this value. Note that actual
620 memory usage is irrelevant here; this is all mock behavior."""
621 ),
622)
624dataset_query_constraint = MWOptionDecorator(
625 "--dataset-query-constraint",
626 help=unwrap(
627 """When constructing a quantum graph constrain by
628 pre-existence of specified dataset types. Valid
629 values are `all` for all inputs dataset types in
630 pipeline, ``off`` to not consider dataset type
631 existence as a constraint, single or comma
632 separated list of dataset type names."""
633 ),
634 default="all",
635)
637summary_option = MWOptionDecorator(
638 "--summary",
639 help=(
640 "Location for storing job summary (JSON file). Note that the"
641 " structure of this file may not be stable."
642 ),
643 type=MWPath(dir_okay=False, file_okay=True, writable=True),
644)
647recursive_option = MWOptionDecorator(
648 "--recursive",
649 is_flag=True,
650)
652config_search_path_option = MWOptionDecorator(
653 "--config-search-path",
654 callback=split_commas,
655 default=[],
656 help="Additional search paths for butler configuration.",
657 metavar="PATH",
658 multiple=True,
659)
661update_graph_id_option = MWOptionDecorator(
662 "--update-graph-id",
663 help=unwrap("Update graph ID with new unique value."),
664 is_flag=True,
665)
667metadata_run_key_option = MWOptionDecorator(
668 "--metadata-run-key",
669 help=(
670 "Quantum graph metadata key for the name of the output run. "
671 "Empty string disables update of the metadata. "
672 "Default value: output_run."
673 ),
674 default="output_run",
675)