API Reference

Top-level package for H&H Ensemble Modeling Toolkit.

CLIValidationError

Bases: TRITONSWMMError

CLI argument validation failure (exit code 2).

Raised when command-line arguments fail business logic validation, such as mutually exclusive flags, conditional requirements, or invalid argument combinations.

Attributes: argument: The argument(s) that failed validation fix_hint: Optional hint for how to fix the issue

Source code in src/hhemt/exceptions.py

class CLIValidationError(TRITONSWMMError):
    """CLI argument validation failure (exit code 2).

    Raised when command-line arguments fail business logic validation,
    such as mutually exclusive flags, conditional requirements, or
    invalid argument combinations.

    Attributes:
        argument: The argument(s) that failed validation
        fix_hint: Optional hint for how to fix the issue
    """

    def __init__(self, argument: str, message: str, fix_hint: str = ""):
        self.argument = argument
        self.fix_hint = fix_hint

        lines = [f"Invalid argument: {argument}", f"  {message}"]
        if fix_hint:
            lines.append(f"  Fix: {fix_hint}")

        super().__init__("\n".join(lines))

CompilationError

Bases: TRITONSWMMError

TRITON/SWMM compilation failure.

Raised when CMake build or make compilation fails for any model type.

Attributes: model_type: Which model failed ('triton', 'tritonswmm', 'swmm') backend: Compilation backend ('cpu', 'gpu', 'openmp', etc.) logfile: Path to compilation log file with detailed error output return_code: Process return code from compilation command

Source code in src/hhemt/exceptions.py

class CompilationError(TRITONSWMMError):
    """TRITON/SWMM compilation failure.

    Raised when CMake build or make compilation fails for any model type.

    Attributes:
        model_type: Which model failed ('triton', 'tritonswmm', 'swmm')
        backend: Compilation backend ('cpu', 'gpu', 'openmp', etc.)
        logfile: Path to compilation log file with detailed error output
        return_code: Process return code from compilation command
    """

    def __init__(self, model_type: str, backend: str, logfile: Path, return_code: int):
        self.model_type = model_type
        self.backend = backend
        self.logfile = logfile
        self.return_code = return_code

        super().__init__(
            f"{model_type.upper()} {backend.upper()} compilation failed\n"
            f"  Return code: {return_code}\n"
            f"  Log: {logfile}\n"
            f"  Run: cat {logfile}"
        )

ConfigurationError

Bases: TRITONSWMMError

Invalid configuration values or toggle conflicts.

Raised when: - Required fields are missing based on toggle states - Mutually exclusive options are both enabled - Configuration values fail validation rules

Attributes: field: The configuration field that failed validation config_path: Optional path to the configuration file

Source code in src/hhemt/exceptions.py

class ConfigurationError(TRITONSWMMError):
    """Invalid configuration values or toggle conflicts.

    Raised when:
    - Required fields are missing based on toggle states
    - Mutually exclusive options are both enabled
    - Configuration values fail validation rules

    Attributes:
        field: The configuration field that failed validation
        config_path: Optional path to the configuration file
    """

    def __init__(self, field: str, message: str, config_path: Path | None = None):
        self.field = field
        self.config_path = config_path

        lines = [f"Configuration error in field '{field}'"]
        if config_path:
            lines.append(f"  Config: {config_path}")
        lines.append(f"  {message}")

        super().__init__("\n".join(lines))

ProcessingError

Bases: TRITONSWMMError

Output processing failure.

Raised when post-simulation processing operations fail (parsing outputs, compressing files, generating summaries).

Attributes: operation: Description of the operation that failed filepath: Optional path to the file being processed reason: Optional detailed error reason

Source code in src/hhemt/exceptions.py

class ProcessingError(TRITONSWMMError):
    """Output processing failure.

    Raised when post-simulation processing operations fail (parsing
    outputs, compressing files, generating summaries).

    Attributes:
        operation: Description of the operation that failed
        filepath: Optional path to the file being processed
        reason: Optional detailed error reason
    """

    def __init__(self, operation: str, filepath: Path | None = None, reason: str = ""):
        self.operation = operation
        self.filepath = filepath
        self.reason = reason

        lines = [f"Output processing failed: {operation}"]
        if filepath:
            lines.append(f"  File: {filepath}")
        if reason:
            lines.append(f"  Reason: {reason}")

        super().__init__("\n".join(lines))

ResourceAllocationError

Bases: TRITONSWMMError

Resource allocation failure for simulations.

Raised when CPU/GPU/memory resource allocation fails or is inconsistent with configuration.

Attributes: resource_type: Which resource failed ('cpu', 'gpu', 'memory') requested: Requested resource amount available: Available resource amount (if known)

Source code in src/hhemt/exceptions.py

class ResourceAllocationError(TRITONSWMMError):
    """Resource allocation failure for simulations.

    Raised when CPU/GPU/memory resource allocation fails or is
    inconsistent with configuration.

    Attributes:
        resource_type: Which resource failed ('cpu', 'gpu', 'memory')
        requested: Requested resource amount
        available: Available resource amount (if known)
    """

    def __init__(self, resource_type: str, requested: str, available: str | None = None):
        self.resource_type = resource_type
        self.requested = requested
        self.available = available

        lines = [f"Resource allocation failed for {resource_type}", f"  Requested: {requested}"]
        if available:
            lines.append(f"  Available: {available}")

        super().__init__("\n".join(lines))

SLURMError

Bases: TRITONSWMMError

SLURM job submission or execution failure.

Raised when SLURM operations fail (job submission, resource allocation, job monitoring).

Attributes: operation: Which SLURM operation failed ('submit', 'monitor', 'allocate') job_id: Optional SLURM job ID reason: Optional detailed error reason

Source code in src/hhemt/exceptions.py

class SLURMError(TRITONSWMMError):
    """SLURM job submission or execution failure.

    Raised when SLURM operations fail (job submission, resource
    allocation, job monitoring).

    Attributes:
        operation: Which SLURM operation failed ('submit', 'monitor', 'allocate')
        job_id: Optional SLURM job ID
        reason: Optional detailed error reason
    """

    def __init__(self, operation: str, job_id: str | None = None, reason: str = ""):
        self.operation = operation
        self.job_id = job_id
        self.reason = reason

        lines = [f"SLURM operation failed: {operation}"]
        if job_id:
            lines.append(f"  Job ID: {job_id}")
        if reason:
            lines.append(f"  Reason: {reason}")

        super().__init__("\n".join(lines))

SimulationError

Bases: TRITONSWMMError

Simulation execution failure.

Raised when a TRITON/SWMM simulation process fails during execution.

Attributes: event_iloc: Index of the weather event that failed model_type: Which model failed ('triton', 'tritonswmm', 'swmm') logfile: Optional path to simulation log file

Source code in src/hhemt/exceptions.py

class SimulationError(TRITONSWMMError):
    """Simulation execution failure.

    Raised when a TRITON/SWMM simulation process fails during execution.

    Attributes:
        event_iloc: Index of the weather event that failed
        model_type: Which model failed ('triton', 'tritonswmm', 'swmm')
        logfile: Optional path to simulation log file
    """

    def __init__(self, event_iloc: int, model_type: str, logfile: Path | None = None):
        self.event_iloc = event_iloc
        self.model_type = model_type
        self.logfile = logfile

        lines = [f"Simulation failed for event_iloc={event_iloc} (model={model_type})"]
        if logfile:
            lines.append(f"  Log: {logfile}")

        super().__init__("\n".join(lines))

TRITONSWMMError

Bases: Exception

Base exception for all TRITON-SWMM toolkit errors.

Catch this to handle all toolkit-specific exceptions.

Source code in src/hhemt/exceptions.py

class TRITONSWMMError(Exception):
    """Base exception for all TRITON-SWMM toolkit errors.

    Catch this to handle all toolkit-specific exceptions.
    """

    pass

Toolkit

High-level API facade for TRITON-SWMM workflow orchestration.

This class provides a simplified interface to the toolkit, wrapping the underlying Analysis class with notebook-friendly methods and sensible defaults.

Attributes: system: The TRITONSWMM_system instance containing system configuration analysis: The TRITONSWMM_analysis instance for workflow orchestration

Example: Basic workflow execution:

>>> from hhemt import Toolkit
>>>
>>> # Load configurations
>>> tk = Toolkit.from_configs(
...     system_config="configs/system.yaml",
...     analysis_config="configs/analysis.yaml"
... )
>>>
>>> # Run from scratch
>>> result = tk.run(mode="fresh")
>>> if result.success:
...     print(f"✓ Workflow complete: {len(result.events_processed)} events")
... else:
...     print(f"✗ Workflow failed: {result.message}")

Resume interrupted workflow:

>>> # Check current status
>>> status = tk.get_status()
>>> print(status)
>>> print(f"Recommendation: {status.recommendation}")
>>>
>>> # Resume from last checkpoint
>>> result = tk.run(mode=status.recommended_mode)

Run specific events only:

>>> # Process events 0-4 only
>>> result = tk.run(mode="resume", events=list(range(5)))

Run specific workflow phases:

>>> # Only run simulation phase (skip setup/preparation)
>>> result = tk.run(
...     mode="resume",
...     phases=["simulation"]
... )

Source code in src/hhemt/toolkit.py

class Toolkit:
    """High-level API facade for TRITON-SWMM workflow orchestration.

    This class provides a simplified interface to the toolkit, wrapping the
    underlying Analysis class with notebook-friendly methods and sensible defaults.

    Attributes:
        system: The TRITONSWMM_system instance containing system configuration
        analysis: The TRITONSWMM_analysis instance for workflow orchestration

    Example:
        Basic workflow execution:

        >>> from hhemt import Toolkit
        >>>
        >>> # Load configurations
        >>> tk = Toolkit.from_configs(
        ...     system_config="configs/system.yaml",
        ...     analysis_config="configs/analysis.yaml"
        ... )
        >>>
        >>> # Run from scratch
        >>> result = tk.run(mode="fresh")
        >>> if result.success:
        ...     print(f"✓ Workflow complete: {len(result.events_processed)} events")
        ... else:
        ...     print(f"✗ Workflow failed: {result.message}")

        Resume interrupted workflow:

        >>> # Check current status
        >>> status = tk.get_status()
        >>> print(status)
        >>> print(f"Recommendation: {status.recommendation}")
        >>>
        >>> # Resume from last checkpoint
        >>> result = tk.run(mode=status.recommended_mode)

        Run specific events only:

        >>> # Process events 0-4 only
        >>> result = tk.run(mode="resume", events=list(range(5)))

        Run specific workflow phases:

        >>> # Only run simulation phase (skip setup/preparation)
        >>> result = tk.run(
        ...     mode="resume",
        ...     phases=["simulation"]
        ... )
    """

    def __init__(self, system: "TRITONSWMM_system"):
        """Initialize Toolkit with a system instance.

        Args:
            system: Initialized TRITONSWMM_system instance

        Note:
            Prefer using Toolkit.from_configs() for simpler initialization.
        """
        self.system = system
        self.analysis = system.analysis

    @classmethod
    def from_configs(
        cls,
        system_config: str | Path,
        analysis_config: str | Path,
        hpc_system_config: str | Path | None = None,
        validate: bool = True,
    ) -> "Toolkit":
        """Create Toolkit instance from configuration files.

        This is the recommended way to initialize the toolkit. It handles:
        - Loading and validating configurations
        - Instantiating system and analysis objects
        - Running preflight validation checks (if validate=True)

        Args:
            system_config: Path to system configuration YAML file
            analysis_config: Path to analysis configuration YAML file
            hpc_system_config: Optional path to the per-HPC-system configuration
                YAML file (``hpc_system_config.yaml``). When None (default),
                behavior is byte-identical to today — the HPC config consumers
                wire in later phases.
            validate: Whether to run preflight validation (default: True).
                Raises ConfigurationError if validation fails.

        Returns:
            Initialized Toolkit instance ready for workflow execution

        Raises:
            ConfigurationError: If configuration files are invalid or validation fails
            FileNotFoundError: If configuration files don't exist

        Example:
            >>> from hhemt import Toolkit
            >>>
            >>> tk = Toolkit.from_configs(
            ...     system_config="configs/system.yaml",
            ...     analysis_config="configs/analysis.yaml"
            ... )
            >>>
            >>> # Toolkit is ready - system inputs processed, executables compiled
            >>> print(f"Analysis directory: {tk.analysis.analysis_dir}")
            >>> print(f"Total simulations: {tk.analysis.n_simulations}")
        """
        from .analysis import TRITONSWMM_analysis
        from .system import TRITONSWMM_system

        # Load system and analysis
        system = TRITONSWMM_system(Path(system_config))
        analysis = TRITONSWMM_analysis(
            Path(analysis_config),
            system,
            hpc_system_config_yaml=(Path(hpc_system_config) if hpc_system_config is not None else None),
        )
        system._analysis = analysis  # Link back

        # Run preflight validation if requested
        if validate:
            validation_result = system.analysis.validate()
            validation_result.raise_if_invalid()

        return cls(system)

    @classmethod
    def synthetic_experiment(
        cls,
        config: str | Path,
        *,
        hpc_system_config: str | Path | None = None,
        dest_dir: str | Path | None = None,
        dry_run: bool = False,
    ) -> dict:
        """Load-smoke facade for the synthetic compute-config experiment (PIP-2 Phase 1).

        Loads and validates the ``synthetic_experiment_config`` (firing the
        coupling-invariant AND partition-cap validators — the latter loads the
        per-cluster ``hpc_system_config``), builds the partition-as-axis experiment
        matrix, and — unless ``dry_run`` — writes the matrix CSV and generates the
        synthetic model under ``dest_dir``.

        Args:
            config: Path to a ``synthetic_experiment_config`` YAML.
            hpc_system_config: Optional path to the per-cluster ``hpc_system_config``
                YAML; when given it overrides the config's ``hpc_system_config_yaml``.
            dest_dir: Output dir for the matrix CSV + generated model (non-dry-run).
                Defaults to ``{config parent}/synth_experiment_out``.
            dry_run: When True, validate the config + build the matrix in memory and
                return WITHOUT writing files or generating the model (the DoD
                "load-smoke").

        Returns:
            ``{"config": synthetic_experiment_config, "n_matrix_rows": int,
               "matrix_csv": Path | None, "model_dir": Path | None}``.

        Note:
            This Phase-1 scaffold does NOT compose and run a full analysis ensemble;
            that composition currently lives in
            ``scripts/experiments/synth_compute_config.py`` and is promoted into the
            framework in a later phase (see the deferred follow-up).
        """
        import yaml

        from .config.synthetic_experiment import synthetic_experiment_config
        from .synthetic_experiment import build_experiment_matrix, generate_synthetic_experiment

        config = Path(config)
        cfg_dict = yaml.safe_load(config.read_text(encoding="utf-8")) or {}
        if hpc_system_config is not None:
            cfg_dict["hpc_system_config_yaml"] = str(hpc_system_config)
        cfg = synthetic_experiment_config.model_validate(cfg_dict)  # fires both validators

        matrix = build_experiment_matrix(cfg)
        result: dict = {
            "config": cfg,
            "n_matrix_rows": len(matrix),
            "matrix_csv": None,
            "model_dir": None,
        }
        if not dry_run:
            dest = Path(dest_dir) if dest_dir is not None else config.parent / "synth_experiment_out"
            dest.mkdir(parents=True, exist_ok=True)
            matrix_csv = dest / "experiment_matrix.csv"
            matrix.to_csv(matrix_csv, index=False)
            model_dir = dest / "model"
            generate_synthetic_experiment(cfg, model_dir)
            result["matrix_csv"] = matrix_csv
            result["model_dir"] = model_dir
        return result

    def run(
        self,
        mode: Literal["fresh", "resume", "overwrite"] = "resume",
        phases: list[str] | None = None,
        events: list[int] | None = None,
        dry_run: bool = False,
        verbose: bool = True,
        report_config: Path | None = None,
        override_force_rerun=None,
    ) -> WorkflowResult:
        """Run TRITON-SWMM workflow.

        This is the main entry point for workflow execution. It handles:
        - System input processing (DEM, Manning's coefficients)
        - TRITON/SWMM compilation
        - Scenario preparation (SWMM model generation)
        - Simulation execution (TRITON-SWMM runs)
        - Output processing (timeseries extraction, compression)
        - Consolidation (analysis-level aggregation)

        Args:
            mode: Execution mode controlling checkpoint behavior:
                - "fresh": Start from scratch, overwrite all outputs
                - "resume": Resume from last checkpoint (default)
                - "overwrite": Rerun existing scenarios without full reset
            phases: Optional list of phases to run. If None, runs all phases.
                Valid phases: ["setup", "preparation", "simulation",
                              "processing", "consolidation"]
            events: Optional list of event indices to process. If None,
                processes all events in the analysis.
            dry_run: If True, print workflow plan without executing
            verbose: If True, print progress messages during execution

        Returns:
            WorkflowResult with execution details:
                - success (bool): Whether workflow completed successfully
                - mode (str): Mode used for execution
                - execution_time (float): Total runtime in seconds
                - phases_completed (List[str]): Phases that finished
                - events_processed (List[int]): Event indices processed
                - snakefile_path (Path): Path to generated Snakefile
                - job_id (str): SLURM job ID (if HPC execution)
                - message (str): Status message or error description

        Raises:
            ConfigurationError: If configuration is invalid
            WorkflowError: If workflow execution fails

        Example:
            Fresh run (overwrite everything):

            >>> result = tk.run(mode="fresh")
            >>> print(f"Success: {result.success}")
            >>> print(f"Runtime: {result.execution_time:.1f}s")
            >>> print(f"Events: {result.events_processed}")

            Resume interrupted workflow:

            >>> # Check what's done
            >>> status = tk.get_status()
            >>> print(f"Progress: {status.simulations_completed}/{status.total_simulations}")
            >>>
            >>> # Continue from checkpoint
            >>> result = tk.run(mode="resume")

            Run specific events:

            >>> # Process only hurricane Irene and Sandy
            >>> result = tk.run(mode="resume", events=[5, 12])

            Run only simulation phase:

            >>> # Skip setup/preparation, just run simulations
            >>> result = tk.run(
            ...     mode="resume",
            ...     phases=["simulation"]
            ... )

            Dry run (preview without executing):

            >>> result = tk.run(mode="fresh", dry_run=True)
            >>> print(result.message)  # Shows what would be executed

        Notes:
            - Execution mode (local vs SLURM) is auto-detected from configuration
            - Use get_status() to check current progress before resuming
            - For fine-grained control, use analysis.run() directly
        """
        # Auto-detect execution mode
        execution_mode = self._detect_execution_mode()

        # Delegate to analysis.run()
        return self.analysis.run(
            mode=mode,
            phases=phases,
            events=events,
            execution_mode=execution_mode,
            dry_run=dry_run,
            verbose=verbose,
            report_config=report_config,
            override_force_rerun=override_force_rerun,
        )

    def get_status(self) -> WorkflowStatus:
        """Get current workflow status report.

        This method inspects logs and outputs to determine the completion state
        of each workflow phase, providing:
        - Per-phase completion status
        - Number of simulations completed/pending/failed
        - Current workflow phase
        - Recommended mode for next run()
        - Actionable recommendation message

        Returns:
            WorkflowStatus with detailed phase information and recommendations

        Example:
            Check status and decide next action:

            >>> status = tk.get_status()
            >>> print(status)
            Workflow Status Report
            ════════════════════════════════════════
            Analysis: norfolk_coastal_flooding
            Directory: /path/to/analysis

            Phase Status:
            ✓ Setup (complete)
            ✓ Scenario Preparation (complete)
            ⚠ Simulation (in progress: 12/24 complete)
            ✗ Output Processing (not started)
            ✗ Consolidation (not started)

            Progress: 12/24 simulations complete (0 failed)
            Current Phase: simulation

            Recommendation:
            Use mode='resume' to continue simulation execution.
            12 simulations have completed successfully.
            >>>
            >>> # Follow the recommendation
            >>> if not status.simulation.complete:
            ...     result = tk.run(mode=status.recommended_mode)

            Inspect phase details:

            >>> status = tk.get_status()
            >>> print(f"Setup complete: {status.setup.complete}")
            >>> print(f"Simulations done: {status.simulations_completed}")
            >>> print(f"Simulations failed: {status.simulations_failed}")
            >>> print(f"Recommended mode: {status.recommended_mode}")

            Check if workflow is fully complete:

            >>> status = tk.get_status()
            >>> if all([
            ...     status.setup.complete,
            ...     status.preparation.complete,
            ...     status.simulation.complete,
            ...     status.processing.complete,
            ...     status.consolidation.complete,
            ... ]):
            ...     print("✓ Workflow fully complete!")

        Notes:
            - Status is determined by inspecting actual outputs, not cached state
            - Use recommended_mode for next run() to follow best practices
            - Check simulations_failed to detect partial failures
        """
        return self.analysis.get_workflow_status()

    def _detect_execution_mode(self) -> Literal["auto", "local", "slurm"]:
        """Detect appropriate execution mode from configuration and environment.

        Returns:
            "slurm" if in SLURM context or configured for SLURM,
            "local" otherwise
        """
        if self.analysis.in_slurm or self.analysis.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks":
            return "slurm"
        return "local"

    @property
    def analysis_dir(self) -> Path:
        """Get analysis directory path.

        Returns:
            Path to analysis output directory

        Example:
            >>> tk = Toolkit.from_configs(system_cfg, analysis_cfg)
            >>> print(f"Outputs at: {tk.analysis_dir}")
            Outputs at: /path/to/norfolk_coastal_flooding_2024-01-15_143022
        """
        return self.analysis.analysis_paths.analysis_dir

    @property
    def n_simulations(self) -> int:
        """Get total number of simulations in analysis.

        Returns:
            Number of scenarios/events to be processed

        Example:
            >>> tk = Toolkit.from_configs(system_cfg, analysis_cfg)
            >>> print(f"Total simulations: {tk.n_simulations}")
            Total simulations: 24
        """
        return self.analysis.nsims

    def __repr__(self) -> str:
        """Return string representation of Toolkit instance."""
        return (
            f"Toolkit(analysis='{self.analysis.cfg_analysis.analysis_id}', "
            f"n_simulations={self.n_simulations}, "
            f"dir='{self.analysis_dir}')"
        )

analysis_dir property

Get analysis directory path.

Returns: Path to analysis output directory

Example: >>> tk = Toolkit.from_configs(system_cfg, analysis_cfg) >>> print(f"Outputs at: {tk.analysis_dir}") Outputs at: /path/to/norfolk_coastal_flooding_2024-01-15_143022

n_simulations property

Get total number of simulations in analysis.

Returns: Number of scenarios/events to be processed

Example: >>> tk = Toolkit.from_configs(system_cfg, analysis_cfg) >>> print(f"Total simulations: {tk.n_simulations}") Total simulations: 24

from_configs(system_config, analysis_config, hpc_system_config=None, validate=True) classmethod

Create Toolkit instance from configuration files.

This is the recommended way to initialize the toolkit. It handles: - Loading and validating configurations - Instantiating system and analysis objects - Running preflight validation checks (if validate=True)

Args: system_config: Path to system configuration YAML file analysis_config: Path to analysis configuration YAML file hpc_system_config: Optional path to the per-HPC-system configuration YAML file (hpc_system_config.yaml). When None (default), behavior is byte-identical to today — the HPC config consumers wire in later phases. validate: Whether to run preflight validation (default: True). Raises ConfigurationError if validation fails.

Returns: Initialized Toolkit instance ready for workflow execution

Raises: ConfigurationError: If configuration files are invalid or validation fails FileNotFoundError: If configuration files don't exist

Example: >>> from hhemt import Toolkit >>> >>> tk = Toolkit.from_configs( ... system_config="configs/system.yaml", ... analysis_config="configs/analysis.yaml" ... ) >>> >>> # Toolkit is ready - system inputs processed, executables compiled >>> print(f"Analysis directory: {tk.analysis.analysis_dir}") >>> print(f"Total simulations: {tk.analysis.n_simulations}")

Source code in src/hhemt/toolkit.py

@classmethod
def from_configs(
    cls,
    system_config: str | Path,
    analysis_config: str | Path,
    hpc_system_config: str | Path | None = None,
    validate: bool = True,
) -> "Toolkit":
    """Create Toolkit instance from configuration files.

    This is the recommended way to initialize the toolkit. It handles:
    - Loading and validating configurations
    - Instantiating system and analysis objects
    - Running preflight validation checks (if validate=True)

    Args:
        system_config: Path to system configuration YAML file
        analysis_config: Path to analysis configuration YAML file
        hpc_system_config: Optional path to the per-HPC-system configuration
            YAML file (``hpc_system_config.yaml``). When None (default),
            behavior is byte-identical to today — the HPC config consumers
            wire in later phases.
        validate: Whether to run preflight validation (default: True).
            Raises ConfigurationError if validation fails.

    Returns:
        Initialized Toolkit instance ready for workflow execution

    Raises:
        ConfigurationError: If configuration files are invalid or validation fails
        FileNotFoundError: If configuration files don't exist

    Example:
        >>> from hhemt import Toolkit
        >>>
        >>> tk = Toolkit.from_configs(
        ...     system_config="configs/system.yaml",
        ...     analysis_config="configs/analysis.yaml"
        ... )
        >>>
        >>> # Toolkit is ready - system inputs processed, executables compiled
        >>> print(f"Analysis directory: {tk.analysis.analysis_dir}")
        >>> print(f"Total simulations: {tk.analysis.n_simulations}")
    """
    from .analysis import TRITONSWMM_analysis
    from .system import TRITONSWMM_system

    # Load system and analysis
    system = TRITONSWMM_system(Path(system_config))
    analysis = TRITONSWMM_analysis(
        Path(analysis_config),
        system,
        hpc_system_config_yaml=(Path(hpc_system_config) if hpc_system_config is not None else None),
    )
    system._analysis = analysis  # Link back

    # Run preflight validation if requested
    if validate:
        validation_result = system.analysis.validate()
        validation_result.raise_if_invalid()

    return cls(system)

get_status()

Get current workflow status report.

This method inspects logs and outputs to determine the completion state of each workflow phase, providing: - Per-phase completion status - Number of simulations completed/pending/failed - Current workflow phase - Recommended mode for next run() - Actionable recommendation message

Returns: WorkflowStatus with detailed phase information and recommendations

Example: Check status and decide next action:

>>> status = tk.get_status()
>>> print(status)
Workflow Status Report
════════════════════════════════════════
Analysis: norfolk_coastal_flooding
Directory: /path/to/analysis

Phase Status:
✓ Setup (complete)
✓ Scenario Preparation (complete)
⚠ Simulation (in progress: 12/24 complete)
✗ Output Processing (not started)
✗ Consolidation (not started)

Progress: 12/24 simulations complete (0 failed)
Current Phase: simulation

Recommendation:
Use mode='resume' to continue simulation execution.
12 simulations have completed successfully.
>>>
>>> # Follow the recommendation
>>> if not status.simulation.complete:
...     result = tk.run(mode=status.recommended_mode)

Inspect phase details:

>>> status = tk.get_status()
>>> print(f"Setup complete: {status.setup.complete}")
>>> print(f"Simulations done: {status.simulations_completed}")
>>> print(f"Simulations failed: {status.simulations_failed}")
>>> print(f"Recommended mode: {status.recommended_mode}")

Check if workflow is fully complete:

>>> status = tk.get_status()
>>> if all([
...     status.setup.complete,
...     status.preparation.complete,
...     status.simulation.complete,
...     status.processing.complete,
...     status.consolidation.complete,
... ]):
...     print("✓ Workflow fully complete!")

Notes: - Status is determined by inspecting actual outputs, not cached state - Use recommended_mode for next run() to follow best practices - Check simulations_failed to detect partial failures

Source code in src/hhemt/toolkit.py

def get_status(self) -> WorkflowStatus:
    """Get current workflow status report.

    This method inspects logs and outputs to determine the completion state
    of each workflow phase, providing:
    - Per-phase completion status
    - Number of simulations completed/pending/failed
    - Current workflow phase
    - Recommended mode for next run()
    - Actionable recommendation message

    Returns:
        WorkflowStatus with detailed phase information and recommendations

    Example:
        Check status and decide next action:

        >>> status = tk.get_status()
        >>> print(status)
        Workflow Status Report
        ════════════════════════════════════════
        Analysis: norfolk_coastal_flooding
        Directory: /path/to/analysis

        Phase Status:
        ✓ Setup (complete)
        ✓ Scenario Preparation (complete)
        ⚠ Simulation (in progress: 12/24 complete)
        ✗ Output Processing (not started)
        ✗ Consolidation (not started)

        Progress: 12/24 simulations complete (0 failed)
        Current Phase: simulation

        Recommendation:
        Use mode='resume' to continue simulation execution.
        12 simulations have completed successfully.
        >>>
        >>> # Follow the recommendation
        >>> if not status.simulation.complete:
        ...     result = tk.run(mode=status.recommended_mode)

        Inspect phase details:

        >>> status = tk.get_status()
        >>> print(f"Setup complete: {status.setup.complete}")
        >>> print(f"Simulations done: {status.simulations_completed}")
        >>> print(f"Simulations failed: {status.simulations_failed}")
        >>> print(f"Recommended mode: {status.recommended_mode}")

        Check if workflow is fully complete:

        >>> status = tk.get_status()
        >>> if all([
        ...     status.setup.complete,
        ...     status.preparation.complete,
        ...     status.simulation.complete,
        ...     status.processing.complete,
        ...     status.consolidation.complete,
        ... ]):
        ...     print("✓ Workflow fully complete!")

    Notes:
        - Status is determined by inspecting actual outputs, not cached state
        - Use recommended_mode for next run() to follow best practices
        - Check simulations_failed to detect partial failures
    """
    return self.analysis.get_workflow_status()

run(mode='resume', phases=None, events=None, dry_run=False, verbose=True, report_config=None, override_force_rerun=None)

Run TRITON-SWMM workflow.

This is the main entry point for workflow execution. It handles: - System input processing (DEM, Manning's coefficients) - TRITON/SWMM compilation - Scenario preparation (SWMM model generation) - Simulation execution (TRITON-SWMM runs) - Output processing (timeseries extraction, compression) - Consolidation (analysis-level aggregation)

Args: mode: Execution mode controlling checkpoint behavior: - "fresh": Start from scratch, overwrite all outputs - "resume": Resume from last checkpoint (default) - "overwrite": Rerun existing scenarios without full reset phases: Optional list of phases to run. If None, runs all phases. Valid phases: ["setup", "preparation", "simulation", "processing", "consolidation"] events: Optional list of event indices to process. If None, processes all events in the analysis. dry_run: If True, print workflow plan without executing verbose: If True, print progress messages during execution

Returns: WorkflowResult with execution details: - success (bool): Whether workflow completed successfully - mode (str): Mode used for execution - execution_time (float): Total runtime in seconds - phases_completed (List[str]): Phases that finished - events_processed (List[int]): Event indices processed - snakefile_path (Path): Path to generated Snakefile - job_id (str): SLURM job ID (if HPC execution) - message (str): Status message or error description

Raises: ConfigurationError: If configuration is invalid WorkflowError: If workflow execution fails

Example: Fresh run (overwrite everything):

>>> result = tk.run(mode="fresh")
>>> print(f"Success: {result.success}")
>>> print(f"Runtime: {result.execution_time:.1f}s")
>>> print(f"Events: {result.events_processed}")

Resume interrupted workflow:

>>> # Check what's done
>>> status = tk.get_status()
>>> print(f"Progress: {status.simulations_completed}/{status.total_simulations}")
>>>
>>> # Continue from checkpoint
>>> result = tk.run(mode="resume")

Run specific events:

>>> # Process only hurricane Irene and Sandy
>>> result = tk.run(mode="resume", events=[5, 12])

Run only simulation phase:

>>> # Skip setup/preparation, just run simulations
>>> result = tk.run(
...     mode="resume",
...     phases=["simulation"]
... )

Dry run (preview without executing):

>>> result = tk.run(mode="fresh", dry_run=True)
>>> print(result.message)  # Shows what would be executed

Notes: - Execution mode (local vs SLURM) is auto-detected from configuration - Use get_status() to check current progress before resuming - For fine-grained control, use analysis.run() directly

Source code in src/hhemt/toolkit.py

def run(
    self,
    mode: Literal["fresh", "resume", "overwrite"] = "resume",
    phases: list[str] | None = None,
    events: list[int] | None = None,
    dry_run: bool = False,
    verbose: bool = True,
    report_config: Path | None = None,
    override_force_rerun=None,
) -> WorkflowResult:
    """Run TRITON-SWMM workflow.

    This is the main entry point for workflow execution. It handles:
    - System input processing (DEM, Manning's coefficients)
    - TRITON/SWMM compilation
    - Scenario preparation (SWMM model generation)
    - Simulation execution (TRITON-SWMM runs)
    - Output processing (timeseries extraction, compression)
    - Consolidation (analysis-level aggregation)

    Args:
        mode: Execution mode controlling checkpoint behavior:
            - "fresh": Start from scratch, overwrite all outputs
            - "resume": Resume from last checkpoint (default)
            - "overwrite": Rerun existing scenarios without full reset
        phases: Optional list of phases to run. If None, runs all phases.
            Valid phases: ["setup", "preparation", "simulation",
                          "processing", "consolidation"]
        events: Optional list of event indices to process. If None,
            processes all events in the analysis.
        dry_run: If True, print workflow plan without executing
        verbose: If True, print progress messages during execution

    Returns:
        WorkflowResult with execution details:
            - success (bool): Whether workflow completed successfully
            - mode (str): Mode used for execution
            - execution_time (float): Total runtime in seconds
            - phases_completed (List[str]): Phases that finished
            - events_processed (List[int]): Event indices processed
            - snakefile_path (Path): Path to generated Snakefile
            - job_id (str): SLURM job ID (if HPC execution)
            - message (str): Status message or error description

    Raises:
        ConfigurationError: If configuration is invalid
        WorkflowError: If workflow execution fails

    Example:
        Fresh run (overwrite everything):

        >>> result = tk.run(mode="fresh")
        >>> print(f"Success: {result.success}")
        >>> print(f"Runtime: {result.execution_time:.1f}s")
        >>> print(f"Events: {result.events_processed}")

        Resume interrupted workflow:

        >>> # Check what's done
        >>> status = tk.get_status()
        >>> print(f"Progress: {status.simulations_completed}/{status.total_simulations}")
        >>>
        >>> # Continue from checkpoint
        >>> result = tk.run(mode="resume")

        Run specific events:

        >>> # Process only hurricane Irene and Sandy
        >>> result = tk.run(mode="resume", events=[5, 12])

        Run only simulation phase:

        >>> # Skip setup/preparation, just run simulations
        >>> result = tk.run(
        ...     mode="resume",
        ...     phases=["simulation"]
        ... )

        Dry run (preview without executing):

        >>> result = tk.run(mode="fresh", dry_run=True)
        >>> print(result.message)  # Shows what would be executed

    Notes:
        - Execution mode (local vs SLURM) is auto-detected from configuration
        - Use get_status() to check current progress before resuming
        - For fine-grained control, use analysis.run() directly
    """
    # Auto-detect execution mode
    execution_mode = self._detect_execution_mode()

    # Delegate to analysis.run()
    return self.analysis.run(
        mode=mode,
        phases=phases,
        events=events,
        execution_mode=execution_mode,
        dry_run=dry_run,
        verbose=verbose,
        report_config=report_config,
        override_force_rerun=override_force_rerun,
    )

synthetic_experiment(config, *, hpc_system_config=None, dest_dir=None, dry_run=False) classmethod

Load-smoke facade for the synthetic compute-config experiment (PIP-2 Phase 1).

Loads and validates the synthetic_experiment_config (firing the coupling-invariant AND partition-cap validators — the latter loads the per-cluster hpc_system_config), builds the partition-as-axis experiment matrix, and — unless dry_run — writes the matrix CSV and generates the synthetic model under dest_dir.

Args: config: Path to a synthetic_experiment_config YAML. hpc_system_config: Optional path to the per-cluster hpc_system_config YAML; when given it overrides the config's hpc_system_config_yaml. dest_dir: Output dir for the matrix CSV + generated model (non-dry-run). Defaults to {config parent}/synth_experiment_out. dry_run: When True, validate the config + build the matrix in memory and return WITHOUT writing files or generating the model (the DoD "load-smoke").

Returns: {"config": synthetic_experiment_config, "n_matrix_rows": int, "matrix_csv": Path | None, "model_dir": Path | None}.

Note: This Phase-1 scaffold does NOT compose and run a full analysis ensemble; that composition currently lives in scripts/experiments/synth_compute_config.py and is promoted into the framework in a later phase (see the deferred follow-up).

Source code in src/hhemt/toolkit.py

@classmethod
def synthetic_experiment(
    cls,
    config: str | Path,
    *,
    hpc_system_config: str | Path | None = None,
    dest_dir: str | Path | None = None,
    dry_run: bool = False,
) -> dict:
    """Load-smoke facade for the synthetic compute-config experiment (PIP-2 Phase 1).

    Loads and validates the ``synthetic_experiment_config`` (firing the
    coupling-invariant AND partition-cap validators — the latter loads the
    per-cluster ``hpc_system_config``), builds the partition-as-axis experiment
    matrix, and — unless ``dry_run`` — writes the matrix CSV and generates the
    synthetic model under ``dest_dir``.

    Args:
        config: Path to a ``synthetic_experiment_config`` YAML.
        hpc_system_config: Optional path to the per-cluster ``hpc_system_config``
            YAML; when given it overrides the config's ``hpc_system_config_yaml``.
        dest_dir: Output dir for the matrix CSV + generated model (non-dry-run).
            Defaults to ``{config parent}/synth_experiment_out``.
        dry_run: When True, validate the config + build the matrix in memory and
            return WITHOUT writing files or generating the model (the DoD
            "load-smoke").

    Returns:
        ``{"config": synthetic_experiment_config, "n_matrix_rows": int,
           "matrix_csv": Path | None, "model_dir": Path | None}``.

    Note:
        This Phase-1 scaffold does NOT compose and run a full analysis ensemble;
        that composition currently lives in
        ``scripts/experiments/synth_compute_config.py`` and is promoted into the
        framework in a later phase (see the deferred follow-up).
    """
    import yaml

    from .config.synthetic_experiment import synthetic_experiment_config
    from .synthetic_experiment import build_experiment_matrix, generate_synthetic_experiment

    config = Path(config)
    cfg_dict = yaml.safe_load(config.read_text(encoding="utf-8")) or {}
    if hpc_system_config is not None:
        cfg_dict["hpc_system_config_yaml"] = str(hpc_system_config)
    cfg = synthetic_experiment_config.model_validate(cfg_dict)  # fires both validators

    matrix = build_experiment_matrix(cfg)
    result: dict = {
        "config": cfg,
        "n_matrix_rows": len(matrix),
        "matrix_csv": None,
        "model_dir": None,
    }
    if not dry_run:
        dest = Path(dest_dir) if dest_dir is not None else config.parent / "synth_experiment_out"
        dest.mkdir(parents=True, exist_ok=True)
        matrix_csv = dest / "experiment_matrix.csv"
        matrix.to_csv(matrix_csv, index=False)
        model_dir = dest / "model"
        generate_synthetic_experiment(cfg, model_dir)
        result["matrix_csv"] = matrix_csv
        result["model_dir"] = model_dir
    return result

WorkflowError

Bases: TRITONSWMMError

Snakemake workflow failure.

Raised when Snakemake workflow execution fails during any phase (setup, preparation, execution, processing, consolidation).

Attributes: phase: Which workflow phase failed return_code: Process return code from Snakemake stderr: Optional stderr output from Snakemake

Source code in src/hhemt/exceptions.py

class WorkflowError(TRITONSWMMError):
    """Snakemake workflow failure.

    Raised when Snakemake workflow execution fails during any phase
    (setup, preparation, execution, processing, consolidation).

    Attributes:
        phase: Which workflow phase failed
        return_code: Process return code from Snakemake
        stderr: Optional stderr output from Snakemake
    """

    def __init__(self, phase: str, return_code: int, stderr: str = ""):
        self.phase = phase
        self.return_code = return_code
        self.stderr = stderr

        lines = [f"Workflow failed during {phase} phase", f"  Return code: {return_code}"]
        if stderr.strip():
            lines.append(f"  Error output:\n{self._indent(stderr)}")

        super().__init__("\n".join(lines))

    @staticmethod
    def _indent(text: str, prefix: str = "    ") -> str:
        """Indent multi-line text for error message formatting."""
        return "\n".join(prefix + line for line in text.split("\n"))

WorkflowPlanningError

Bases: TRITONSWMMError

Workflow planning/build failure (exit code 3).

Raised when Snakemake workflow generation or DAG planning fails, typically due to invalid target specifications or missing dependencies.

Attributes: phase: The planning phase that failed

Source code in src/hhemt/exceptions.py

class WorkflowPlanningError(TRITONSWMMError):
    """Workflow planning/build failure (exit code 3).

    Raised when Snakemake workflow generation or DAG planning fails,
    typically due to invalid target specifications or missing dependencies.

    Attributes:
        phase: The planning phase that failed
    """

    def __init__(self, phase: str, reason: str):
        self.phase = phase

        super().__init__(f"Workflow planning failed during {phase}\n" f"  Reason: {reason}")

TRITONSWMM_analysis

Source code in src/hhemt/analysis.py

class TRITONSWMM_analysis:
    def __init__(
        self,
        analysis_config_yaml: Path,
        system: "TRITONSWMM_system",
        skip_log_update: bool = False,
        verbose: bool = True,
        is_main_orchestrator: bool = True,
        hpc_system_config_yaml: Path | None = None,
    ) -> None:
        """
        Initialize a TRITON-SWMM analysis orchestrator.

        This class manages the complete lifecycle of a TRITON-SWMM analysis including
        scenario preparation, simulation execution, output processing, and result
        consolidation. It supports multiple execution strategies (serial, local
        concurrent, SLURM) and workflow management via Snakemake.

        Parameters
        ----------
        analysis_config_yaml : Path
            Path to the analysis configuration YAML file
        system : TRITONSWMM_system
            The TRITON-SWMM system object containing system configuration
        skip_log_update : bool, optional
            If True, skip initial log update (default: False)
        verbose : bool, optional
            If True, print a resume status summary when prior ``_status/`` flags
            are detected (default: True)
        hpc_system_config_yaml : Path, optional
            Path to the per-HPC-system configuration YAML
            (``hpc_system_config.yaml``). Loaded ONCE here into
            ``self.cfg_hpc_system`` (None when the path is absent). Consumers
            (the SLURM emitters / preflight) wire in later phases; with the
            argument absent, behavior is byte-identical to today (default: None).
        """
        self._system = system
        self.analysis_config_yaml = analysis_config_yaml
        cfg_analysis = load_analysis_config(analysis_config_yaml)
        self.cfg_analysis = cfg_analysis
        # Settable CaseManifest accessor (C6/ADR-11). Default None keeps the
        # provenance.py graceful-absent stand-in (_resolve_case_manifest); a pure
        # no-op today. Populate wiring (examples.py::from_case_study) is deferred to
        # a follow-up — it flips the embedded core and needs a golden regen.
        # Quotes are required (no `from __future__ import annotations` in this module —
        # the instance-attr annotation is runtime-evaluated; CaseManifest is TYPE_CHECKING-only).
        self._case_manifest: "CaseManifest | None" = None  # noqa: UP037
        # Load the per-HPC-system config ONCE (R2). Store BEFORE any
        # _get_config_args read so the direct attribute read is always safe.
        self.hpc_system_config_yaml = hpc_system_config_yaml
        self.cfg_hpc_system = (
            load_hpc_system_config(hpc_system_config_yaml) if hpc_system_config_yaml is not None else None
        )
        if cfg_analysis.analysis_dir:
            analysis_dir = cfg_analysis.analysis_dir
        else:
            analysis_dir = self._system.cfg_system.system_directory / self.cfg_analysis.analysis_id

        ext = self.cfg_analysis.target_processed_output_type
        cfg_sys = self._system.cfg_system

        analysis_log_directory = analysis_dir / "logs"
        simlog_directory = analysis_log_directory / "sims"

        analysis_paths_kwargs = dict(
            f_log=analysis_dir / "log.json",
            analysis_dir=analysis_dir,
            simulation_directory=analysis_dir / "sims",
            simlog_directory=simlog_directory,
            analysis_log_directory=analysis_log_directory,
        )

        # TRITON-SWMM coupled model consolidated outputs
        if cfg_sys.toggle_tritonswmm_model:
            analysis_paths_kwargs["output_tritonswmm_triton_summary"] = analysis_dir / f"TRITONSWMM_TRITON.{ext}"
            analysis_paths_kwargs["output_tritonswmm_node_summary"] = analysis_dir / f"TRITONSWMM_SWMM_nodes.{ext}"
            analysis_paths_kwargs["output_tritonswmm_link_summary"] = analysis_dir / f"TRITONSWMM_SWMM_links.{ext}"
            analysis_paths_kwargs["output_tritonswmm_performance_summary"] = (
                analysis_dir / f"TRITONSWMM_performance.{ext}"
            )

        # TRITON-only consolidated outputs
        if cfg_sys.toggle_triton_model:
            analysis_paths_kwargs["output_triton_only_summary"] = analysis_dir / f"TRITON_only.{ext}"
            analysis_paths_kwargs["output_triton_only_performance_summary"] = (
                analysis_dir / f"TRITON_only_performance.{ext}"
            )

        # SWMM-only consolidated outputs
        if cfg_sys.toggle_swmm_model:
            analysis_paths_kwargs["output_swmm_only_node_summary"] = analysis_dir / f"SWMM_only_nodes.{ext}"
            analysis_paths_kwargs["output_swmm_only_link_summary"] = analysis_dir / f"SWMM_only_links.{ext}"

        # Hierarchical DataTree consolidation (Phase 2)
        analysis_paths_kwargs["analysis_datatree_zarr"] = analysis_dir / "analysis_datatree.zarr"

        # Sensitivity-level DataTree zarr (Phase 3) — aggregates sub-analyses.
        if cfg_analysis.toggle_sensitivity_analysis:
            analysis_paths_kwargs["sensitivity_datatree_zarr"] = analysis_dir / "sensitivity_datatree.zarr"

        self.analysis_paths = AnalysisPaths(**analysis_paths_kwargs)
        # Ensure the per-analysis simulation directory exists at construction time.
        # Previously this was created incidentally during __init__ by the heavy
        # _update_log()'s per-scenario TRITONSWMM_scenario construction (whose
        # __init__ mkdir's sims/<sim_id>/...). _update_log is now a thin refresh
        # (log-write-race-fix), so create the directory explicitly here to
        # preserve the construction-time contract relied on by callers/tests.
        self.analysis_paths.simulation_directory.mkdir(parents=True, exist_ok=True)

        self.df_sims = pd.read_csv(self.cfg_analysis.weather_events_to_simulate).loc[
            :, self.cfg_analysis.weather_event_indices
        ]
        self._sim_run_objects: dict = {}
        self._sim_run_processing_objects: dict = {}
        self.backend = "gpu" if self.cfg_analysis.run_mode == "gpu" else "cpu"

        # self._system.compilation_successful = False
        self.in_slurm = "SLURM_JOB_ID" in os.environ.copy() or (
            cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks"
        )
        self._execution_strategy = self._select_execution_strategy()
        # Phase-4 (4d): python_path retired off analysis_config (no hpc_system_config
        # home — re-addable if a cluster needs a bespoke interpreter). Rule shells use
        # "python", resolved by the conda-env activation emitted in the shell prefix —
        # byte-identical to the prior python_path-absent emission.
        self._python_executable = "python"
        self._workflow_builder = SnakemakeWorkflowBuilder(self)
        self.process = TRITONSWMM_analysis_post_processing(self)
        self.plot = TRITONSWMM_analysis_plotting(self)
        self.nsims = len(self.df_sims)

        if self.cfg_analysis.toggle_sensitivity_analysis is True:
            self.sensitivity = TRITONSWMM_sensitivity_analysis(
                self, is_main_orchestrator=is_main_orchestrator, skip_log_update=skip_log_update
            )
            self.nsims *= len(self.sensitivity.df_setup)
        # Always LOAD the log from disk (read-only safe; _refresh_log creates a
        # default when the log file is absent). Only the WRITE-BACK side is gated
        # on skip_log_update, so a read-only consumer (renderer) gets self.log
        # populated WITHOUT mutating the shared log.
        self._refresh_log()
        if not skip_log_update:
            # Record available backends at analysis creation time
            self.log.cpu_backend_available.set(self._system.compilation_cpu_successful)
            self.log.gpu_backend_available.set(self._system.compilation_gpu_successful)

            self._update_log()
        self._resource_manager = ResourceManager(self)
        if verbose:
            self._print_resume_status()

    def _print_resume_status(self) -> None:
        """Print a resume status summary if prior _status/ flags are detected.

        Fires at the end of ``__init__()`` when ``verbose=True``. Skips silently
        on first runs (no flags present). For ``1_job_many_srun_tasks`` analyses
        with incomplete sims, also prints a node recommendation.
        """
        status_dir = self.analysis_paths.analysis_dir / "_status"
        if not status_dir.exists() or not any(status_dir.glob("*.flag")):
            return  # first run — no flags yet

        # Determine primary model type for counting c_run_* flags
        cfg_sys = self._system.cfg_system
        if cfg_sys.toggle_tritonswmm_model:
            primary_model_type = "tritonswmm"
        elif cfg_sys.toggle_triton_model:
            primary_model_type = "triton"
        else:
            primary_model_type = "swmm"

        # Count completed sims from the canonical c_run completion-flag set — the
        # same per-(event) flags ``_all_sim_flags_present`` attests and that the
        # Snakemake run rule emits gated on ``model_run_completed()`` / "Simulation
        # ends" (resume-retry-resilience P3 research). Enumerating the canonical
        # ``c_run_{model}_evt-{event_id}_complete.flag`` names — rather than an
        # ad-hoc wildcard glob that could match non-canonical/stale flags — aligns
        # the printed N/M with the planner's authoritative completion signal while
        # staying O(flag-stat): no scenario instantiation. (c_run flags are durable
        # post-completion; ``_invalidate_downstream_flags`` never deletes them.)
        total_sims = self.nsims
        if self.cfg_analysis.toggle_sensitivity_analysis:
            # The canonical per-(sa_id, event_id) enumeration is owned by the
            # sub-analyses; the sa-flag glob is the existing sensitivity count
            # (``c_run_*_sa-*`` flags are likewise post-completion + durable).
            sim_flags = list(status_dir.glob(f"c_run_{primary_model_type}_sa*_complete.flag"))
            n_complete = len(sim_flags)
        else:
            from hhemt.scenario import compute_event_id_slug

            n_complete = sum(
                1
                for i in range(len(self.df_sims))
                if (
                    status_dir
                    / (
                        f"c_run_{primary_model_type}_evt-"
                        f"{compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i))}"
                        "_complete.flag"
                    )
                ).exists()
            )
        n_incomplete = total_sims - n_complete

        analysis_id = self.cfg_analysis.analysis_id
        print(f"[Analysis] Resuming {analysis_id} — {n_complete}/{total_sims} sims complete.", flush=True)

        # #3 — surface the effective per-sim retry budget so the operator sees the
        # cap before a silent post-cap non-completion. This prints the config value;
        # a runtime override (override_hpc_restart_times_simulate) passed to
        # run()/submit_workflow() can raise it for that invocation.
        _attempts = self.cfg_analysis.hpc_restart_times_simulate + 1
        print(
            f"[Analysis] Each sim gets {_attempts} attempt(s) "
            f"(hpc_restart_times_simulate={self.cfg_analysis.hpc_restart_times_simulate}) "
            "before silent non-completion.",
            flush=True,
        )

        if n_incomplete == 0:
            return

        # Node recommendation — only for 1_job_many_srun_tasks
        if self.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks":
            # Compute per-sim node requirement for each incomplete sub-analysis
            failures = self._classify_incomplete_sim_failures()
            if self.cfg_analysis.toggle_sensitivity_analysis:
                incomplete_nodes: list[int] = []
                incomplete_sa_ids = {re.search(r"sa-(.+?)_evt-", k).group(1) for k in failures}
                for sa_id in incomplete_sa_ids:
                    sa = self.sensitivity.sub_analyses[sa_id]
                    n_gpus = sa.cfg_analysis.n_gpus or 0
                    # Phase-4 (4d): per-node GPU topology resolves from the sub-analysis
                    # ensemble partition's PartitionSpec (retired off analysis_config).
                    from hhemt.config.hpc_system import resolve_gpus_per_node

                    gpus_per_node = (
                        resolve_gpus_per_node(sa.cfg_hpc_system, sa.cfg_analysis.hpc_ensemble_partition) or 1
                    )
                    if n_gpus > 0:
                        nodes = math.ceil(n_gpus / gpus_per_node)
                    else:
                        nodes = sa.cfg_analysis.n_nodes or 1
                    incomplete_nodes.append(nodes)
                max_per_sim_nodes = max(incomplete_nodes) if incomplete_nodes else 1
                recommended_nodes = max_per_sim_nodes
            else:
                n_nodes = self.cfg_analysis.n_nodes or 1
                max_per_sim_nodes = n_nodes
                recommended_nodes = n_incomplete * n_nodes

            current_nodes = self.cfg_analysis.hpc_total_nodes
            print(
                f"[Analysis] Node recommendation for re-run:\n"
                f"  Max per-sim nodes (across incomplete sims): {max_per_sim_nodes}\n"
                f"  Recommended override_hpc_total_nodes={recommended_nodes}\n"
                f"  (Current hpc_total_nodes={current_nodes})",
                flush=True,
            )

            if failures:
                if self._is_timeout_only_failure:
                    print("[Analysis] All failures are SLURM time limits — increase --time and re-run.", flush=True)
                else:
                    print(
                        "[Analysis] Some failures are not time limits — see debugging docs for root cause.",
                        flush=True,
                    )

    def _warn_resume_zero_progress(self, pickup_where_leftoff: bool) -> None:
        """Warn before launch when a hotstart-resume risks zero progress per round.

        Fires only when actually resuming (``pickup_where_leftoff``) in an HPC mode
        (``batch_job`` / ``1_job_many_srun_tasks``) with a per-sim walltime set: if
        that walltime is below the first-checkpoint latency, every resume round
        restarts from zero and the run never completes. Checkpoint cadence is
        model-dependent and is NOT a config field, so this is an ADVISORY (no
        numeric floor) rather than a threshold-gated error — see the
        resume-retry-resilience P3 friction design-recommendation (Option A).
        """
        if not pickup_where_leftoff:
            return
        if self.cfg_analysis.multi_sim_run_method not in ("batch_job", "1_job_many_srun_tasks"):
            return
        walltime = self.cfg_analysis.hpc_time_min_per_sim
        if walltime is None:
            return
        print(
            f"[Analysis] WARNING: resuming (pickup_where_leftoff=True) with "
            f"hpc_time_min_per_sim={walltime} min. Each resume round makes ZERO "
            "progress unless the per-sim walltime exceeds the first-checkpoint "
            "latency (the sim never reaches a config_NNNN.cfg checkpoint to resume "
            "from). Checkpoint cadence is model-dependent and not a config field, so "
            "this is advisory — confirm the walltime clears your model's "
            "first-checkpoint time before launching a resume sweep.",
            flush=True,
        )

    def _enumerate_stale_metadata_paths(self) -> list[str]:
        """Return Snakemake-output-path strings whose ``.snakemake/metadata/``
        records are known stale due to past rule-output renames.

        Currently enumerates the four Phase 8 rule-rename orphans:

        - ``plots/system_overview.png``
        - ``plots/per_sim/{event_id}/peak_flood_depth.png`` (one per event_iloc)
        - ``plots/per_sim/{event_id}/conduit_flow.png`` (one per event_iloc)
        - ``plots/sensitivity/benchmarking/{independent_var}_vs_total.svg``
          (one per ``sensitivity.independent_vars`` when sensitivity is enabled
          at the master analysis level)

        The enumeration is deterministic — paths are constructed from
        ``self.df_sims.index`` (event_ilocs) plus the canonical event-id
        slug (``compute_event_id_slug``) and, when sensitivity is enabled,
        ``self.sensitivity.independent_vars``. No filesystem inspection;
        Snakemake's ``cleanup_metadata`` is idempotent on non-existent records.

        Returns paths as strings (relative to ``analysis_dir``).
        """
        from hhemt.scenario import compute_event_id_slug

        orphans: list[str] = ["plots/system_overview.png"]
        for event_iloc in self.df_sims.index:
            ev = self._retrieve_weather_indexer_using_integer_index(event_iloc)
            event_id = compute_event_id_slug(ev)
            orphans.append(f"plots/per_sim/{event_id}/peak_flood_depth.png")
            orphans.append(f"plots/per_sim/{event_id}/conduit_flow.png")
        if self.cfg_analysis.toggle_sensitivity_analysis and not self.cfg_analysis.is_subanalysis:
            for ind_var in self.sensitivity.independent_vars:
                orphans.append(f"plots/sensitivity/benchmarking/{ind_var}_vs_total.svg")
        return orphans

    def _invoke_snakemake_cleanup_metadata(self, orphan_paths: list[str]) -> None:
        """Subprocess-invoke ``snakemake --cleanup-metadata`` against orphan paths.

        Snakemake's ``cleanup_metadata`` is idempotent (no-op without error on
        non-existent records), so passing paths that have no record on disk is
        safe — the cost is one subprocess call per ``analysis.run()`` when the
        gate fires.

        Raises ``WorkflowError`` on non-zero subprocess exit, capturing the
        last 50 lines of combined stdout+stderr in the ``stderr`` field.
        """
        import subprocess

        from hhemt.exceptions import WorkflowError

        cmd = [
            "snakemake",
            "--cleanup-metadata",
            *orphan_paths,
            "--directory",
            str(self.analysis_paths.analysis_dir),
            "--cores",
            "1",
        ]
        result = subprocess.run(cmd, capture_output=True, text=True)
        if result.returncode != 0:
            combined = result.stdout + "\n" + result.stderr
            tail = "\n".join(combined.splitlines()[-50:])
            # Best-effort hygiene: "No Snakefile found" is a benign no-op for
            # analyses whose Snakefile has been removed or never written —
            # there is no metadata to interpret without a Snakefile, but
            # there is also no harm in skipping cleanup in that case.
            if "No Snakefile found" in combined:
                return
            raise WorkflowError(
                phase="cleanup_stale_metadata",
                return_code=result.returncode,
                stderr=(f"snakemake --cleanup-metadata exit {result.returncode}; last 50 lines:\n{tail}"),
            )

    def _prune_settled_markers(self, *, dry_run: bool = False) -> list[Path]:
        """Prune settled _status/_completed and _status/_failed markers.

        A marker is *settled* when its sibling _status/_submitted/{token}.json is
        absent — the runner's try/finally wrote the marker then deleted the
        submitted-sentinel, so the marker is pure accumulation that the reconcile
        will never re-read (the reconcile only reads markers for tokens that HAVE
        a submitted-sentinel; see _classify_via_state_markers). Returns the list of
        settled-marker paths (deleted when dry_run=False).
        """
        status_dir = self.analysis_paths.analysis_dir / "_status"
        submitted_dir = status_dir / "_submitted"
        submitted_tokens = {p.stem for p in submitted_dir.glob("*.json")} if submitted_dir.exists() else set()
        settled: list[Path] = []
        for marker_subdir in ("_completed", "_failed"):
            d = status_dir / marker_subdir
            if not d.exists():
                continue
            for marker in d.glob("*.json"):
                if marker.stem not in submitted_tokens:
                    settled.append(marker)
        settled = sorted(settled)
        if not dry_run:
            for m in settled:
                # EXEMPT-DU: status-flag
                m.unlink(missing_ok=True)
        return settled

    def validate(self) -> ValidationResult:
        """Run preflight validation on system and analysis configurations.

        This method performs comprehensive validation of both system and analysis
        configurations before launching expensive simulation work. It checks:

        - System config: paths, toggle dependencies, model selection
        - Analysis config: weather data, run-mode consistency, HPC settings
        - Data consistency: event alignment, storm tide data, units

        Returns
        -------
        ValidationResult
            Validation result with any errors and warnings. Use result.is_valid
            to check if validation passed, or result.raise_if_invalid() to raise
            ConfigurationError if any errors exist.

        Examples
        --------
        >>> analysis = system.analysis
        >>> result = analysis.validate()
        >>> if not result.is_valid:
        >>>     print(result)  # Show all errors and warnings
        >>>     result.raise_if_invalid()  # Raise ConfigurationError

        >>> # Or validate and raise in one step:
        >>> analysis.validate().raise_if_invalid()

        Notes
        -----
        Validation is NOT automatically called in __init__ to avoid breaking
        existing workflows. Users should explicitly call validate() before
        launching simulations, or CLI/API entry points can call it automatically.
        """
        return preflight_validate(
            cfg_system=self._system.cfg_system,
            cfg_analysis=self.cfg_analysis,
            cfg_hpc_system=self.cfg_hpc_system,
        )

    def _refresh_log(self):
        if self.analysis_paths.f_log.exists():
            self.log = TRITONSWMM_analysis_log.from_json(self.analysis_paths.f_log)
        else:
            self.log = TRITONSWMM_analysis_log(logfile=self.analysis_paths.f_log)

    def _select_execution_strategy(self):
        """
        Select the appropriate execution strategy based on configuration.

        Returns
        -------
        ExecutionStrategy
            The appropriate executor (SerialExecutor, LocalConcurrentExecutor, or SlurmExecutor)
        """
        method = self.cfg_analysis.multi_sim_run_method
        if method == "1_job_many_srun_tasks":
            return SlurmExecutor(self)
        elif method == "local":
            return LocalConcurrentExecutor(self)
        else:
            # Default to serial execution for safety
            return SerialExecutor(self)

    def print_cfg(self, which: Literal["system", "analysis", "both"] = "both"):
        """
        Print configuration settings in tabular format.

        Parameters
        ----------
        which : Literal["system", "analysis", "both"], optional
            Which configuration to print (default: "both")
        """
        if which in ["system", "both"]:
            print("=== System Configuration ===", flush=True)
            self._system.cfg_system.display_tabulate_cfg()
        if which == "both":
            print("\n", flush=True)
        if which in ["analysis", "both"]:
            print("=== analysis Configuration ===", flush=True)
            self.cfg_analysis.display_tabulate_cfg()

    def globus_to_local(self, transfer_yaml: "Path") -> str:
        """Transfer HPC results to local machine via Globus.

        Args:
            transfer_yaml: Path to a transfer spec YAML in configs/transfers/.
                           See configs/transfers/template_transfer.yaml.

        Returns:
            Globus task ID. Pass to ``GlobusTransferManager().wait(task_id)``
            to block until complete, or monitor at app.globus.org.

        Example::

            task_id = analysis.globus_to_local(
                Path("configs/transfers/my_frontier_run.yaml")
            )
        """
        from pathlib import Path as _Path

        from hhemt.config.loaders import load_transfer_config
        from hhemt.globus_transfer import GlobusTransferManager

        spec = load_transfer_config(_Path(transfer_yaml))
        manager = GlobusTransferManager(collection_uuids=[spec.endpoints.source_uuid])
        return manager.transfer(spec)

    def globus_to_hpc(self, transfer_yaml: "Path") -> str:
        """Transfer local inputs to HPC via Globus.

        Args:
            transfer_yaml: Path to a transfer spec YAML in configs/transfers/.

        Returns:
            Globus task ID.
        """
        from pathlib import Path as _Path

        from hhemt.config.loaders import load_transfer_config
        from hhemt.globus_transfer import GlobusTransferManager

        spec = load_transfer_config(_Path(transfer_yaml))
        manager = GlobusTransferManager(collection_uuids=[spec.endpoints.destination_uuid])
        return manager.transfer(spec)

    def transfer_results(
        self,
        config: "PostRunTransferConfig",
    ) -> str:
        """Transfer analysis results to local machine via Globus.

        This is a standalone method — it does not require ``run()`` to have
        been called first.  Use it for the "submit on HPC, poll squeue,
        transfer when done" workflow.

        Args:
            config: User-facing transfer configuration.  See
                :class:`~hhemt.config.globus.PostRunTransferConfig`.

        Returns:
            Globus task ID.

        Raises:
            GlobusTransferError: If the transfer fails or is cancelled
                (only when ``config.wait_for_transfer`` is True).

        Example::

            from hhemt.config.globus import PostRunTransferConfig

            config = PostRunTransferConfig(
                destination_root=r"D:\\Dropbox\\_GradSchool\\repos\\hhemt\\frontier",
                system="frontier",
            )
            task_id = analysis.transfer_results(config)
        """
        from hhemt.config.globus import (
            _get_endpoint_uuids,
            _normalize_wsl_path,
        )
        from hhemt.globus_transfer import GlobusTransferManager

        spec = config.to_transfer_spec(
            analysis_dir=self.analysis_paths.analysis_dir,
            analysis_id=self.cfg_analysis.analysis_id,
        )

        # Handle destination conflict
        dest_path = _normalize_wsl_path(config.destination_root).rstrip("/")
        dest_dir = Path(f"{dest_path}/{self.cfg_analysis.analysis_id}")
        if dest_dir.exists():
            self._handle_destination_conflict(dest_dir, config.conflict_policy)

        # Only pass collection_uuids for endpoints that need data_access consent;
        # pass session_required_domains for domain-restricted endpoints (e.g. OLCF).
        _uuid, _base, needs_data_access, session_domain = _get_endpoint_uuids(config.system)
        consent_uuids = [spec.endpoints.source_uuid] if needs_data_access else []
        session_domains = [session_domain] if session_domain else None
        manager = GlobusTransferManager(
            collection_uuids=consent_uuids,
            session_required_domains=session_domains,
        )
        task_id = manager.transfer(spec, exclude_dirs=config.exclude_patterns)

        if config.wait_for_transfer:
            manager.wait(task_id, timeout_minutes=config.timeout_minutes)

        return task_id

    # Conforms to hhemt.bundle._protocol.BundleableAnalysis
    # via duck typing (Protocol is structural; no registration needed).
    def bundle_report_data(
        self,
        output_path: "Path | None" = None,
    ) -> "Path":
        """Emit a portable render bundle for local renderer iteration.

        Opt-in only — NEVER invoked from analysis.run() or
        submit_workflow(). The bundle is a self-contained tar including
        every source path declared via prov.artist().add_channel(...)
        during the most recent render_report() execution, plus configs
        with relative paths, the Snakefile, and the HPC-baseline
        analysis_report.{html,zip} under bundle_baseline/.

        Args:
            output_path: Optional target path for the bundle tar.
                Defaults to
                {analysis_dir}/render_bundle/{analysis_id}_{git_sha}_v{schema}.tar.

        Returns:
            Path to the emitted bundle tar.

        Raises:
            FileNotFoundError: If render_report() has not been invoked
                on this analysis (no *.manifest.json sidecars exist).
        """
        from hhemt.bundle import emit_bundle

        return emit_bundle(self, output_path)

    def reprex_bundle(self, output_path: "Path | None" = None) -> "Path":
        """Emit a reprex-ready Workflow-Run-Crate bundle and return its extracted
        directory root (ADR-10, D3).

        Peer of ``bundle_report_data()``; opt-in only (never invoked from
        ``run()``/``submit_workflow()``). ``emit_bundle`` already carries the reprex
        runnable-template set + WRC crate (Phase 2), so this facade emits the bundle and
        extracts the zip to a sibling directory so the round-trip consumes a directory
        root directly (``Bundle.from_directory(...).reprex(...)``). The HARD emit-time
        zero-user-info gate is deferred to the emit-hardening follow-up (its
        prerequisite); ``Bundle.reprex()`` runs a consume-side informational scan.

        Returns:
            Path to the extracted reprex-bundle directory.
        """
        from hhemt.bundle import emit_bundle
        from hhemt.bundle._reprex import extract_reprex_bundle

        return extract_reprex_bundle(emit_bundle(self, output_path))

    def publish(
        self,
        target: "Literal['hydroshare', 'zenodo']",
        *,
        override_dataset_license: "Literal['CC0-1.0', 'CC-BY-NC-4.0'] | None" = None,
        software_doi: "str | None" = None,
    ) -> dict:
        """Deposit this analysis to a DOI-minting repository (C6, ADR-11).

        Opt-in only — NEVER invoked from analysis.run() or submit_workflow(), mirroring
        bundle_report_data()/transfer_results(). Requires the analysis to have consolidated
        (ro-crate-metadata.json + analysis_datatree.zarr present); the license is read from
        the emitted crate sidecar. override_dataset_license does NOT re-stamp the archived
        license (baked at consolidation into the immutable crate) — it ASSERTS your expected
        value against the sidecar and raises PublishError on mismatch, directing you to set
        analysis_config.dataset_license and reprocess(start_with='consolidate'). Returns
        {"target","data_doi","software_doi","record_url"}.
        """
        from hhemt.publishing import publish_analysis

        return publish_analysis(
            self,
            target=target,
            override_dataset_license=override_dataset_license,
            software_doi=software_doi,
        )

    def eda(
        self,
        *,
        override_eda_config: "Path | None" = None,
        notebook_filename: "str | None" = None,
    ) -> "EdaReportResult":
        """Run the in-process EDA loop: calc -> plots -> doc (ADR-10).

        A LIGHTER non-Snakemake facade. Resolves the EDA config (override-or-cfg
        per the override_ convention), runs the calc members, renders the EDA
        plots under plots/eda/, and assembles eda_report/eda_report.html. Returns
        an EdaReportResult. Bundle carriage: run this BEFORE bundle_report_data()
        so the EDA plots' declared eda/<plot_id>.zarr sources are harvested into
        the bundle (the plots emit under plots/eda/ and declare the zarr as a
        source); bundling before eda() silently omits EDA content.
        """
        from hhemt.config.eda import eda_config
        from hhemt.config.loaders import yaml_to_model
        from hhemt.eda import (
            EdaReportResult,
            check_cross_sim_identity,
            render_eda_plots,
        )
        from hhemt.eda._html_export import export_eda_html
        from hhemt.eda._local_surface import emit_eda_local_surface
        from hhemt.eda._notebook import emit_eda_notebook

        eda_cfg = (
            yaml_to_model(override_eda_config, eda_config) if override_eda_config is not None else self.cfg_analysis.eda
        )
        root = Path(self.analysis_paths.analysis_dir)
        verdict_result = check_cross_sim_identity(self)
        verdicts = [verdict_result.verdict] if verdict_result.verdict is not None else []
        # F-B Flag 2 (hhemt-specialist review): load_eda_context reads the fixed-name
        # cfg_analysis.yaml / cfg_system.yaml at `root`; the LIVE analysis_dir does NOT
        # carry them by construction (only the bundle staging dir does, via
        # _copy_configs_with_relative_paths). Materialize them here so the loader's
        # fixed-name contract holds for the live root too (without this, the notebook's
        # first executed cell `load_eda_context(root)` raises FileNotFoundError).
        import yaml as _yaml

        (root / "cfg_analysis.yaml").write_text(_yaml.safe_dump(self.cfg_analysis.model_dump(mode="json")))
        (root / "cfg_system.yaml").write_text(_yaml.safe_dump(self._system.cfg_system.model_dump(mode="json")))
        # ADR-12: emit the source-independent eda_local/ package skeleton at root.
        emit_eda_local_surface(root)
        # Non-sensitivity analyses produce no eda/<plot_id>.zarr (the cross-sim check
        # skips), so render_eda_plots would open a non-existent zarr — skip rendering
        # on the figureless branch. The NOTEBOOK is still emitted (ADR-14: primary
        # artifact); its zarr-dependent seed cell is gated at execution.
        figureless = verdict_result.skipped or verdict_result.artifact_path is None
        plot_paths = [] if figureless else render_eda_plots(root, cfg_analysis=self.cfg_analysis, eda_cfg=eda_cfg)
        notebook_path = emit_eda_notebook(
            root,
            cfg_analysis=self.cfg_analysis,
            eda_cfg=eda_cfg,
            is_bundle=False,
            notebook_filename=notebook_filename,
        )
        # ADR-14: HTML is a best-effort nbconvert export of the notebook (the source
        # of truth); a kernel/exec failure degrades to None, never fails the loop.
        report_path = export_eda_html(notebook_path, root=root)
        return EdaReportResult(
            report_path=report_path,
            notebook_path=notebook_path,
            plot_paths=plot_paths,
            verdicts=verdicts,
        )

    def static_plots(
        self,
        *,
        static_config_ids: "list[str] | None" = None,
        execution_mode: "Literal['auto','local','slurm']" = "auto",
        override_static_plot_configs: "list[Path] | None" = None,
        verbose: bool = True,
        dry_run: bool = False,
    ) -> dict:
        """Generate publication static figures, one Snakemake rule per static-plot ID,
        distributed via the existing executor (ADR-8). Adjacent to run()/reprocess()/
        bundle_report_data()/eda().

        ``override_static_plot_configs`` (override-prefixed; default None reads
        ``cfg_analysis.static_plot_configs``) is the resolved config list, threaded
        DOWN to the workflow builder so a passed override is honored (anti facade-
        drift). ``static_config_ids`` optionally restricts the render to the named
        subset. A lazy ``stamp_new_target`` performs the 13->14 layout stamp (the
        ``static_plots/`` output dir is created lazily on first render; V0014 is a
        no-op against persisted state).
        """
        from hhemt.version_migration import LAYOUT_VERSION
        from hhemt.version_migration.state import stamp_new_target

        resolved = (
            override_static_plot_configs
            if override_static_plot_configs is not None
            else self.cfg_analysis.static_plot_configs
        )
        if not resolved:
            from .exceptions import ConfigurationError

            raise ConfigurationError(
                field="static_plot_configs",
                message=(
                    "static_plots() requires >=1 static-plot config; "
                    "cfg_analysis.static_plot_configs is empty and no override was passed."
                ),
                config_path=self.analysis_config_yaml,
            )
        stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)
        return self._workflow_builder.submit_static_plots_workflow(
            resolved_static_plot_configs=resolved,
            static_config_ids=static_config_ids,
            execution_mode=execution_mode,
            dry_run=dry_run,
            verbose=verbose,
        )

    def promote_eda_plot(self, plot_id: str, *, target: str, **kwargs):
        """Promote an EDA plot into a standard static-plot config OR a named reporting set (ADR-11).

        Thin facade over eda/_promote.py. ``target="static_plot"`` emits an ADR-4
        StaticPlotBaseConfig YAML keyed on the plot-ID (kwargs: ``output_path``,
        ``caption``); ``target="reporting_set"`` records the plot-ID's promotion intent
        against an ADR-5 ReportingSet (kwargs: ``set_name``). Promotion is a deliberate
        per-plot user action -- it is NOT folded into eda() and is NOT exposed on Bundle.
        """
        from hhemt.eda._promote import (
            promote_eda_plot_to_static_config,
            register_eda_plot_in_reporting_set,
        )

        if target == "static_plot":
            return promote_eda_plot_to_static_config(plot_id, **kwargs)
        if target == "reporting_set":
            return register_eda_plot_in_reporting_set(plot_id, **kwargs)
        raise ValueError(f"target must be 'static_plot' or 'reporting_set', got {target!r}.")

    @staticmethod
    def _handle_destination_conflict(
        dest_dir: Path,
        policy: str,
    ) -> None:
        """Handle an existing destination directory before transfer.

        Args:
            dest_dir: The local destination directory that already exists.
            policy: One of ``"prompt"``, ``"archive"``, ``"clear"``.

        Raises:
            ConfigurationError: If *policy* is ``"prompt"`` and stdin is
                not a TTY.
        """
        import shutil
        import sys

        from hhemt.exceptions import ConfigurationError

        if policy == "archive":
            import datetime

            suffix = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
            archive_dir = dest_dir.parent / "archived"
            archive_dir.mkdir(parents=True, exist_ok=True)
            archived = archive_dir / f"{dest_dir.name}_{suffix}"
            print(
                f"[Transfer] Archiving existing destination → {archived}",
                flush=True,
            )
            shutil.move(str(dest_dir), str(archived))

        elif policy == "clear":
            print(
                f"[Transfer] Clearing existing destination: {dest_dir}",
                flush=True,
            )
            shutil.rmtree(dest_dir)

        elif policy == "prompt":
            if not sys.stdin.isatty():
                raise ConfigurationError(
                    field="conflict_policy",
                    message=(
                        "conflict_policy='prompt' requires an interactive terminal. "
                        "Use 'archive' or 'clear' for non-interactive contexts."
                    ),
                )
            print(
                f"\n[Transfer] Destination already exists: {dest_dir}",
                flush=True,
            )
            print("  (a) Archive to archived/ subfolder", flush=True)
            print("  (c) Clear and overwrite", flush=True)
            print("  (s) Skip — proceed with sync_level transfer", flush=True)
            choice = input("  Choice [a/c/s]: ").strip().lower()
            if choice == "a":
                TRITONSWMM_analysis._handle_destination_conflict(dest_dir, "archive")
            elif choice == "c":
                TRITONSWMM_analysis._handle_destination_conflict(dest_dir, "clear")
            # "s" or anything else: skip, let Globus handle via sync_level

    def _print_all_yaml_defined_input_files(self):
        print_json_file_tree(self._dict_of_exp_and_sys_config())

    def _dict_of_exp_and_sys_config(self):
        dic_exp = self._system.cfg_system.model_dump()
        dic_sys = self.cfg_analysis.model_dump()
        return dic_exp | dic_sys

    def _dict_of_all_sim_files(self, event_iloc):
        dic_syspaths = self._system.sys_paths.as_dict()
        dic_analysis_paths = self.analysis_paths.as_dict()
        scen = TRITONSWMM_scenario(event_iloc, self)
        dic_sim_paths = scen.scen_paths.as_dict()
        dic_all_paths = dic_syspaths | dic_analysis_paths | dic_sim_paths
        return dic_all_paths

    def _print_all_sim_files(self, event_iloc):
        dic_all_paths = self._dict_of_all_sim_files(event_iloc)
        print_json_file_tree(dic_all_paths)

    def _retrieve_weather_indexer_using_integer_index(
        self,
        event_iloc,
    ):
        row = self.df_sims.loc[event_iloc, self.cfg_analysis.weather_event_indices]
        weather_event_indexers = row.to_dict()
        return weather_event_indexers

    @property
    def _scenarios_not_created(self):
        """
        Get list of scenarios that have not been created successfully.

        Returns
        -------
        list of str
            Paths to scenario directories where creation is incomplete
        """
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.scenarios_not_created
        scens_not_created = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            scen.log.refresh()
            if scen.log.scenario_creation_complete.get() is not True:
                scens_not_created.append(str(scen.log.logfile.parent))
        return scens_not_created

    @property
    def _scenarios_not_run(self):
        """
        Get list of scenarios that have not been run successfully.

        Returns
        -------
        list of str
            Paths to scenario directories where simulation is incomplete
        """
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.scenarios_not_run
        scens_not_run = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            # Check if all enabled models completed for this scenario
            enabled_models = scen.run.model_types_enabled
            all_models_completed = all(scen.model_run_completed(model_type) for model_type in enabled_models)
            if not all_models_completed:
                scens_not_run.append(str(scen.log.logfile.parent))
        return scens_not_run

    def _classify_incomplete_sim_failures(self) -> dict[str, str]:
        """Scan model logs for all incomplete simulations and classify each failure.

        Reads the analysis-level model log for each incomplete simulation and
        searches for known SLURM failure markers. Works for both
        ``"1_job_many_srun_tasks"`` and ``"batch_job"`` execution methods —
        the SLURM cancellation marker appears in the model log in both cases.

        Returns
        -------
        dict[str, str]
            Maps scenario identifier (e.g. ``"sa1_0"``) to failure class:

            - ``"timeout"`` — log contains ``DUE TO TIME LIMIT``
            - ``"unclassified"`` — log exists but no known failure marker found
            - ``"no_log"`` — model log file does not exist
        """
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.classify_incomplete_sim_failures()

        results: dict[str, str] = {}
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            enabled_models = scen.run.model_types_enabled
            for model_type in enabled_models:
                if not scen.model_run_completed(model_type):
                    key = f"{event_iloc}"
                    results[key] = scen.run._classify_model_log_failure(model_type)
        return results

    @property
    def _is_timeout_only_failure(self) -> bool:
        """True iff all incomplete simulations have timeout-classified failures.

        Returns False if there are no incomplete sims (all done), or if any
        incomplete sim has an unclassified or no_log failure.
        """
        failures = self._classify_incomplete_sim_failures()
        if not failures:
            return False
        return all(v == "timeout" for v in failures.values())

    @property
    def _all_scenarios_created(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_scenarios_created
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            scen.log.refresh()
            if not bool(scen.log.scenario_creation_complete.get()):
                return False
        return True

    @property
    def _all_sims_run(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_sims_run
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            if not all(scen.model_run_completed(m) for m in scen.run.model_types_enabled):
                return False
        return True

    @property
    def _all_TRITONSWMM_performance_timeseries_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_TRITONSWMM_performance_timeseries_processed
        return len(self._TRITONSWMM_performance_time_series_not_processed) == 0

    @property
    def _TRITONSWMM_performance_time_series_not_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.TRITONSWMM_performance_time_series_not_processed
        scens_not_processed = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            # Check model-specific logs (race-condition free!)
            perf_ok = True
            if self._system.cfg_system.toggle_tritonswmm_model:
                log = scen.get_log("tritonswmm")
                perf_ok = perf_ok and bool(
                    log.performance_timeseries_written and log.performance_timeseries_written.get()
                )
            if self._system.cfg_system.toggle_triton_model:
                log = scen.get_log("triton")
                perf_ok = perf_ok and bool(
                    log.performance_timeseries_written and log.performance_timeseries_written.get()
                )
            if not perf_ok:
                scens_not_processed.append(str(scen.scen_paths.sim_folder))
        return scens_not_processed

    @property
    def _all_SWMM_timeseries_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_SWMM_timeseries_processed
        # Uses model-specific logs - race-condition free!
        return len(self._SWMM_time_series_not_processed) == 0

    @property
    def _TRITON_time_series_not_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.TRITON_time_series_not_processed
        scens_not_processed = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            # Check model-specific logs (race-condition free!)
            triton_ok = True
            if self._system.cfg_system.toggle_tritonswmm_model:
                log = scen.get_log("tritonswmm")
                triton_ok = triton_ok and (log.TRITON_timeseries_written and bool(log.TRITON_timeseries_written.get()))
            if self._system.cfg_system.toggle_triton_model:
                log = scen.get_log("triton")
                triton_ok = triton_ok and (log.TRITON_timeseries_written and bool(log.TRITON_timeseries_written.get()))
            if not triton_ok:
                scens_not_processed.append(str(scen.scen_paths.sim_folder))
        return scens_not_processed

    @property
    def _SWMM_time_series_not_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.SWMM_time_series_not_processed
        scens_not_processed = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            # Check model-specific logs (race-condition free!)
            swmm_ok = True
            if self._system.cfg_system.toggle_tritonswmm_model:
                log = scen.get_log("tritonswmm")
                node_ok = log.SWMM_node_timeseries_written and bool(log.SWMM_node_timeseries_written.get())
                link_ok = log.SWMM_link_timeseries_written and bool(log.SWMM_link_timeseries_written.get())
                swmm_ok = swmm_ok and (node_ok and link_ok)
            if self._system.cfg_system.toggle_swmm_model:
                log = scen.get_log("swmm")
                node_ok = log.SWMM_node_timeseries_written and bool(log.SWMM_node_timeseries_written.get())
                link_ok = log.SWMM_link_timeseries_written and bool(log.SWMM_link_timeseries_written.get())
                swmm_ok = swmm_ok and (node_ok and link_ok)
            if not swmm_ok:
                scens_not_processed.append(str(scen.scen_paths.sim_folder))
        return scens_not_processed

    @property
    def _all_TRITON_timeseries_processed(self):
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_TRITON_timeseries_processed
        # Uses model-specific logs - race-condition free!
        return len(self._TRITON_time_series_not_processed) == 0

    @property
    def _all_raw_TRITON_outputs_cleared(self) -> bool:
        """Computed on read from primitive per-model raw_TRITON_outputs_cleared flags."""
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_raw_TRITON_outputs_cleared
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            for model_type in scen.run.model_types_enabled:
                if model_type in ("triton", "tritonswmm"):
                    ml = scen.get_log(model_type)
                    if not bool(ml.raw_TRITON_outputs_cleared and ml.raw_TRITON_outputs_cleared.get()):
                        return False
        return True

    @property
    def _all_raw_SWMM_outputs_cleared(self) -> bool:
        """Computed on read from primitive per-model raw_SWMM_outputs_cleared flags."""
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.all_raw_SWMM_outputs_cleared
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            for model_type in scen.run.model_types_enabled:
                if model_type in ("swmm", "tritonswmm"):
                    ml = scen.get_log(model_type)
                    if not bool(ml.raw_SWMM_outputs_cleared and ml.raw_SWMM_outputs_cleared.get()):
                        return False
        return True

    def _update_log(self):
        """Reload this analysis's log from disk.

        Historically this recomputed and PERSISTED seven `all_*` rollup flags.
        Those rollups are pure derived state (a function of primitive
        per-scenario / per-model flags) and are now computed on read via the
        `all_*` properties above — so this method no longer authors any rollup
        write. Persisting derived state created an observer-recompute-write path
        that clobbered owner-authored primitives (e.g.
        `datatree_consolidation_complete`) under concurrency; removing the
        persistence removes the clobber vector. Retained as a thin refresh
        wrapper for call-site compatibility; primitives are still persisted by
        their own `.set()` calls at their owner sites.
        """
        self._refresh_log()
        return

    def retrieve_prepare_scenario_launchers(
        self,
        overwrite_scenario_if_already_set_up: bool = False,
        rerun_swmm_hydro_if_outputs_exist: bool = False,
        verbose: bool = False,
    ):
        """
        Create subprocess-based launchers for scenario preparation.

        Each launcher runs scenario preparation in an isolated subprocess to avoid
        PySwmm's MultiSimulationError when preparing multiple scenarios concurrently.

        Parameters
        ----------
        overwrite_scenario_if_already_set_up : bool
            If True, overwrite existing scenarios
        rerun_swmm_hydro_if_outputs_exist : bool
            If True, rerun SWMM hydrology model even if outputs exist
        verbose : bool
            If True, print progress messages

        Returns
        -------
        list
            List of launcher functions that execute scenario preparation in subprocesses
        """
        prepare_scenario_launchers = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)

            # Create a subprocess-based launcher
            launcher = scen._create_subprocess_prepare_scenario_launcher(
                overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                verbose=verbose,
            )
            prepare_scenario_launchers.append(launcher)

        return prepare_scenario_launchers

    def retrieve_scenario_timeseries_processing_launchers(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        """
        Create subprocess-based launchers for scenario timeseries processing.

        Each launcher runs timeseries processing in an isolated subprocess to avoid
        potential conflicts when processing multiple scenarios' outputs concurrently.

        Parameters
        ----------
        which : Literal["TRITON", "SWMM", "both"]
            Which outputs to process: TRITON, SWMM, or both
        override_clear_raw : ClearRawValue | None
            Runtime override for ``cfg_analysis.clear_raw``. ``None`` (the default)
            reads from the YAML config; a concrete value overrides for this run.
        verbose : bool
            If True, print progress messages
        compression_level : int
            Compression level for output files (0-9)

        Returns
        -------
        list
            List of launcher functions that execute timeseries processing in subprocesses
        """
        scenario_timeseries_processing_launchers = []
        for event_iloc in self.df_sims.index:
            proc = self._retrieve_sim_run_processing_object(event_iloc=event_iloc)

            # Create a subprocess-based launcher
            launcher = proc._create_subprocess_timeseries_processing_launcher(
                which=which,
                override_clear_raw=override_clear_raw,
                verbose=verbose,
                compression_level=compression_level,
            )
            scenario_timeseries_processing_launchers.append(launcher)

        return scenario_timeseries_processing_launchers

    def _calculate_effective_max_parallel(
        self,
        min_memory_per_function_MiB: int | None = 1024,
        max_concurrent: int | None = None,
        verbose: bool = False,
    ) -> int:
        """
        Calculate the effective maximum parallelism based on CPU, GPU, memory, and SLURM constraints.

        This method delegates to ResourceManager for resource allocation calculations.

        Parameters
        ----------
        min_memory_per_function_MiB : int | None
            Minimum memory required per function (MiB).
            If provided, concurrency is reduced to avoid oversubscription.
        max_concurrent : int | None
            CPU-based upper bound on parallelism (e.g., based on cores/threads per task).
            If None, defaults to physical CPU count - 1 (or SLURM allocation if in SLURM).
        verbose : bool
            Print progress messages.

        Returns
        -------
        int
            The effective maximum number of parallel tasks.
        """
        return self._resource_manager.calculate_effective_max_parallel(
            min_memory_per_function_MiB=min_memory_per_function_MiB,
            max_concurrent=max_concurrent,
            verbose=verbose,
        )

    def run_python_functions_concurrently(
        self,
        function_launchers: list[Callable[[], None]],
        min_memory_per_function_MiB: int | None = 1024,
        max_parallel: int | None = None,
        verbose: bool = True,
    ) -> list[int]:
        """
        Run Python functions concurrently, limiting parallelism by CPU and memory.

        Parameters
        ----------
        function_launchers : List[Callable[[], None]]
            Functions to execute concurrently.
        max_parallel : int | None
            Upper bound on parallelism (defaults to CPU count).
        min_memory_per_function_MiB : int | None
            Minimum memory required per function (MiB).
            If provided, concurrency is reduced to avoid oversubscription.
        verbose : bool
            Print progress messages.

        Returns
        -------
        List[int]
            Indices of functions that completed successfully.
        """

        effective_max_parallel = self._calculate_effective_max_parallel(
            min_memory_per_function_MiB=min_memory_per_function_MiB,
            max_concurrent=max_parallel,
            verbose=verbose,
        )

        if verbose:
            print(
                f"Running {len(function_launchers)} functions (max parallel = {effective_max_parallel})",
                flush=True,
            )

        results: list[int] = []
        batch_start = time.time()  # Reference point for all tasks

        def wrapper(idx: int, launcher: Callable[[], None]):
            task_start = time.time()
            launcher()
            task_end = time.time()

            duration = task_end - task_start
            completed_at = task_end - batch_start

            if verbose:
                print(
                    f"Function {idx}: duration={duration:.2f}s, completed_at={completed_at:.2f}s",
                    flush=True,
                )
            return idx

        # ----------------------------
        # Execute
        # ----------------------------
        with ThreadPoolExecutor(max_workers=effective_max_parallel) as executor:
            futures = {executor.submit(wrapper, idx, launcher): idx for idx, launcher in enumerate(function_launchers)}

            for future in as_completed(futures):
                idx = futures[future]
                try:
                    results.append(future.result())
                except Exception as e:
                    if verbose:
                        print(f"Function {idx} failed with error: {e}", flush=True)

        self._update_log()
        return results

    def run_prepare_scenarios_serially(
        self,
        overwrite_scenario_if_already_set_up: bool = False,
        rerun_swmm_hydro_if_outputs_exist: bool = False,
        verbose: bool = False,
    ):
        """
        Prepare all scenarios sequentially.

        Executes scenario preparation for all scenarios in serial order, updating
        logs after each scenario completes.

        Parameters
        ----------
        overwrite_scenario_if_already_set_up : bool, optional
            If True, overwrite existing scenarios (default: False)
        rerun_swmm_hydro_if_outputs_exist : bool, optional
            If True, rerun SWMM hydrology model even if outputs exist (default: False)
        verbose : bool, optional
            If True, print progress messages (default: False)
        """
        prepare_scenario_launchers = self.retrieve_prepare_scenario_launchers(
            overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
            rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
            verbose=verbose,
        )
        for launcher in prepare_scenario_launchers:
            launcher()
            self._update_log()  # update logs
        self._update_log()
        return

    def print_logfile_for_scenario(self, event_iloc):
        scen = TRITONSWMM_scenario(event_iloc, self)
        scen.log.print()

    def _get_enabled_model_types(self) -> list[str]:
        """
        Return enabled model types based on system toggles.

        Returns
        -------
        list[str]
            Enabled model types: "triton", "tritonswmm", and/or "swmm"
        """
        cfg_sys = self._system.cfg_system
        models = []
        if cfg_sys.toggle_triton_model:
            models.append("triton")
        if cfg_sys.toggle_tritonswmm_model:
            models.append("tritonswmm")
        if cfg_sys.toggle_swmm_model:
            models.append("swmm")
        return models

    def _retrieve_snakemake_allocations(
        self,
    ) -> tuple[dict[str, dict[str, int]], str | None]:
        """Retrieve parsed per-model Snakemake allocations.

        Routing is strict and context-aware:
        - regular analysis: parse `run_<model>` rules from this analysis Snakefile
        - sensitivity sub-analysis: parse `simulation_sa*_evt*` rules from the
          parent/master sensitivity Snakefile and select this sub-analysis id

        Raises
        ------
        FileNotFoundError
            If the workflow Snakefile does not exist.
        SnakefileParsingError
            If allocations cannot be parsed from the Snakefile.
        """
        enabled_models = self._get_enabled_model_types()

        if self.cfg_analysis.toggle_sensitivity_analysis:
            snakefile_path = self.analysis_paths.analysis_dir / "Snakefile"
            expected_sa_ids = sorted(self.sensitivity.sub_analyses.keys())
            sa_allocations = parse_sensitivity_analysis_workflow_model_allocations(
                snakefile_path=snakefile_path,
                expected_subanalysis_ids=expected_sa_ids,
                strict=False,
            )
            allocations = {
                model_type: alloc.copy() for model_type in enabled_models for alloc in sa_allocations.values()
            }
            return allocations, None

        snakefile_path = self.analysis_paths.analysis_dir / "Snakefile"
        try:
            allocations = parse_regular_workflow_model_allocations(
                snakefile_path=snakefile_path,
                enabled_model_types=enabled_models,
            )
        except SnakefileParsingError as exc:
            # Regular analysis whose run_* rule was wait-rule-substituted (v2
            # graceful-rerun) or otherwise absent: tolerate rather than crash the
            # consolidate/report cascade. Empty allocations → NaN allocation rows
            # annotated below (R2 parity with the sensitivity branch).
            return {}, str(exc)

        return allocations, None

    def _run_sim(
        self,
        event_iloc: int,
        pickup_where_leftoff,
        process_outputs_after_sim_completion: bool,
        which: Literal["TRITON", "SWMM", "both"],
        compression_level: int,
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose=False,
        model_type: Literal["triton", "tritonswmm", "swmm"] = "tritonswmm",
    ):
        """
        Run a single simulation for the specified scenario.

        Executes the TRITON-SWMM simulation for a specific weather event scenario,
        optionally processing outputs after completion.

        Parameters
        ----------
        event_iloc : int
            Integer index of the scenario in df_sims
        pickup_where_leftoff : bool
            If True, resume simulation from last checkpoint
        process_outputs_after_sim_completion : bool
            If True, process timeseries outputs after simulation completes
        which : Literal["TRITON", "SWMM", "both"]
            Which outputs to process (only used if process_outputs_after_sim_completion=True)
        compression_level : int
            Compression level for output files, 0-9
        override_clear_raw : ClearRawValue | None
            Runtime override for ``cfg_analysis.clear_raw`` (None reads from YAML).
        verbose : bool, optional
            If True, print progress messages (default: False)
        model_type : Literal["triton", "tritonswmm", "swmm"], optional
            Model type to run (default: "tritonswmm")

        Raises
        ------
        ValueError
            If scenario creation is incomplete or TRITONSWMM is not compiled
        """
        scen = TRITONSWMM_scenario(event_iloc, self)

        if not scen.log.scenario_creation_complete.get():
            print("Log file:", flush=True)
            print(scen.log.print())
            raise ValueError("scenario_creation_complete must be 'success'")
        valid_types = ("triton", "tritonswmm", "swmm")
        if model_type not in valid_types:
            raise ValueError(f"model_type must be one of {valid_types}, got {model_type}")

        if model_type == "triton":
            if not self._system.compilation_triton_only_successful:
                print("Log file:", flush=True)
                print(scen.log.print())
                raise ValueError("TRITON-only has not been compiled")
        elif model_type == "tritonswmm":
            if not self._system.compilation_successful:
                print("Log file:", flush=True)
                print(scen.log.print())
                raise ValueError("TRITONSWMM has not been compiled")
        elif model_type == "swmm":
            if not self._system.compilation_swmm_successful:
                print("Log file:", flush=True)
                print(scen.log.print())
                raise ValueError("SWMM has not been compiled")
        run = self._retrieve_sim_runs(event_iloc)
        if verbose:
            print("run instance instantiated", flush=True)

        self.analysis_paths.simlog_directory.mkdir(parents=True, exist_ok=True)
        # Use the subprocess launcher pattern, mirroring process_sim_timeseries
        launcher, finalize_sim = run._create_subprocess_sim_run_launcher(
            pickup_where_leftoff=pickup_where_leftoff,
            verbose=verbose,
            model_type=model_type,
        )
        # Launch the simulation (non-blocking)
        proc, start_time, sim_logfile, lf = launcher()
        # Wait for simulation to complete and update simlog
        finalize_sim(proc, start_time, sim_logfile, lf)

        # self._update_log()  # updates analysis log
        if process_outputs_after_sim_completion and run._scenario.model_run_completed(model_type):
            if model_type == "triton":
                outputs_to_process = "TRITON"
            elif model_type == "swmm":
                outputs_to_process = "SWMM"
            else:
                outputs_to_process = which
            self.process_sim_timeseries(
                event_iloc,
                outputs_to_process,
                override_clear_raw=override_clear_raw,
                verbose=verbose,
                compression_level=compression_level,
            )
        return

    def process_sim_timeseries(
        self,
        event_iloc,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        """
        Process and write timeseries outputs for a single simulation.

        Converts raw TRITON and/or SWMM outputs into processed timeseries files,
        optionally clearing raw outputs after processing.

        Parameters
        ----------
        event_iloc : int
            Integer index of the scenario in df_sims
        which : Literal["TRITON", "SWMM", "both"], optional
            Which outputs to process (default: "both")
        override_clear_raw : ClearRawValue | None, optional
            Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
            reads the YAML; a concrete value overrides for this invocation.
        verbose : bool, optional
            If True, print progress messages (default: False)
        compression_level : int, optional
            Compression level for output files, 0-9 (default: 5)
        """
        proc = self._retrieve_sim_run_processing_object(event_iloc=event_iloc)
        proc.write_timeseries_outputs(
            which=which,
            override_clear_raw=override_clear_raw,
            verbose=verbose,
            compression_level=compression_level,
        )
        proc.write_summary_outputs(
            which=which,
            verbose=verbose,
            compression_level=compression_level,
        )

    def _process_all_sim_timeseries_serially(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        for event_iloc in self.df_sims.index:
            self.process_sim_timeseries(
                event_iloc=event_iloc,
                which=which,
                override_clear_raw=override_clear_raw,
                verbose=verbose,
                compression_level=compression_level,
            )
        self._update_log()
        return

    def _consolidate_analysis_outputs(
        self,
        *,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        self.process.consolidate_to_datatree(
            verbose=verbose,
            compression_level=compression_level,
        )
        return

    def _retrieve_sim_runs(self, event_iloc):
        scen = TRITONSWMM_scenario(event_iloc, self)
        run = scen.run
        self._sim_run_objects[event_iloc] = run
        return run

    def _retrieve_sim_run_processing_object(self, event_iloc):
        run = self._retrieve_sim_runs(event_iloc)
        proc = TRITONSWMM_sim_post_processing(run)
        self._sim_run_processing_objects[event_iloc] = proc
        return proc

    def _create_launchable_sims(
        self,
        pickup_where_leftoff: bool = False,
        verbose: bool = False,
    ):
        """
        Create launcher functions for all simulations.

        Uses the consolidated _create_subprocess_sim_run_launcher pattern
        which handles the complete simulation lifecycle including simlog updates.

        The execution method (local, batch_job, or 1_job_many_srun_tasks) is
        determined by self.cfg_analysis.multi_sim_run_method.

        Parameters
        ----------
        pickup_where_leftoff : bool
            If True, resume simulations from last checkpoint
        verbose : bool
            If True, print progress messages

        Returns
        -------
        list
            List of launcher functions
        """
        launch_and_finalize_functions_tuples = []
        enabled_model_types = self._get_enabled_model_types()
        scenario_locks = {event_iloc: threading.Lock() for event_iloc in self.df_sims.index}

        for event_iloc in self.df_sims.index:
            run = self._retrieve_sim_runs(event_iloc)
            lock = scenario_locks[event_iloc]
            for model_type in enabled_model_types:
                launch_and_finalize_functions_tuple = run._create_subprocess_sim_run_launcher(
                    pickup_where_leftoff=pickup_where_leftoff,
                    verbose=verbose,
                    model_type=model_type,
                )
                if launch_and_finalize_functions_tuple is None:
                    continue
                launcher, finalize_sim = launch_and_finalize_functions_tuple

                def locked_launcher(
                    _launcher=launcher,
                    _lock=lock,
                ):
                    _lock.acquire()
                    try:
                        return _launcher()
                    except Exception:
                        _lock.release()
                        raise

                def locked_finalize(
                    proc,
                    start_time,
                    sim_logfile,
                    lf,
                    _finalize=finalize_sim,
                    _lock=lock,
                ):
                    try:
                        _finalize(proc, start_time, sim_logfile, lf)
                    finally:
                        _lock.release()

                launch_and_finalize_functions_tuples.append((locked_launcher, locked_finalize))

        return launch_and_finalize_functions_tuples

    def run_simulations_concurrently(
        self,
        launch_functions: list[tuple],
        max_concurrent: int | None = None,
        verbose: bool = True,
    ):
        """
        Run simulations concurrently using the configured execution strategy.

        Automatically selects the appropriate executor based on cfg_analysis.multi_sim_run_method:
        - "1_job_many_srun_tasks": Uses SlurmExecutor for HPC execution
        - "local": Uses LocalConcurrentExecutor for parallel local execution
        - Other: Uses SerialExecutor for sequential execution

        Parameters
        ----------
        launch_functions : list[tuple]
            List of tuples (launcher, finalize_sim) from _create_subprocess_sim_run_launcher()
        max_concurrent : Optional[int]
            Maximum number of concurrent simulations
        verbose : bool
            If True, print progress messages

        Returns
        -------
        list
            List of simulation statuses
        """
        return self._execution_strategy.execute_simulations(launch_functions, max_concurrent, verbose)

    def run_sims_in_sequence(
        self,
        pickup_where_leftoff,
        process_outputs_after_sim_completion: bool = False,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        compression_level: int = 5,
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose=False,
    ):
        """
        Arguments passed to run:
            - mode: Mode | Literal["single_core"]
            - pickup_where_leftoff
        Arguments passed to processing process_sim_timeseriess
        (only needed if process_outputs_after_sim_completion=True):
            - which: Literal["TRITON", "SWMM", "both"]
            - override_clear_raw: ClearRawValue | None
            - compression_level: int
        """
        if verbose:
            print("Running all sims in series...", flush=True)
        enabled_model_types = self._get_enabled_model_types()
        for event_iloc in self.df_sims.index:
            for model_type in enabled_model_types:
                if verbose:
                    print(
                        f"Running sim {event_iloc} ({model_type}) and pickup_where_leftoff = {pickup_where_leftoff}",
                        flush=True,
                    )
                self._run_sim(
                    event_iloc=event_iloc,
                    pickup_where_leftoff=pickup_where_leftoff,
                    verbose=verbose,
                    process_outputs_after_sim_completion=process_outputs_after_sim_completion,
                    which=which,
                    override_clear_raw=override_clear_raw,
                    compression_level=compression_level,
                    model_type=model_type,  # type: ignore
                )
        self._update_log()

    def run(
        self,
        from_scratch: bool = False,
        dry_run: bool = False,
        events: list[int] | None = None,
        execution_mode: Literal["auto", "local", "slurm"] = "auto",
        verbose: bool = True,
        wait_for_job_completion: bool | None = None,
        override_clear_raw: ClearRawValue | None = None,
        override_force_rerun: ForceRerunValue | None = None,
        override_hpc_total_nodes: int | None = None,
        override_hpc_restart_times_simulate: int | None = None,
        override_hpc_restart_times_other: int | None = None,
        override_pickup_where_leftoff: bool | None = None,
        transfer_config: "PostRunTransferConfig | None" = None,
        report_config: "Path | None" = None,
        override_brand_theme: "Path | None" = None,
        report_formats: list[Literal["html", "zip"]] | None = None,
        cleanup_orphans: bool = False,
        cleanup_stale_metadata: bool = True,
        prune_settled_markers: bool = True,
        extra_sbatch_args: list[str] | None = None,
        snakemake_diagnostics: SnakemakeDiagnostics | None = None,
    ) -> "WorkflowResult":
        """
        High-level orchestration method for running TRITON-SWMM workflows.

        Parameters
        ----------
        from_scratch : bool
            If True, delete all analysis artifacts and start fresh.
            If False (default), resume from last completed checkpoint.
        dry_run : bool
            If True, validate workflow but don't execute.
        events : list[int] | None
            Subset of event_ilocs to process. If None, processes all events.
        execution_mode : Literal["auto", "local", "slurm"]
            Where to execute: auto-detect (default), force local, or force SLURM.
        verbose : bool
            If True, print progress messages.
        override_clear_raw : ClearRawValue | None
            Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
            reads the YAML; pass ``"none"`` / ``"all"`` / a list of model types
            (e.g. ``["tritonswmm", "swmm"]``) to override for this invocation.
            Per the ``override_`` prefix convention introduced by
            cleanup-rerun-delete-redesign Phase 1.
        wait_for_job_completion : bool | None
            If True, block until the SLURM job finishes. Mainly for tests.
        override_hpc_total_nodes : int | None
            Overrides `hpc_total_nodes` in the SBATCH script without mutating
            the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.
        transfer_config : PostRunTransferConfig | None
            If provided, automatically transfer results to the local machine
            after successful completion (requires ``wait_for_job_completion=True``).
        cleanup_orphans : bool, default False
            When True, deletes orphan sub-analysis artifacts (subanalysis dirs,
            status flags, sensitivity_datatree.zarr groups) detected when an
            ``sa_id`` is removed from the sensitivity CSV/XLSX. Opt-in because
            the blast radius is irrecoverable (subanalysis data deleted).
        cleanup_stale_metadata : bool, default True
            When True (default), subprocess-invokes ``snakemake --cleanup-metadata``
            against orphaned ``.snakemake/metadata/`` records left by past
            rule-output renames (e.g., Phase 8's ``.png``/``.svg`` → ``.html``
            flip). When False, skips the cleanup; users may experience a
            one-shot full plot rebuild on first post-rename invocation per
            Phase 8 Risks. Asymmetric with ``cleanup_orphans`` default (False)
            because metadata-cleanup blast radius is bounded to records, not
            data; safe to auto-apply.
        prune_settled_markers : bool, default True
            When True (default), prunes settled ``_status/_completed`` and
            ``_status/_failed`` markers (a marker is settled when its sibling
            ``_status/_submitted/{token}.json`` is absent — pure accumulation the
            reconcile will never re-read). Opt-out, mirroring
            ``cleanup_stale_metadata``: the blast radius is bounded to inert
            settled markers, so auto-applying is safe and bounds unbounded marker
            growth over long resumable campaigns.
        extra_sbatch_args : list[str] | None
            Optional list of additional SBATCH directive strings (e.g.,
            ``["--qos=debug"]`` to route the job to Frontier's debug queue) to
            append to the generated ``run_workflow_1job.sh`` script. Each list
            element is emitted as one ``#SBATCH <element>`` line, after every
            other source of ``#SBATCH`` directives in the script — both the
            always-emitted directives derived from config fields
            (``--job-name``, ``--partition`` from
            ``cfg_analysis.hpc_ensemble_partition``, ``--account`` from
            ``cfg_analysis.hpc_account``, ``--nodes`` from
            ``cfg_analysis.hpc_total_nodes`` (or the
            ``override_hpc_total_nodes`` runtime kwarg), ``--exclusive``,
            ``--gres`` from ``cfg_analysis.hpc_gpus_per_node`` +
            ``cfg_system.gpu_hardware``, ``--time`` from
            ``cfg_analysis.hpc_total_job_duration_min``, ``--output``,
            ``--error``) AND the directives in the
            ``cfg_analysis.additional_SBATCH_params`` config list.

            **Override behavior**: any flag in ``extra_sbatch_args`` that
            matches a flag emitted earlier in the script — whether that earlier
            directive came from a top-level ``cfg_analysis`` field
            (``hpc_ensemble_partition``, ``hpc_account``, ``hpc_total_nodes``,
            ``hpc_total_job_duration_min``, ``hpc_gpus_per_node``, etc.) or
            from the ``cfg_analysis.additional_SBATCH_params`` list — WILL
            OVERRIDE the config-derived value via SLURM's last-directive-wins
            parser semantics. When such an override is detected, an
            informational ``[extra_sbatch_args] OVERRIDE: ...`` message is
            printed naming the flag, the origin of the original value
            (e.g. ``cfg_analysis.hpc_ensemble_partition``), and the new
            runtime value, so the user can confirm the override took effect
            as intended.

            Only valid for ``multi_sim_run_method="1_job_many_srun_tasks"``;
            raises ``ConfigurationError`` otherwise (a fail-fast guard
            preventing the user from believing they are controlling the
            experiment when the kwarg would silently no-op in another mode).

        Returns
        -------
        WorkflowResult
            Structured result object with success status and execution details.

        Examples
        --------
        Resume (default):

        >>> result = analysis.run()

        Fresh start:

        >>> result = analysis.run(from_scratch=True)

        Dry-run validation:

        >>> result = analysis.run(dry_run=True, verbose=True)

        Auto-transfer after completion:

        >>> from hhemt.config.globus import PostRunTransferConfig
        >>> result = analysis.run(
        ...     wait_for_job_completion=True,
        ...     transfer_config=PostRunTransferConfig(
        ...         destination_root=r"D:\\Dropbox\\results",
        ...         system="frontier",
        ...     ),
        ... )

        See Also
        --------
        submit_workflow : Lower-level workflow submission (15+ parameters)
        transfer_results : Standalone transfer method
        """
        # TODO - if from_scratch = True, user should be prompted for manual input to
        # type something like 'y' 'yes' or 'proceed' if the status of the
        # analysis shows that some steps have been completed. This should be
        # accompanied by a print statement of the current status.

        import time

        from .config.loaders import load_brand_theme, yaml_to_model
        from .config.report import (
            report_config as ReportConfigModel,
        )
        from .config.report import (
            validate_active_reporting_set,
        )
        from .exceptions import ConfigurationError
        from .orchestration import WorkflowResult, translate_mode, translate_phases
        from .report_renderers._reporting_sets import get_reporting_set

        # _test/ deletion offer (R9): if a leftover smoke-test subtree from a
        # prior analysis.test() exists, offer to delete it before the real run.
        # _test/ is excluded from the analysis _du.json (Phase 2 du change), so
        # size it directly via du_sentinels._walk_root_bytes rather than via
        # self.disk_utilization_bytes. Mirrors the Globus-conflict prompt's
        # sys.stdin.isatty() non-TTY guard (A6); adds a 15 s select-based
        # auto-'no' so an unattended run is never blocked. `select`/`sys` and
        # `du_sentinels` are imported at module top.
        test_dir = self.analysis_paths.analysis_dir / "_test"
        if test_dir.exists():
            size_bytes, _walk_errors = du_sentinels._walk_root_bytes(test_dir)
            size_mb = size_bytes / (1024 * 1024)
            if not sys.stdin.isatty():
                print(
                    f"[test] Existing _test/ ({size_mb:.1f} MB) left in place (non-interactive stdin — not prompting).",
                    flush=True,
                )
            else:
                print(
                    f"[test] An existing _test/ smoke-test subtree ({size_mb:.1f} MB) "
                    f"was found at {test_dir}. Delete it? [y/N] (auto-'no' in 15s): ",
                    end="",
                    flush=True,
                )
                ready, _, _ = select.select([sys.stdin], [], [], 15)
                answer = sys.stdin.readline().strip().lower() if ready else "n"
                if answer in ("y", "yes"):
                    fast_rmtree(test_dir, analysis_dir=self.analysis_paths.analysis_dir)
                    print(f"[test] Deleted {test_dir}.", flush=True)
                else:
                    print("[test] Keeping _test/.", flush=True)

        # Pre-run report_config resolution (post-F2 v2 — 2-step, fail-fast).
        # Resolution order:
        #   (a) explicit `report_config=` argument → load and use
        #   (b) self.cfg_analysis.report (guaranteed non-None by R1 — required
        #       Pydantic field; loading a cfg_analysis.yaml without `report:`
        #       raises ValidationError before this code is reached)
        # No DEFAULT_REPORT_CONFIG fallback — the field is required.
        if report_config is not None:
            report_config = Path(report_config)
            try:
                cfg_report = yaml_to_model(report_config, ReportConfigModel)
            except Exception as e:
                raise ConfigurationError(
                    field="report_config",
                    message=f"Failed to load/validate {report_config}: {e}",
                    config_path=report_config,
                ) from e
        else:
            cfg_report = self.cfg_analysis.report

        sa_csv = self.cfg_analysis.sensitivity_analysis if self.cfg_analysis.toggle_sensitivity_analysis else None
        _resolved_set_name = validate_active_reporting_set(
            cfg_report,
            is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
            sensitivity_csv_path=sa_csv,
        )
        self._active_reporting_set_name = _resolved_set_name
        self._active_reporting_set = get_reporting_set(_resolved_set_name)
        self._cfg_report = cfg_report

        # ADR-17: non-blocking invalidating-fix registry emission at run()-entry.
        # Prints a console warning for any registered calculation-invalidating fix
        # that affects this analysis's ADR-15-stamped scopes, reflecting the CURRENT
        # registry even when the persisted validation_report.json is stale (OE-3).
        # NEVER raises (load-time emission is non-blocking by construction,
        # C-NON-BLOCKING-BUGEMIT); the durable report surface is
        # check_invalidating_fixes in validate_analysis. Graceful-absent: an absent
        # registry or an unstamped/unconsolidated tree prints nothing.
        try:
            from .recompute import match_registry_against_stamps

            _fix_matches = match_registry_against_stamps(self)
            if _fix_matches:
                print(
                    f"[hhemt] WARNING: {len(_fix_matches)} registered "
                    "calculation-invalidating fix(es) affect this analysis:",
                    flush=True,
                )
                for _m in _fix_matches:
                    _action = _m.recommended_action.value if _m.recommended_action else "-"
                    print(
                        f"  - {_m.commit_id} ({_m.severity}) -> recommended: "
                        f"{_action} [scope {_m.affected_scope}]",
                        flush=True,
                    )
                print(
                    "  Run `check-invalidating-fixes` for details; this is non-blocking.",
                    flush=True,
                )
        except Exception:
            pass  # load-time emission is strictly non-blocking — never crash run()

        # Pre-run brand-theme resolution (ADR-7 layer 2 — 3-step, fail-fast).
        from .config.brand_theme import DEFAULT_BRAND_THEME
        from .config.brand_theme import brand_theme as BrandThemeModel

        if override_brand_theme is not None:
            override_brand_theme = Path(override_brand_theme)
            try:
                resolved_theme = yaml_to_model(override_brand_theme, BrandThemeModel)
            except Exception as e:
                raise ConfigurationError(
                    field="brand_theme",
                    message=f"Failed to load/validate {override_brand_theme}: {e}",
                    config_path=override_brand_theme,
                ) from e
        elif self.cfg_analysis.brand_theme is not None:
            resolved_theme = load_brand_theme(self.cfg_analysis.brand_theme)
        else:
            resolved_theme = DEFAULT_BRAND_THEME
        self._brand_theme = resolved_theme

        # D-5: source the HTML-table brand-derived defaults from the resolved
        # theme via model_validate overlay (NOT setattr — per the per-row-config-
        # overlay stipulation). Semantic pass/fail + th/body text colors stay frozen.
        _t = self._brand_theme
        _table_overlay = {
            "primary_color": _t.primary_color,
            "cell_border_color": _t.neutral_medium,
            "row_alt_bg_color": _t.neutral_light,
            "row_hover_bg_color": _t.accent_color,
        }
        self._cfg_report = type(self._cfg_report).model_validate(
            {
                **self._cfg_report.model_dump(),
                "errors_and_warnings": {
                    **self._cfg_report.errors_and_warnings.model_dump(),
                    **_table_overlay,
                },
                "scenario_status_appendix": {
                    **self._cfg_report.scenario_status_appendix.model_dump(),
                    **_table_overlay,
                },
            }
        )

        # Pre-run transfer validation — fail fast before submitting the workflow
        if transfer_config is not None:
            transfer_config.to_transfer_spec(
                analysis_dir=self.analysis_paths.analysis_dir,
                analysis_id=self.cfg_analysis.analysis_id,
            )

        start_time = time.time()

        # Event filtering not yet implemented - validate parameter
        if events is not None:
            raise NotImplementedError(
                "Event filtering via events parameter not yet implemented. "
                "For now, all events in analysis will be processed."
            )

        if from_scratch and not dry_run:
            # remove analysis folder. Use the DERIVED analysis_paths.analysis_dir
            # (never None) — NOT the raw cfg_analysis.analysis_dir Optional field,
            # which defaults None and made fast_rmtree(None) crash here. Every
            # other analysis_dir reference in this module already uses the derived
            # path; this site was the lone holdout.
            # EXEMPT-DU: full-analysis-root-wipe
            fast_rmtree(self.analysis_paths.analysis_dir)

        # Orphan detection gate (sensitivity-only; non-sensitivity covered by
        # follow-up plan per D-EVENT-PARITY).
        if not from_scratch and self.cfg_analysis.toggle_sensitivity_analysis and not self.cfg_analysis.is_subanalysis:
            from hhemt.exceptions import ConfigurationError as _CfgErr

            _dirs = self.sensitivity.find_orphan_subanalysis_dirs()
            _flags = self.sensitivity.find_orphan_status_flags()
            _groups = self.sensitivity.find_orphan_datatree_groups()
            _fingerprints = self.sensitivity.find_orphan_input_fingerprints()
            _has_orphans = bool(_dirs or _flags or _groups or _fingerprints)
            if _has_orphans and not cleanup_orphans:
                raise _CfgErr(
                    field="cleanup_orphans",
                    message=(
                        "Detected orphan sub-analysis artifacts on disk that are "
                        "absent from the current sensitivity CSV: "
                        f"{len(_dirs)} subanalysis dir(s), "
                        f"{len(_flags)} _status flag(s), "
                        f"{len(_groups)} datatree group(s). "
                        f"{len(_fingerprints)} input-fingerprint(s). "
                        "Re-invoke analysis.run(cleanup_orphans=True) to delete them, "
                        "or run `triton-swmm cleanup-orphans --apply --force` from the CLI."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )
            if _has_orphans and cleanup_orphans:
                self.sensitivity.cleanup_all_orphans(
                    dry_run=dry_run,
                    force=True,
                    verbose=verbose,
                )

        # Stale-metadata cleanup gate — analysis-level, not sensitivity-specific
        # (per Phase 8.5 of interactive_report_renderers plan). Asymmetric with
        # cleanup_orphans default: cleanup_stale_metadata defaults to True
        # because metadata-cleanup blast radius is bounded to .snakemake/metadata/
        # records — no data is deleted; worst-case auto-apply result is the same
        # one-shot full plot rebuild Phase 8 Risks documents.
        # Precondition for the subprocess invocation: `snakemake
        # --cleanup-metadata` requires BOTH a Snakefile in the working
        # directory (to interpret path arguments) AND a
        # `.snakemake/metadata/` directory (the records to clean). The
        # Snakefile is generated by `submit_workflow()` later in this
        # method, so at this gate site it exists only on resumed analyses
        # (the use case cleanup_stale_metadata targets — fresh analyses
        # have no stale metadata to clean).
        _snakefile = self.analysis_paths.analysis_dir / "Snakefile"
        _metadata_dir = self.analysis_paths.analysis_dir / ".snakemake" / "metadata"
        if cleanup_stale_metadata and not from_scratch and _snakefile.exists() and _metadata_dir.exists():
            orphan_paths = self._enumerate_stale_metadata_paths()
            if orphan_paths:
                if verbose:
                    print(
                        f"[cleanup-stale-metadata] Cleaning {len(orphan_paths)} "
                        f"orphan metadata record(s) from "
                        f"{self.analysis_paths.analysis_dir}/.snakemake/metadata/",
                        flush=True,
                    )
                    for p in orphan_paths:
                        print(f"  orphan: {p}", flush=True)
                self._invoke_snakemake_cleanup_metadata(orphan_paths)

        # Prune settled _status/_completed and _status/_failed markers (opt-out).
        # A settled marker is inert (its _submitted/ sibling is gone, so the
        # reconcile will never re-read it); pruning bounds unbounded marker
        # accumulation over long resumable campaigns. Independent of the reconcile
        # second-pass — settled markers are by definition not in-flight.
        if prune_settled_markers:
            pruned = self._prune_settled_markers()
            if verbose and pruned:
                print(
                    f"[prune-settled-markers] Pruned {len(pruned)} settled "
                    f"marker(s) from {self.analysis_paths.analysis_dir}/_status/",
                    flush=True,
                )

        # Stamp _version.json at LAYOUT_VERSION on first materialization (lazy
        # stamp per version_migration_system master plan PI-1). Idempotent
        # under concurrent writers.
        from hhemt.version_migration import LAYOUT_VERSION
        from hhemt.version_migration.state import stamp_new_target

        stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

        # Translate user-friendly parameters to workflow parameters
        # from_scratch's fast_rmtree(analysis_dir) above is the fresh mechanism for the
        # per-analysis tier; the "fresh" mode params additionally force a re-derive of the
        # SHARED system tier (DEM/Manning's in system_directory/) via overwrite_system_inputs.
        # The other four mode flags are moot post-wipe (no scenarios/checkpoints survive).
        mode_params = translate_mode("fresh" if from_scratch else "resume")
        phase_params = translate_phases(None)  # TODO - hardcoded while troubleshooting

        # Detect system input processing needs

        swmm_used = False
        triton_used = False
        for model_used in self._get_enabled_model_types():
            if "swmm" in model_used.lower():
                swmm_used = True
            if "triton" in model_used.lower():
                triton_used = True
        if swmm_used and triton_used:
            which = "both"
        elif swmm_used and not triton_used:
            which = "SWMM"
        else:
            which = "TRITON"

        # Determine execution mode
        if execution_mode == "auto":
            if (
                self.in_slurm
                or self.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks"
                or self.cfg_analysis.multi_sim_run_method == "batch_job"
            ):
                exec_mode = "slurm"
            else:
                exec_mode = "local"
        else:
            exec_mode = execution_mode

        if wait_for_job_completion is None:
            wait_for_job_completion = exec_mode != "slurm"

        # Build complete parameter dict for submit_workflow
        workflow_params = {
            **mode_params,
            **phase_params,
            "mode": exec_mode,
            "which": which,
            "override_clear_raw": override_clear_raw,
            "override_force_rerun": override_force_rerun,
            "compression_level": 5,
            "wait_for_completion": wait_for_job_completion,
            "dry_run": dry_run,
            "verbose": verbose,
            "override_hpc_total_nodes": override_hpc_total_nodes,
            "override_hpc_restart_times_simulate": override_hpc_restart_times_simulate,
            "override_hpc_restart_times_other": override_hpc_restart_times_other,
            "report_formats": report_formats,
            "extra_sbatch_args": extra_sbatch_args,
            "snakemake_diagnostics": snakemake_diagnostics,
        }
        # override_pickup_where_leftoff decouples resume-on-retry from the mode:
        # translate_mode("fresh") (from_scratch=True) sets pickup_where_leftoff=False,
        # but a resume EXPERIMENT wants a fresh wipe AND within-run hotstart-resume (the
        # wipe is once at run-start, before any checkpoint exists; pickup governs the
        # Snakemake-retry behavior AFTER checkpoints are written). None = use the
        # mode-derived value (no behavior change for existing callers).
        if override_pickup_where_leftoff is not None:
            workflow_params["pickup_where_leftoff"] = override_pickup_where_leftoff

        if verbose:
            print("Submitting workflow with args:")
            print(workflow_params)

        # Call underlying submit_workflow
        result_dict = self.submit_workflow(**workflow_params)

        # Calculate execution time
        elapsed = time.time() - start_time

        # Determine which phases were completed based on parameters
        phases_completed = []
        if workflow_params["process_system_level_inputs"] or workflow_params["compile_TRITON_SWMM"]:
            phases_completed.append("setup")
        if workflow_params["prepare_scenarios"]:
            phases_completed.append("prepare")
        if workflow_params["prepare_scenarios"]:  # Simulate always runs if scenarios prepared
            phases_completed.append("simulate")
        if workflow_params["process_timeseries"]:
            phases_completed.append("process")
        if workflow_params["process_timeseries"]:  # Consolidate happens after processing
            phases_completed.append("consolidate")

        # Get event list (all events in analysis)
        events_processed = list(self.df_sims.index)

        # Post-completion auto-transfer
        if transfer_config is not None and wait_for_job_completion and result_dict.get("success", False):
            if verbose:
                print("[Transfer] Workflow succeeded — initiating Globus transfer...", flush=True)
            self.transfer_results(transfer_config)

        # Build WorkflowResult
        return WorkflowResult(
            success=result_dict.get("success", False),
            mode=result_dict.get("mode", exec_mode),
            execution_time=(elapsed if result_dict.get("success") and exec_mode == "local" else None),
            phases_completed=phases_completed if result_dict.get("success") else [],
            events_processed=events_processed if result_dict.get("success") else [],
            snakefile_path=result_dict.get("snakefile_path"),
            job_id=result_dict.get("job_id"),
            message=result_dict.get("message", ""),
            partial_failures=result_dict.get("partial_failures", []),
        )

    # ---- analysis.test(): end-to-end smoke test on a strict _test/ subset ----

    def test(
        self,
        *,
        n_reporting_timesteps: int = cnst.TEST_N_REPORTING_TSTEPS_PER_SIM,
        reporting_timestep_s: int = cnst.TEST_TRITON_REPORTING_TIMESTEP_S,
        execution_mode: Literal["auto", "local", "slurm"] = "auto",
        verbose: bool = True,
        wait_for_job_completion: bool | None = None,
        dry_run: bool = False,
    ) -> "TestResult":
        """Run a strict, least-demanding subset of THIS analysis end-to-end.

        Builds one minimum-device representative per unique
        (enabled-model-toggles x compilation-backend x partition x compute-config)
        group present in the analysis, materializes each under
        ``{analysis_dir}/_test/``, truncates its inputs to ~``n_reporting_timesteps``
        reporting frames, and runs the full compile->run->process->consolidate->
        report path. A strict subset of the user's defined analysis -- no sweeps,
        no synthetic substitution (PIP O-f requirements 1-7).

        Note:
            This is a user-facing smoke-test ENTRY POINT (ADR-8), not a
            pytest-collected test function. pytest's default collection
            (``python_functions = test*``) collects only module-level
            functions and ``Test``-prefixed-class methods, so this instance
            method on ``TRITONSWMM_analysis`` is never collected as a test.
            Invoke it directly (``analysis.test()`` or via the
            ``test_analysis_test_end_to_end.py`` real-data smoke).
        """
        reps = self._select_test_representatives()
        # Truncation happens INSIDE _build_test_subanalyses: the sliced-weather path
        # + reporting interval are carried THROUGH the model_validate overlay dict so
        # the on-disk _test YAML the runner subprocess reloads
        # (prepare_scenario_runner.py:139) points at the short weather. An in-memory
        # setattr would be discarded at that reload (Gotcha 15). Scenario-prep
        # regenerates the SWMM .inp window from the sliced weather automatically
        # (swmm_utils.py:105-110), so no separate .inp-edit pass is needed.
        test_subs = self._build_test_subanalyses(
            reps,
            n_reporting_timesteps=n_reporting_timesteps,
            reporting_timestep_s=reporting_timestep_s,
        )
        results: list = []
        # reps and test_subs are index-aligned: _build_test_subanalyses appends
        # one sub per rep in `representatives` order.
        for rep, sub in zip(reps, test_subs, strict=True):
            wf_result = sub.run(
                from_scratch=True,
                execution_mode=execution_mode,
                verbose=verbose,
                wait_for_job_completion=wait_for_job_completion,
                dry_run=dry_run,
            )  # full compile->run->process->consolidate->report
            results.append(
                TestSubResult(
                    representative=rep,
                    analysis_id=sub.cfg_analysis.analysis_id,
                    analysis=sub,
                    workflow_result=wf_result,
                    binary_provenance=_capture_binary_provenance(sub),
                )
            )
        return TestResult(
            representatives=reps,
            subanalyses=results,
            root=self.analysis_paths.analysis_dir / "_test",
        )

    def _select_test_representatives(self) -> "list[TestRepresentative]":
        """Group candidate runs by
        (model-toggles, compilation-backend, partition, compute-config) and pick the
        minimum-device row per group (D-AXES Option A: compute-config is a KEY, so
        every unique config is represented; device count is the tiebreak). Branches
        on ``toggle_sensitivity_analysis`` (Gotcha 26): a sensitivity master
        enumerates its already-built ``self.sensitivity.sub_analyses`` (each a
        fully-validated TRITONSWMM_analysis), a plain analysis is its own single
        candidate."""
        sensitivity = getattr(self, "sensitivity", None)
        if self.cfg_analysis.toggle_sensitivity_analysis and sensitivity is not None:
            candidates: list = list(sensitivity.sub_analyses.values())
        else:
            candidates = [self]

        def _group_key(a: "TRITONSWMM_analysis") -> tuple:
            cfg_a = a.cfg_analysis
            partition = cfg_a.hpc_ensemble_partition
            backend = None
            if cfg_a.n_gpus and a.cfg_hpc_system is not None:
                # resolve_gpu_target returns (gpu_hardware, gpu_compilation_backend)
                # (config/hpc_system.py:190-211) -- unpack hardware-first, matching
                # _build_unique_system_targets.
                _hardware, backend = resolve_gpu_target(a.cfg_hpc_system, partition)
            cfg_s = a._system.cfg_system
            model_toggles = (
                cfg_s.toggle_tritonswmm_model,
                cfg_s.toggle_triton_model,
                cfg_s.toggle_swmm_model,
            )
            compute_config = (
                cfg_a.run_mode,
                cfg_a.n_mpi_procs,
                cfg_a.n_omp_threads,
                cfg_a.n_gpus,
                cfg_a.n_nodes,
            )
            return (model_toggles, backend, partition, compute_config)

        def _device_demand(a: "TRITONSWMM_analysis") -> tuple:
            # Ascending (n_nodes, n_gpus, n_mpi_procs, n_omp_threads), then the
            # least-demanding non-keyed axis (DEM resolution). None coalesces to 0
            # so the sort key never compares None against an int.
            cfg_a = a.cfg_analysis
            return (
                cfg_a.n_nodes or 0,
                cfg_a.n_gpus or 0,
                cfg_a.n_mpi_procs or 0,
                cfg_a.n_omp_threads or 0,
                getattr(a._system.cfg_system, "target_dem_resolution", 0.0) or 0.0,
            )

        groups: dict[tuple, list] = {}
        for candidate in candidates:
            groups.setdefault(_group_key(candidate), []).append(candidate)

        reps: list[TestRepresentative] = []
        for key, members in groups.items():
            best = min(members, key=_device_demand)
            reps.append(
                TestRepresentative(
                    key=key,
                    source_analysis=best,
                    partition=best.cfg_analysis.hpc_ensemble_partition,
                )
            )
        return reps

    def _build_test_subanalyses(
        self,
        representatives: "list[TestRepresentative]",
        *,
        n_reporting_timesteps: int,
        reporting_timestep_s: int,
    ) -> "list[TRITONSWMM_analysis]":
        """Materialize one TRITONSWMM_analysis per representative under
        ``{analysis_dir}/_test/``, reusing the ``_create_sub_analyses`` overlay
        recipe (sensitivity_analysis.py:2021-2074): ``model_validate`` (never
        ``model_copy``+``setattr``), atomic YAML write, ``is_subanalysis=True``,
        ``toggle_sensitivity_analysis=False``.

        Truncation (SE F-B Flag 1; the former ``_truncate_test_inputs`` is dissolved
        here): the real weather is sliced to its first ``n_reporting_timesteps``
        frames (+1 endpoint) along the configured time dimension, and the SHORT
        weather path + ``TRITON_reporting_timestep_s`` are carried THROUGH the
        ``model_validate`` overlay dict -- so the on-disk _test YAML that
        prepare_scenario_runner.py:139 reloads points at the short weather (an
        in-memory setattr would be silently discarded at that reload, Gotcha 15).
        Scenario-prep regenerates the SWMM .inp window from the sliced weather
        (swmm_utils.py:105-110), so no .inp-edit pass is needed. ``analysis_dir``
        and ``master_analysis_cfg_yaml`` are ALSO carried through the overlay: the
        is_subanalysis model-validator (config/analysis.py:529-541) requires both,
        and ``analysis_dir`` is what keeps every test artifact under ``_test/`` (R2 --
        without it ``__init__`` derives ``system_directory/{analysis_id}``)."""
        import xarray as xr

        test_root = self.analysis_paths.analysis_dir / "_test"
        test_root.mkdir(parents=True, exist_ok=True)
        subs: list = []
        for i, rep in enumerate(representatives):
            group_id = f"group_{i}"
            sub_dir = test_root / group_id
            sub_dir.mkdir(parents=True, exist_ok=True)
            base_cfg = rep.source_analysis.cfg_analysis
            # --- slice the real weather to the first N reporting frames (+endpoint) ---
            time_dim = base_cfg.weather_time_series_timestep_dimension_name
            with xr.open_dataset(base_cfg.weather_timeseries, engine="h5netcdf") as wx:
                wx_short = wx.isel({time_dim: slice(0, n_reporting_timesteps + 1)}).load()
            # Write the sliced weather to the _test/ root (sibling of group_0/), NOT
            # inside sub_dir: the sub runs with from_scratch=True (test(), below) which
            # fast_rmtree's sub_dir (== analysis_dir == group_0/), so a weather file
            # written inside it would be deleted before prepare_scenario reads it
            # (FileNotFoundError on _test_weather.nc). The _test/ root survives the wipe;
            # the {group_id}-suffixed name keeps multiple groups from colliding.
            short_weather = test_root / f"_test_weather_{group_id}.nc"
            wx_short.to_netcdf(short_weather, engine="h5netcdf")
            overlay = {
                "analysis_id": f"{self.cfg_analysis.analysis_id}_test_{group_id}",
                "toggle_sensitivity_analysis": False,
                "is_subanalysis": True,  # cfg field, NOT a constructor kwarg
                "analysis_dir": str(sub_dir),  # keeps all artifacts under _test/ (R2)
                "master_analysis_cfg_yaml": str(self.analysis_config_yaml),
                "weather_timeseries": str(short_weather),  # short weather survives the YAML reload
                "TRITON_reporting_timestep_s": reporting_timestep_s,
            }
            cfg_a = analysis_config.model_validate({**base_cfg.model_dump(), **overlay})
            # The sub runs with from_scratch=True (test(), below), which fast_rmtree's
            # the sub's analysis_dir (== sub_dir / group_0). Writing the config INSIDE
            # sub_dir would self-delete it before the setup runner reads it ("Analysis
            # config not found"). Write it to the _test/ root (sibling of group_0/) so
            # the wipe of group_0/ cannot reach it; the sub's analysis_dir stays
            # group_0/ via the overlay. (analysis_id is group-unique, so no collision.)
            sub_yaml = self._atomic_write_subanalysis_yaml(cfg_a, test_root)
            # TRITONSWMM_analysis.__init__ has NO is_subanalysis param; is_subanalysis
            # is set in the cfg overlay above (mirrors _create_sub_analyses,
            # sensitivity_analysis.py:2031). is_main_orchestrator=False -- a _test/
            # sub is not a main orchestrator. The representative's own _system carries
            # the right model toggles / resolved target hardware.
            sub = TRITONSWMM_analysis(
                sub_yaml,
                system=rep.source_analysis._system,
                hpc_system_config_yaml=self.hpc_system_config_yaml,
                is_main_orchestrator=False,
            )
            subs.append(sub)
        return subs

    def _atomic_write_subanalysis_yaml(self, cfg_a: "analysis_config", sub_dir: Path) -> Path:
        """Atomically write a sub-analysis overlay config to ``{sub_dir}/{id}.yaml``
        via a PID-keyed temp file + ``Path.replace`` (POSIX-atomic on one
        filesystem), so a concurrent reader never catches a truncated file.

        DRY-DEBT: shared atomic-overlay-write contract -- mirrors the temp+replace
        overlay-YAML write at ``sensitivity_analysis.py::_create_sub_analyses``
        (:2056-2063). The source module is layout-relevant
        (``_layout_relevant_files.yaml``) and cannot be edited here without tripping
        CI Check B, so the dance is duplicated rather than extracted into one shared
        helper; mirror any change across BOTH sites."""
        cfg_yaml = sub_dir / f"{cfg_a.analysis_id}.yaml"
        _tmp = cfg_yaml.with_suffix(cfg_yaml.suffix + f".{os.getpid()}.tmp")
        _tmp.write_text(yaml.safe_dump(cfg_a.model_dump(mode="json"), sort_keys=False))
        _tmp.replace(cfg_yaml)
        return cfg_yaml

    def render_report(self, format: "Literal['html','zip']" = "zip", *, reprocess: bool = False) -> "Path":
        """Render the report from already-completed workflow outputs.

        Idempotent: invokes ``snakemake --report`` against the existing Snakefile
        without re-executing any rules. Requires the workflow to have completed
        (so the report() outputs exist) and the Snakefile to be on disk.

        Parameters
        ----------
        format : Literal["html", "zip"], default "zip"
            Output format. ``"html"`` produces a single self-contained
            ``analysis_report.html`` with all figures inlined as base64, plus
            React-bundle post-process surgery. ``"zip"`` produces
            ``analysis_report.zip`` containing the unbundled report tree;
            no post-process surgery is applied.
        reprocess : bool, default False
            When ``True``, render against ``Snakefile.reprocess`` (the filtered
            reprocess DAG) instead of the production ``Snakefile``, so the
            ``snakemake --report`` step only expects the figures the reprocess
            DAG built. Keyword-only; set by the reprocess ``render_report`` rule
            shell. Default ``False`` keeps the production render path
            byte-identical.

        Returns
        -------
        Path
            Path to the rendered ``analysis_report.{format}``.
        """
        import subprocess
        import sys

        from .exceptions import WorkflowError
        from .workflow import _assert_snakefile_package_current

        snakefile_name = "Snakefile.reprocess" if reprocess else "Snakefile"
        snakefile = self.analysis_paths.analysis_dir / snakefile_name
        _assert_snakefile_package_current(snakefile)
        out = self.analysis_paths.analysis_dir / f"analysis_report.{format}"
        css_path = self.analysis_paths.analysis_dir / "report" / "report.css"
        # Brand-theme resolution (ADR-7 layer 2). The dominant render path is
        # render_report_runner.main() → a FRESH TRITONSWMM_analysis(..., is_main_
        # orchestrator=False) that never had run() called, so self._brand_theme
        # (set only in run()) does NOT exist on it. Resolve once here via
        # getattr-fallback and serve BOTH the CSS emit below and the navbar
        # surgery later (D-6; plan-review SE Flag 1).
        from .config.brand_theme import DEFAULT_BRAND_THEME
        from .config.loaders import load_brand_theme
        from .workflow import _brand_theme_css_map

        _theme = getattr(self, "_brand_theme", None)
        if _theme is None:
            _theme = (
                load_brand_theme(self.cfg_analysis.brand_theme)
                if self.cfg_analysis.brand_theme is not None
                else DEFAULT_BRAND_THEME
            )
        # Re-emit report artifacts (report.css + workflow_description template)
        # from package resources so render_report picks up edits made to the
        # source-tree report_templates/ since the analysis was last run.
        _emit_report_artifacts(
            self.analysis_paths.analysis_dir,
            brand_theme=_brand_theme_css_map(_theme),
        )
        # --cores 1 is required by Snakemake's CLI even though --report is a
        # post-execution render that does not execute rules.
        cmd = [
            sys.executable,
            "-m",
            "snakemake",
            "--snakefile",
            str(snakefile),
            "--directory",
            str(self.analysis_paths.analysis_dir),
            "--report",
            str(out),
            "--report-stylesheet",
            str(css_path),
            "--cores",
            "1",
        ]
        result = subprocess.run(cmd, capture_output=True, text=True)
        if result.returncode != 0:
            tail = "\n".join((result.stdout + "\n" + result.stderr).splitlines()[-50:])
            raise WorkflowError(
                phase="render_report",
                return_code=result.returncode,
                stderr=f"snakemake --report exit {result.returncode}; last 50 lines:\n{tail}",
            )
        # Apply React-bundle post-process surgery (title, navbar, sort order,
        # placeholder category, showCategory auto-pop, row-click delegate).
        # Both formats need the surgery:
        #  - HTML: edit the single rendered file in place.
        #  - Zip: extract, edit `analysis_report/report.html` inside, re-zip.
        # Without surgery in zip mode, the eye-icon-hiding CSS in report.css
        # leaves figure tables with no clickable affordance (the JS click
        # delegate that makes rows clickable lives only in the surgery).
        from .report_renderers._react_surgery import (
            apply_post_process_surgery,
            apply_post_process_surgery_to_zip,
        )

        # Navbar upper-left brand text: brand_theme.upper_left_text (ADR-7),
        # defaulting to analysis_id when None (D-6). _theme is resolved above.
        _navbar = _theme.upper_left_text or self.cfg_analysis.analysis_id
        # Resolve the active set's category_order. render_report() is dominantly
        # invoked from render_report_runner.main() on a FRESH analysis that never
        # called run() (see the _brand_theme getattr-fallback above for the
        # identical hazard), so self._active_reporting_set may not exist. getattr-
        # fallback to a config-only resolution (no CSV cross-validation at render
        # time) mirroring the _theme fallback above. Never let the bare attribute
        # AttributeError be swallowed by the surrounding `except Exception: pass`.
        _active_set = getattr(self, "_active_reporting_set", None)
        if _active_set is None:
            # render-without-run() fallback. Fail SOFT (SE F-I-3): the render path
            # bypasses validate_active_reporting_set, so a stale/unknown
            # reporting_set would raise here and surface as an opaque Snakemake
            # rule failure. Degrade to the historical "default" sidebar order + a
            # one-line warning instead of crashing the render rule.
            import logging

            from .config.report import resolve_active_reporting_set_name
            from .report_renderers._reporting_sets import get_reporting_set

            try:
                _cfg_report = getattr(self, "_cfg_report", None)
                if _cfg_report is None:
                    _cfg_report = self.cfg_analysis.report
                _set_name = resolve_active_reporting_set_name(
                    _cfg_report,
                    is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
                )
                _active_set = get_reporting_set(_set_name)
            except Exception as _e:
                logging.getLogger(__name__).warning(
                    "render-path reporting_set resolution failed (%s); falling back to 'default' category order",
                    _e,
                )
                _active_set = get_reporting_set("default")
        _category_order = list(_active_set.category_order)
        try:
            if format == "html":
                out.write_text(
                    apply_post_process_surgery(
                        out.read_text(),
                        navbar_text=_navbar,
                        category_order=_category_order,
                    )
                )
            else:
                apply_post_process_surgery_to_zip(out, navbar_text=_navbar, category_order=_category_order)
        except Exception:
            pass
        if format != "html":
            return out
        out_html = out
        # Snap-confined browsers (Ubuntu Firefox snap) cannot read files under
        # ~/.cache/. If the rendered report lands there, surface a one-line
        # workaround so the user does not hit "Access to the file was denied".
        try:
            if "/.cache/" in str(out_html):
                print(
                    f"[render_report] {out_html}\n"
                    f"[render_report] Note: snap-confined browsers cannot read ~/.cache; "
                    f"copy to ~/Downloads to view: cp {out_html} ~/Downloads/",
                    flush=True,
                )
        except Exception:
            pass
        return out_html

    @property
    def n_scenarios(self):
        sensitivity_scenario = 1
        if self.cfg_analysis.toggle_sensitivity_analysis:
            sens = self.sensitivity
            sensitivity_scenario = len(sens.df_setup)

        n_total = len(self.df_sims) * sensitivity_scenario
        return n_total

    @property
    def n_sims(self):
        sensitivity_scenario = 1
        if self.cfg_analysis.toggle_sensitivity_analysis:
            sens = self.sensitivity
            sensitivity_scenario = len(sens.df_setup)

        n_total = len(self.df_sims) * len(self._get_enabled_model_types()) * sensitivity_scenario
        return n_total

    def get_workflow_status(self) -> "WorkflowStatus":
        """Generate workflow status report.

        Inspects logs and outputs to determine completion state of each phase,
        providing actionable recommendations for which execution mode to use.

        Returns
        -------
        WorkflowStatus
            Structured status report with phase details and recommendations

        Examples
        --------
        Check status before running:

        >>> status = analysis.get_workflow_status()
        >>> print(status)
        >>> if not status.simulation.complete:
        ...     print(f"Retry {len(status.simulation.failed_items)} failed sims")

        Use recommended mode:

        >>> status = analysis.get_workflow_status()
        >>> result = analysis.run(mode=status.recommended_mode)

        Notes
        -----
        This method is read-only and does not modify any state. It provides
        transparency into workflow progress to help users make informed
        decisions about execution modes.

        See Also
        --------
        run : High-level workflow execution method
        """
        from .orchestration import PhaseStatus, WorkflowStatus

        # Check setup phase
        system_log = self._system.log
        dem_done = system_log.dem_processed.get()
        mannings_done = self._system.cfg_system.toggle_use_constant_mannings or system_log.mannings_processed.get()
        compiled = system_log.compilation_tritonswmm_cpu_successful.get()

        setup_complete = dem_done and mannings_done and compiled
        setup_progress = 1.0 if setup_complete else 0.5 if (dem_done or compiled) else 0.0
        setup_details = {
            "dem": f"{'✓' if dem_done else '✗'} DEM processed",
            "mannings": f"{'✓' if mannings_done else '✗'} Manning's processed",
            "compiled": f"{'✓' if compiled else '✗'} TRITON-SWMM compiled",
        }

        setup_phase = PhaseStatus(
            name="setup",
            complete=setup_complete,
            progress=setup_progress,
            details=setup_details,
        )

        # Check scenario preparation
        all_prepared = self._all_scenarios_created
        not_prepared = self._scenarios_not_created

        n_total = self.n_sims

        n_prepared = n_total - len(not_prepared)

        prep_phase = PhaseStatus(
            name="preparation",
            complete=all_prepared,
            progress=n_prepared / n_total if n_total > 0 else 0.0,
            details={"scenarios": f"{'✓' if all_prepared else '⚠'} {n_prepared}/{n_total} scenarios created"},
            failed_items=[str(p) for p in not_prepared],
        )

        # Check simulations
        all_run = self._all_sims_run
        not_run = self._scenarios_not_run
        n_run = n_total - len(not_run)

        sim_phase = PhaseStatus(
            name="simulation",
            complete=all_run,
            progress=n_run / n_total if n_total > 0 else 0.0,
            details={"sims": f"{'✓' if all_run else '⚠'} {n_run}/{n_total} simulations completed"},
            failed_items=[str(p) for p in not_run],
        )

        # Check processing
        enabled_models = self._get_enabled_model_types()
        triton_enabled = "triton" in enabled_models or "tritonswmm" in enabled_models
        swmm_enabled = "swmm" in enabled_models or "tritonswmm" in enabled_models

        triton_missing = len(self._TRITON_time_series_not_processed) if triton_enabled else 0
        swmm_missing = len(self._SWMM_time_series_not_processed) if swmm_enabled else 0

        triton_total = n_total if triton_enabled else 0
        swmm_total = n_total if swmm_enabled else 0

        triton_processed = max(triton_total - triton_missing, 0)
        swmm_processed = max(swmm_total - swmm_missing, 0)

        processed_total = triton_processed + swmm_processed
        total_needed = triton_total + swmm_total
        proc_progress = processed_total / total_needed if total_needed else 0.0

        triton_proc_complete = triton_missing == 0 if triton_enabled else True
        swmm_proc_complete = swmm_missing == 0 if swmm_enabled else True
        proc_complete = triton_proc_complete and swmm_proc_complete

        proc_phase = PhaseStatus(
            name="processing",
            complete=proc_complete,
            progress=proc_progress,
            details={
                "triton": (
                    f"{'✓' if triton_proc_complete else '✗'} TRITON outputs processed: "
                    f"{triton_processed}/{triton_total}"
                    if triton_enabled
                    else "✓ TRITON outputs processed: n/a"
                ),
                "swmm": (
                    f"{'✓' if swmm_proc_complete else '✗'} SWMM outputs processed: {swmm_processed}/{swmm_total}"
                    if swmm_enabled
                    else "✓ SWMM outputs processed: n/a"
                ),
            },
        )

        # Check consolidation: under Option B (render_bundle plan), the
        # canonical master-level artifact is analysis_datatree.zarr; the
        # per-mode flat zarrs no longer exist. The log marker
        # `datatree_consolidation_complete` is the single canonical signal
        # for "consolidation has completed."
        summaries_exist = (
            hasattr(self.log, "datatree_consolidation_complete")
            and self.log.datatree_consolidation_complete.get() is True
        )

        consol_details = {
            "datatree": (
                f"{'✓' if summaries_exist else '✗'} analysis_datatree.zarr "
                f"({'present' if summaries_exist else 'not yet built'})"
            )
        }

        consol_phase = PhaseStatus(
            name="consolidation",
            complete=summaries_exist,
            progress=1.0 if summaries_exist else 0.0,
            details=consol_details,
        )

        # Determine current phase and recommendation
        if not setup_complete:
            current = "setup"
            rec_mode = "fresh"
            rec_text = "Setup incomplete. Use 'fresh' mode to process system inputs."
        elif not all_prepared:
            current = "preparation"
            rec_mode = "resume"
            rec_text = f"Use 'resume' to create {len(not_prepared)} remaining scenarios."
        elif not all_run:
            current = "simulation"
            rec_mode = "resume"
            rec_text = f"Use 'resume' to run {len(not_run)} pending/failed simulations."
        elif not proc_complete:
            current = "processing"
            rec_mode = "resume"
            rec_text = "Use 'resume' to process simulation outputs."
        elif not summaries_exist:
            current = "consolidation"
            rec_mode = "resume"
            rec_text = "Use 'resume' to consolidate analysis summaries."
        else:
            current = "complete"
            # 'fresh' is the only actionable mode for a complete analysis (resume has
            # nothing left to do); it is a valid translate_mode() input, so
            # analysis.run(mode=status.recommended_mode) works. 'n/a' is not a member
            # of the documented {fresh, resume} run-mode set and is not run()-able.
            rec_mode = "fresh"
            rec_text = "All phases complete. Use 'fresh' to redo the analysis from scratch."

        return WorkflowStatus(
            analysis_id=self.cfg_analysis.analysis_id,
            analysis_dir=self.analysis_paths.analysis_dir,
            setup=setup_phase,
            preparation=prep_phase,
            simulation=sim_phase,
            processing=proc_phase,
            consolidation=consol_phase,
            total_simulations=n_total,
            simulations_completed=n_run,
            simulations_failed=len(not_run),
            simulations_pending=0,  # Would need more logic to distinguish failed vs pending
            current_phase=current,
            recommended_mode=rec_mode,
            recommendation=rec_text,
        )

    def submit_workflow(
        self,
        mode: Literal["local", "slurm", "auto"] = "auto",
        process_system_level_inputs: bool = False,
        overwrite_system_inputs: bool = False,
        compile_TRITON_SWMM: bool = True,
        recompile_if_already_done_successfully: bool = False,
        prepare_scenarios: bool = True,
        overwrite_scenario_if_already_set_up: bool = False,
        rerun_swmm_hydro_if_outputs_exist: bool = False,
        process_timeseries: bool = True,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        override_clear_raw: ClearRawValue | None = None,
        override_force_rerun: ForceRerunValue | None = None,
        compression_level: int = 5,
        pickup_where_leftoff: bool = False,
        wait_for_completion: bool = False,  # relevant for slurm jobs only
        dry_run: bool = False,
        verbose: bool = True,
        override_hpc_total_nodes: int | None = None,
        override_hpc_restart_times_simulate: int | None = None,
        override_hpc_restart_times_other: int | None = None,
        report_formats: list[str] | None = None,
        extra_sbatch_args: list[str] | None = None,
        snakemake_diagnostics: SnakemakeDiagnostics | None = None,
    ) -> dict:
        """
        Submit workflow using Snakemake (replaces submit_SLURM_job_array).

        Automatically detects execution context (local vs. HPC) and submits accordingly.

        Delegates to SnakemakeWorkflowBuilder.

        Parameters
        ----------
        mode : Literal["local", "slurm", "auto"]
            Execution mode. If "auto", detects based on SLURM environment variables.
        process_system_level_inputs : bool
            If True, process system-level inputs (DEM, Mannings) in Phase 1
        overwrite_system_inputs : bool
            If True, overwrite existing system input files
        compile_TRITON_SWMM : bool
            If True, compile TRITON-SWMM in Phase 1
        recompile_if_already_done_successfully : bool
            If True, recompile even if already compiled successfully
        prepare_scenarios : bool
            If True, each simulation will prepare its scenario before running
        overwrite_scenario_if_already_set_up : bool
            If True, overwrite existing scenarios
        rerun_swmm_hydro_if_outputs_exist : bool
            If True, rerun SWMM hydrology model even if outputs exist
        process_timeseries : bool
            If True, process timeseries outputs after each simulation
        which : Literal["TRITON", "SWMM", "both"]
            Which outputs to process (only used if process_timeseries=True)
        override_clear_raw : ClearRawValue | None
            Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
            reads from YAML; concrete values follow the override-prefix convention.
        compression_level : int
            Compression level for output files (0-9)
        pickup_where_leftoff : bool
            If True, resume simulations from last checkpoint
        wait_for_completion : bool
            If True, wait for workflow completion (relevant for slurm jobs only)
        dry_run : bool
            If True, only perform a dry run and return that result
        verbose : bool
            If True, print progress messages
        override_hpc_total_nodes : int | None
            If set, overrides `hpc_total_nodes` in the generated SBATCH script without
            mutating the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.

        Returns
        -------
        dict
            Status dictionary with keys:
            - success: bool - Whether workflow succeeded
            - mode: str - "local" or "slurm"
            - snakefile_path: Path - Path to generated Snakefile
            - job_id: str | None - Job ID (only for slurm mode)
            - message: str - Status message
        """
        # Stamp _version.json at LAYOUT_VERSION on first materialization (lazy
        # stamp per version_migration_system master plan PI-1). Idempotent
        # under concurrent writers.
        from hhemt.version_migration import LAYOUT_VERSION
        from hhemt.version_migration.state import stamp_new_target

        stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

        # resume-retry-resilience P3 — surface the zero-progress-resume foot-gun
        # before launch (friction Option A). Fires only when actually resuming in an
        # HPC mode with a per-sim walltime set; run() reaches here via delegation,
        # so this single site covers both run() and direct submit_workflow() callers.
        self._warn_resume_zero_progress(pickup_where_leftoff)

        # Driver-start orchestrator-liveness sentinel (Phase 2 of the reprocess
        # concurrency gate). Single-writer per logical driver: a sensitivity
        # run() delegates below to self.sensitivity.submit_workflow, which writes
        # its OWN master-keyed sentinel — writing here too would double-write into
        # the same _status/_orchestrator/ dir for one logical driver, so guard on
        # NOT sensitivity (sensitivity runs leave _driver_id None here).
        _driver_id = None
        _eff_mode = self.cfg_analysis.multi_sim_run_method
        if not self.cfg_analysis.toggle_sensitivity_analysis:
            _driver_id = _osent.new_driver_id()
            _osent.write_orchestrator_sentinel(
                self.analysis_paths.analysis_dir,
                driver_id=_driver_id,
                workflow_submission_mode=_eff_mode,
            )

        try:
            # Force-rerun pre-delete (login-node responsibility per master plan
            # Strategy). Resolve + validate + delete BEFORE Snakemake plans the DAG
            # so MTIME-input triggers cascade re-fire automatically.
            self._apply_force_rerun(override_force_rerun)

            if self.cfg_analysis.toggle_sensitivity_analysis:
                result = self.sensitivity.submit_workflow(
                    mode=mode,
                    process_system_level_inputs=process_system_level_inputs,
                    overwrite_system_inputs=overwrite_system_inputs,
                    compile_TRITON_SWMM=compile_TRITON_SWMM,
                    recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                    prepare_scenarios=prepare_scenarios,
                    overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                    rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                    process_timeseries=process_timeseries,
                    which=which,
                    override_clear_raw=override_clear_raw,
                    override_force_rerun=override_force_rerun,
                    compression_level=compression_level,
                    pickup_where_leftoff=pickup_where_leftoff,
                    wait_for_completion=wait_for_completion,
                    dry_run=dry_run,
                    verbose=verbose,
                    override_hpc_total_nodes=override_hpc_total_nodes,
                    override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
                    override_hpc_restart_times_other=override_hpc_restart_times_other,
                    report_formats=report_formats,
                    extra_sbatch_args=extra_sbatch_args,
                    snakemake_diagnostics=snakemake_diagnostics,
                )
            else:
                # NOTE: override_force_rerun is NOT threaded into the inner builder
                # — the pre-delete already happened at this layer
                # (self._apply_force_rerun above) and the builder's
                # submit_workflow does not need a runtime force-rerun parameter.
                if pickup_where_leftoff:
                    # Scenario-set-change invalidation (multi_sim resume). The toolkit
                    # uses --rerun-triggers mtime only (workflow.py), so a present
                    # e_consolidate_complete.flag short-circuits the missing-intermediate
                    # demand for an ADDED scenario's prepare->run->process->consolidate
                    # chain (the per-sim plot rules, enumerated over the full SIM_IDS,
                    # would then fire and crash reading the un-prepared hydraulics.inp).
                    # Mirror the sensitivity path's scenario-set-change invalidation:
                    # when the config event-id set differs from the on-disk prepared set,
                    # delete the analysis-level consolidate flag so consolidate re-demands
                    # its full SIM_IDS input expand, and delete orphan per-event flags for
                    # events no longer in the config.
                    self._invalidate_consolidate_flag_on_scenario_set_change()
                result = self._workflow_builder.submit_workflow(
                    mode=mode,
                    process_system_level_inputs=process_system_level_inputs,
                    overwrite_system_inputs=overwrite_system_inputs,
                    compile_TRITON_SWMM=compile_TRITON_SWMM,
                    recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                    prepare_scenarios=prepare_scenarios,
                    overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                    rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                    process_timeseries=process_timeseries,
                    which=which,
                    override_clear_raw=override_clear_raw,
                    compression_level=compression_level,
                    pickup_where_leftoff=pickup_where_leftoff,
                    wait_for_completion=wait_for_completion,
                    dry_run=dry_run,
                    verbose=verbose,
                    override_hpc_total_nodes=override_hpc_total_nodes,
                    override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
                    override_hpc_restart_times_other=override_hpc_restart_times_other,
                    report_formats=report_formats,
                    extra_sbatch_args=extra_sbatch_args,
                    snakemake_diagnostics=snakemake_diagnostics,
                )

            if dry_run and result.get("success"):
                snakemake_logfile = result.get("snakemake_logfile")
                if snakemake_logfile is not None:
                    report_path = generate_dry_run_report_markdown(
                        snakemake_logfile=Path(snakemake_logfile),
                        analysis_dir=self.analysis_paths.analysis_dir,
                        verbose=verbose,
                    )
                    result["dry_run_report_markdown"] = report_path
        finally:
            # Blocking-local drivers (this Python process WAS the driver and the
            # builder call blocked to completion) remove the sentinel on return;
            # detached drivers leave a durable sentinel reclaimed by the gate's
            # liveness probes. Only the local arm removes here.
            if _driver_id is not None and _eff_mode == "local":
                _osent.remove_orchestrator_sentinel(self.analysis_paths.analysis_dir, _driver_id)

        # Detached drivers: enrich (persist, do not remove) with the driver's
        # slurm_jobid (single-job) / tmux_session_name (batch_job) so the gate's
        # sacct/tmux arms can probe it. Skipped for sensitivity runs (_driver_id
        # is None — the sensitivity-master submit owns its own sentinel).
        if _driver_id is not None and _eff_mode != "local" and isinstance(result, dict):
            _osent.enrich_orchestrator_sentinel(
                self.analysis_paths.analysis_dir,
                driver_id=_driver_id,
                slurm_jobid=result.get("job_id"),
                tmux_session_name=result.get("session_name"),
            )

        return result

    def reprocess(
        self,
        start_with: "Literal['process','consolidate','render']" = "consolidate",
        execution_mode: "Literal['auto','local','slurm']" = "auto",
        which: "Literal['TRITON','SWMM','both']" = "both",
        *,
        regenerate_existing: bool = False,
        delete_via_slurm: bool | None = None,
        override_clear_raw: ClearRawValue | None = "none",
        override_force_rerun: ForceRerunValue | None = None,
        verbose: bool = True,
        dry_run: bool = False,
        prune_settled_markers: bool = True,
        report_formats: list[Literal["html", "zip"]] | None = None,
    ) -> dict:
        """Re-run downstream stages against existing sim outputs.

        Re-runs processing / consolidation / plotting / report rendering
        without re-running the simulation rules. Runs the Phase-1
        reconciliation guard against ``_status/_submitted/`` before
        submitting, so a parallel live sim driver cannot be double-submitted.
        Emits a scope-limited Snakefile at
        ``{analysis_dir}/Snakefile.reprocess`` and runs it against the shared
        ``.snakemake/`` with ``--nolock``; the ``_status/_orchestrator/``
        liveness gate (not the Snakemake lock) prevents collision with a live
        orchestration driver.

        Parameters
        ----------
        start_with
            Stage to re-fire from. ``"consolidate"`` is the common case —
            re-aggregates the analysis datatree zarr and re-renders the
            report against existing sim outputs.
        execution_mode
            ``"auto"`` (default) detects SLURM context; ``"local"`` /
            ``"slurm"`` force the mode.
        which
            ``"both"`` (default) / ``"TRITON"`` / ``"SWMM"`` — passes through
            to ``rule consolidate``'s ``--which`` flag.
        regenerate_existing
            **Default False.** When False, reprocess regenerates ONLY report +
            plot artifacts against the EXISTING consolidated zarr — no zarr
            deletion, no DU restamp walk. When True, the legacy destructive
            delete-and-rebuild runs. A plain reprocess-only toggle (NOT
            ``override_``-prefixed: bridges no config field; meaningless on
            ``run()``). Matches the ``prune_settled_markers`` plain-bool
            precedent.
        delete_via_slurm
            **None (default) auto-resolves**: route the opt-in deletion through
            the SLURM ``analysis.delete()`` architecture iff
            ``multi_sim_run_method`` is an HPC mode (``batch_job`` /
            ``1_job_many_srun_tasks``); ``local`` → in-process ``fast_rmtree``.
            Pass ``True`` / ``False`` to force. (CLI exposes
            ``--delete-via-slurm/--no-delete-via-slurm``.)
        override_clear_raw
            **Hard-default "none"** to preserve historic ``reprocess`` semantics
            (reprocess never auto-clears unless the caller explicitly opts in).
            Pass ``None`` to read ``cfg_analysis.clear_raw``; pass ``"all"`` /
            ``"none"`` / a list of model types to override. When the resolved
            value is anything other than ``"none"``, two guards must both pass:
            (a) every enabled sim's ``c_run_*`` flag must exist (no
            never-started sims); (b) no ``_status/_submitted/`` sentinel
            may be present (no in-flight / just-died sims). Cites
            stipulation ``clear raw triton outputs deferred until last allocation``
            (under ``library/docs/stipulations/hhemt/``).
        verbose
            If True, print progress messages.
        dry_run
            If True, runs ``snakemake --dry-run`` only.
        prune_settled_markers
            When True (default), prunes settled ``_status/_completed`` /
            ``_status/_failed`` markers (those whose ``_submitted/`` sibling is
            gone) at the master ``_status/`` level before submitting. Opt-out;
            mirrors ``run()``. Inert hygiene — does not affect reconcile
            correctness.

        Returns
        -------
        dict
            Status dictionary from
            :meth:`SnakemakeWorkflowBuilder.submit_reprocess_workflow`.

        Raises
        ------
        ConfigurationError
            When the resolved ``clear_raw`` would clear and either guard fails.
        """
        resolved_clear_raw = override_clear_raw if override_clear_raw is not None else self.cfg_analysis.clear_raw
        # True iff the resolved value would trigger any cleanup for any model.
        would_clear = resolved_clear_raw != "none"
        # Lazy-stamp _version.json at LAYOUT_VERSION (PI-1 pattern, mirroring
        # run() and submit_workflow). Idempotent under concurrent writers;
        # if _version.json is missing or stamped at an older version, this
        # writes a fresh stamp at the current LAYOUT_VERSION.
        from hhemt.version_migration import LAYOUT_VERSION
        from hhemt.version_migration.state import stamp_new_target

        from .exceptions import ConfigurationError

        stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

        # Prune settled _status markers (opt-out) — inert hygiene, mirrors run().
        if prune_settled_markers:
            pruned = self._prune_settled_markers()
            if verbose and pruned:
                print(
                    f"[prune-settled-markers] Pruned {len(pruned)} settled "
                    f"marker(s) from {self.analysis_paths.analysis_dir}/_status/",
                    flush=True,
                )

        # Dispatch to the sensitivity-master reprocess path for sensitivity-toggled
        # analyses. The non-sensitivity reprocess generator emits a `rule consolidate`
        # that consumes from `analysis_dir/sims/`, which for sensitivity layouts does
        # not exist — sims live under `subanalyses/sa_*/sims/`. The sensitivity-master
        # generator (SensitivityAnalysisWorkflowBuilder.generate_reprocess_master_snakefile_content)
        # emits per-sa consolidate rules + a master_consolidation rule that consume
        # from the correct paths. Pattern mirrors analysis.py:683-801 property
        # dispatches and the bundle CLI dispatch at cli.py:1026.
        if self.cfg_analysis.toggle_sensitivity_analysis:
            if would_clear:
                raise ConfigurationError(
                    field="override_clear_raw",
                    message=(
                        "TRITONSWMM_analysis.reprocess does not support clearing raw outputs "
                        "for sensitivity-toggled analyses (resolved clear_raw="
                        f"{resolved_clear_raw!r}). The sensitivity-master reprocess "
                        "path deliberately omits the clear-raw gate (see "
                        "TRITONSWMM_sensitivity_analysis.reprocess docstring). Invoke "
                        "self.sensitivity.reprocess(...) directly with explicit sa_ids if "
                        "raw-output clearing is required."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )
            # R7 master-level in-flight guard (Phase 3) — refuse processed-output
            # deletion while live workers may be re-writing the master analysis's
            # processed dirs. Mirrors the non-sensitivity guard below. (Per-sub
            # subanalyses/sa_*/_status/_submitted/ recursion is a documented later
            # refinement; the conservative guard refuses on master presence.)
            if start_with == "process" and regenerate_existing and not dry_run:
                submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
                if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
                    raise ConfigurationError(
                        field="regenerate_existing",
                        message=(
                            "reprocess refuses processed-output deletion "
                            "(start_with='process', regenerate_existing=True) while "
                            "_submitted/ sentinels are present in the sensitivity master "
                            "_status/ — simulations may still be in flight or recently "
                            "died and could be re-writing the same processed/ directories. "
                            "Run the reconciliation guard or `scancel` outstanding jobs first."
                        ),
                        config_path=str(self.analysis_config_yaml),
                    )
            return self.sensitivity.reprocess(
                start_with=start_with,
                execution_mode=execution_mode,
                which=which,
                regenerate_existing=regenerate_existing,
                delete_via_slurm=delete_via_slurm,
                override_force_rerun=override_force_rerun,
                verbose=verbose,
                dry_run=dry_run,
                report_formats=report_formats,
            )

        if would_clear:
            # Guard (a): every enabled sim must have a c_run_* flag.
            if not self._all_sim_flags_present():
                raise ConfigurationError(
                    field="override_clear_raw",
                    message=(
                        "reprocess refuses raw-output clearing while c_run_* flags are absent "
                        f"(resolved clear_raw={resolved_clear_raw!r}; some sims have not completed). "
                        "See stipulation `clear raw triton outputs deferred until last allocation`."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )
            # Guard (b): no in-flight or unreconciled _submitted/ sentinel.
            submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
            if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
                raise ConfigurationError(
                    field="override_clear_raw",
                    message=(
                        "reprocess refuses raw-output clearing while _submitted/ sentinels are present "
                        f"(resolved clear_raw={resolved_clear_raw!r}; simulations may still be in flight "
                        "or recently died). Run the Phase-1 reconciliation guard or `scancel` outstanding "
                        "jobs first."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )

        # Reprocess overrides 1_job_many_srun_tasks → batch_job at submission
        # time. 1_job_many_srun_tasks reserves an exclusive multi-node SLURM
        # allocation that the downstream-only reprocess does not need, and the
        # method cannot decouple driver-cancel from job-cancel (master plan
        # Assumptions + FQ1 research). The override is local — the analysis's
        # original cfg_analysis.multi_sim_run_method is not mutated.
        effective_method: str | None = None
        if self.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks":
            effective_method = "batch_job"
            if verbose:
                print(
                    "[reprocess] NOTE: 1_job_many_srun_tasks reprocess overridden to "
                    "batch_job (per-rule sbatch). The original analysis_config is unchanged.",
                    flush=True,
                )

        # Invalidate from start_with onward — deletes the upstream flag/artifact
        # that triggers Snakemake's mtime-driven re-fire. Per D-INVALIDATE
        # option 1: delete flags + rely on the generator's baked overwrite.
        # On dry_run, the flag deletion still happens (it is the cheap,
        # rerun-recreated trigger that makes the --dry-run DAG meaningful);
        # only the destructive consolidated-zarr deletion + DU restamp are
        # skipped (see `reprocess dry_run performs no destructive mutation`
        # stipulation).
        # Opt-in processed-output deletion (rebuild-from-raw, FQ2). Refuse while
        # live sim workers may be re-writing the same processed/ dir — mirrors
        # the clear-raw in-flight guard (analysis.py:2607-2619). reprocess
        # coexists with live workers generally, but DELETING the artifact a live
        # worker is writing is unsafe.
        if start_with == "process" and regenerate_existing and not dry_run:
            submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
            if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
                raise ConfigurationError(
                    field="regenerate_existing",
                    message=(
                        "reprocess refuses processed-output deletion "
                        "(start_with='process', regenerate_existing=True) while "
                        "_submitted/ sentinels are present — simulations may still be "
                        "in flight or recently died and could be re-writing the same "
                        "processed/ directories. Run the reconciliation guard or "
                        "`scancel` outstanding jobs first."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )

        # R8 routing — computed ONCE; shared by both deletion sites. None
        # auto-resolves to slurm-offload on HPC modes (user D6 refinement 1).
        _hpc = self.cfg_analysis.multi_sim_run_method in ("batch_job", "1_job_many_srun_tasks")
        _resolved_delete_via_slurm = _hpc if delete_via_slurm is None else delete_via_slurm
        route_delete_via_slurm = regenerate_existing and _resolved_delete_via_slurm and not dry_run and _hpc
        # Divergence self-heal (FIX 2) — fires on the process path REGARDLESS of
        # regenerate_existing (D2). Reconciles d_process flag + per-model
        # processing_log against on-disk summary presence: where a flag survives
        # but the enabled-model summary set is absent (the May-31 divergence),
        # unlink the flag + clear the log so the workflow.py:6684 emit gate
        # re-emits the process rule and _already_written (Gotcha 28) lets it
        # write. No-op when every enabled summary is present (healthy analysis).
        if start_with == "process" and not dry_run:
            _reconciled = self._reconcile_stale_process_flags_against_summaries()
            self._assert_reprocess_rebuild_sources_present(_reconciled)
        if route_delete_via_slurm:
            # ONE scoped reprocess-delete workflow handles BOTH the consolidated
            # zarr(s) AND (start_with=='process') the per-scenario processed/ dirs.
            self._workflow_builder.submit_reprocess_delete_workflow(
                start_with=start_with,
                override_in_flight=False,
            )
        self._invalidate_downstream_flags(
            start_with,
            regenerate_existing=regenerate_existing,
            dry_run=dry_run,
            skip_destructive_delete=route_delete_via_slurm,
        )

        # Force-rerun pre-delete (login-node responsibility). Resolve +
        # validate + delete BEFORE Snakemake plans the reprocess DAG. Per
        # cleanup-rerun-delete-redesign Phase 4 + R10. Skipped on dry_run —
        # it deletes flags and clears per-scenario processing-log records,
        # both filesystem mutations that the dry-run no-destructive-mutation
        # contract forbids.
        if not dry_run:
            self._apply_force_rerun(override_force_rerun)

        # Processed-output deletion (Phase 3). The per-model PROCESSING-LOG
        # clear (the _already_written invalidation, Gotcha #28) is CHEAP
        # (per-scenario JSON rewrites, no GPFS tree walk) and MUST run on
        # both routes: the SLURM runner deletes processed/ but never clears
        # log_{model_type}.json, so without this the rebuilt process rule
        # would emit but _already_written would skip every _export_* write.
        # The HEAVY processed/+zarr fast_rmtree stays SLURM-routed.
        if route_delete_via_slurm:
            # SLURM deleted the artifacts; clear only the per-model log here.
            if start_with == "process" and regenerate_existing and not dry_run:
                from hhemt.workflow import ResolvedForceRerunSpec

                self._invalidate_processing_log_for_force_rerun(ResolvedForceRerunSpec(scope="all", tokens=()))
        else:
            self._delete_processed_outputs_for_reprocess(
                start_with, regenerate_existing=regenerate_existing, dry_run=dry_run
            )

        # Delegate to the workflow builder. The submit method writes the
        # reprocess Snakefile and orchestrates the snakemake invocation with
        # `--snakefile Snakefile.reprocess --rerun-triggers mtime --nolock`
        # against the shared analysis_dir/.snakemake/; the
        # _status/_orchestrator/ liveness gate is the concurrency authority.
        result = self._workflow_builder.submit_reprocess_workflow(
            start_with=start_with,
            execution_mode=execution_mode,
            multi_sim_run_method_override=effective_method,
            dry_run=dry_run,
            verbose=verbose,
        )
        return result

    def delete(
        self,
        override_in_flight: bool = False,
        *,
        override_multi_sim_run_method: Literal["local", "batch_job", "1_job_many_srun_tasks"] | None = None,
    ) -> None:
        """Distributed Snakemake workflow that deletes the entire analysis_dir.

        Refuses by default when ``_status/_submitted/*.json`` sentinels
        indicate live SLURM jobs. Pass ``override_in_flight=True`` to bypass
        the guard.

        Mirrors the dispatch pattern of :meth:`reprocess` — sensitivity-toggled
        analyses dispatch to
        :meth:`TRITONSWMM_sensitivity_analysis.delete`.

        Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction
        + D-DeleteBoundary resolutions) and distributed-delete-and-du-recording
        Phase 3 (SLURM lift; ``override_multi_sim_run_method`` mirrors the
        run-mode override pattern from :meth:`submit_workflow`).
        """
        if self.cfg_analysis.toggle_sensitivity_analysis:
            return self.sensitivity.delete(
                override_in_flight=override_in_flight,
                override_multi_sim_run_method=override_multi_sim_run_method,
            )

        analysis_dir = self.analysis_paths.analysis_dir

        # 1. Clear any stale sentinels from a prior failed delete attempt.
        # Without this, the post-check at step 3 could falsely pass on a
        # half-completed previous delete and fast_rmtree a partially-deleted
        # tree.
        stale_dir = analysis_dir / "_status" / "_deleting"
        if stale_dir.exists():
            # EXEMPT-DU: status-dir-cleanup
            fast_rmtree(stale_dir)

        # 2. Submit the distributed delete workflow. The workflow builder's
        # _pre_delete_guards (live-sentinel refusal + scoped lock-check) runs
        # inside submit_delete_workflow; orchestrator does not invoke it
        # directly.
        self._workflow_builder.submit_delete_workflow(
            override_in_flight=override_in_flight,
            override_multi_sim_run_method=override_multi_sim_run_method,
        )

        # 3. Verify all expected sentinels present; remove analysis_dir atomically.
        expected = self._enumerate_expected_delete_sentinels()
        deleting_dir = analysis_dir / "_status" / "_deleting"
        actual = set(deleting_dir.glob("*.flag")) if deleting_dir.exists() else set()
        missing = expected - actual
        if missing:
            print(
                f"[delete] {len(missing)} per-rule sentinels missing — preserving analysis_dir for debugging.",
                flush=True,
            )
            print(f"[delete] missing: {sorted(p.name for p in missing)}", flush=True)
            return
        print(
            f"[delete] all {len(expected)} per-rule sentinels present — removing analysis_dir.",
            flush=True,
        )
        # EXEMPT-DU: full-analysis-root-wipe
        fast_rmtree(analysis_dir)

    def _enumerate_expected_delete_sentinels(self) -> set[Path]:
        """Compute the set of ``_status/_deleting/*.flag`` paths the delete
        workflow will produce on full success.

        One per scenario for regular analyses (sensitivity-master analyses
        delegate to :meth:`TRITONSWMM_sensitivity_analysis._enumerate_expected_delete_sentinels`
        before reaching this method); plus one for the consolidation rule.
        """
        from hhemt.scenario import compute_event_id_slug

        delete_dir = self.analysis_paths.analysis_dir / "_status" / "_deleting"
        expected = {delete_dir / "analysis_consolidation.flag"}
        for i in range(len(self.df_sims)):
            event_id = compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i))
            expected.add(delete_dir / f"scenario_evt-{event_id}.flag")
        return expected

    def _all_sim_flags_present(self) -> bool:
        """True iff every enabled sim's ``c_run_*`` completion flag exists.

        Used as the *flag-presence* component of the ``override_clear_raw``
        guard; the sentinel-presence component (in-flight detection) is
        checked separately at the :meth:`reprocess` call site.

        Enumeration contract
        --------------------
        For a non-sensitivity analysis: for each enabled model_type (from
        cfg_system's ``toggle_*_model`` fields) and each event_id in the
        analysis's event set, expect
        ``{_status}/c_run_{model_type}_evt-{event_id}.flag``.

        For a sensitivity master analysis: recurse into each sub-analysis's
        ``_status/`` directory and check
        ``c_run_{model_type}_sa-{sa_id}_evt-{event_id}.flag`` for every
        (sa_id, event_id, enabled_model_type) tuple.

        Returns True only if every expected flag exists. Missing → False.
        Does NOT consult ``_submitted/`` sentinels; that signal is the
        in-flight guard layered on top of this method at the
        :meth:`reprocess` call site.
        """
        from hhemt.scenario import compute_event_id_slug

        cfg_sys = self._system.cfg_system
        enabled_models: list[str] = []
        if cfg_sys.toggle_triton_model:
            enabled_models.append("triton")
        if cfg_sys.toggle_tritonswmm_model:
            enabled_models.append("tritonswmm")
        if cfg_sys.toggle_swmm_model:
            enabled_models.append("swmm")
        if not enabled_models:
            return False  # No models enabled — nothing to attest.

        # Non-sensitivity path (sensitivity paths handled by
        # TRITONSWMM_sensitivity_analysis.reprocess in Phase 3).
        if getattr(self.cfg_analysis, "toggle_sensitivity_analysis", False):
            # Sensitivity master analyses are out of scope for Phase 2's
            # reprocess; the sensitivity-master reprocess is Phase 3. Until
            # then, conservatively return False so the guard short-circuits
            # rather than admitting a false-positive.
            return False

        status_dir = self.analysis_paths.analysis_dir / "_status"
        for i in range(len(self.df_sims)):
            event_id = compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i))
            for model in enabled_models:
                flag = status_dir / f"c_run_{model}_evt-{event_id}_complete.flag"
                if not flag.exists():
                    return False
        return True

    def _invalidate_downstream_flags(
        self,
        start_with: str,
        *,
        regenerate_existing: bool = False,
        dry_run: bool = False,
        skip_destructive_delete: bool = False,
    ) -> None:
        """Delete ``_status`` flags / artifacts from ``start_with`` onward.

        When ``regenerate_existing`` is False (default), the process/consolidate
        arms PRESERVE the consolidate flag AND the consolidated zarr (completion
        is signalled by flag + log per D5, not ``.exists()``), and re-fire ONLY
        the report + plot rules by deleting their artifacts — so the GPFS DU
        restamp walk never runs. When True, the legacy destructive rebuild path
        runs (delete consolidate flag + zarr). The report always regenerates.
        Never deletes ``c_run_*`` (sim) flags. (Phase 2 — FQ1 Option A.)
        """
        from hhemt.du_sentinels import restamp_parent_sentinels

        analysis_dir = self.analysis_paths.analysis_dir
        sd = analysis_dir / "_status"

        def _delete_report_and_plot_artifacts() -> None:
            """Re-fire render_report + plot rules by deleting their outputs.

            Under --rerun-triggers mtime, an absent output is the only reliable
            re-fire trigger (INPUT/CODE triggers are off). Deletes the report
            shell and the report-feeding plot artifacts.
            """
            report_html = analysis_dir / "analysis_report.html"
            report_zip = analysis_dir / "analysis_report.zip"
            # D3 — capture deleted-artifact sizes BEFORE unlink so the O(1)
            # decrement has the bytes to subtract (post-unlink stat is impossible).
            _html_bytes = report_html.stat().st_size if report_html.exists() else 0
            _zip_bytes = report_zip.stat().st_size if report_zip.exists() else 0
            # EXEMPT-DU: du-handled-by-decrement
            report_html.unlink(missing_ok=True)
            # EXEMPT-DU: du-handled-by-decrement
            report_zip.unlink(missing_ok=True)
            plots_dir = analysis_dir / "plots"
            plots_total_bytes = 0
            if plots_dir.exists():
                for _art in plots_dir.rglob("*"):
                    if _art.is_file():
                        try:
                            plots_total_bytes += _art.stat().st_size
                        except OSError:
                            pass
                for art in plots_dir.rglob("*"):
                    if art.is_file():
                        # EXEMPT-DU: du-handled-by-decrement
                        art.unlink(missing_ok=True)
            if not dry_run:
                # PATTERN B replaced by D3 — O(1)/O(plots) decrement instead of a
                # full-tree walk. FIX 3: on the regenerate_existing
                # process/consolidate arms a LATER zarr deletion restamps the tree
                # anyway, so skip the redundant decrement there (the default
                # regenerate_existing=False path decrements). Sizes captured above
                # BEFORE unlink (post-unlink stat is impossible); routes through
                # write_du_sentinel so the compare-and-write mtime invariant holds.
                if not (start_with in ("process", "consolidate") and regenerate_existing):
                    from hhemt.du_sentinels import decrement_scope_sentinel

                    child_deltas: dict[str, int] = {}
                    if _html_bytes:
                        child_deltas["analysis_report.html"] = _html_bytes
                    if _zip_bytes:
                        child_deltas["analysis_report.zip"] = _zip_bytes
                    if plots_total_bytes:
                        child_deltas["plots"] = plots_total_bytes
                    if child_deltas:
                        decrement_scope_sentinel(analysis_dir, scope="analysis", child_deltas=child_deltas)

        if start_with == "process":
            if regenerate_existing:
                for f in sd.glob("d_process_*"):
                    # EXEMPT-DU: status-flag
                    f.unlink(missing_ok=True)
                # EXEMPT-DU: status-flag
                (sd / "e_consolidate_complete.flag").unlink(missing_ok=True)
                if not dry_run and not skip_destructive_delete:
                    _zarr = self.analysis_paths.analysis_datatree_zarr
                    if _zarr is not None and _zarr.exists():
                        fast_rmtree(_zarr, analysis_dir=analysis_dir)  # PATTERN A
            _delete_report_and_plot_artifacts()
        elif start_with == "consolidate":
            if regenerate_existing:
                # EXEMPT-DU: status-flag
                (sd / "e_consolidate_complete.flag").unlink(missing_ok=True)
                if not dry_run and not skip_destructive_delete:
                    _zarr = self.analysis_paths.analysis_datatree_zarr
                    if _zarr is not None and _zarr.exists():
                        fast_rmtree(_zarr, analysis_dir=analysis_dir)  # PATTERN A
            # regenerate_existing=False: leave consolidate flag AND zarr intact
            # (the flag IS the completion signal per D5); only report+plots re-fire.
            _delete_report_and_plot_artifacts()
        elif start_with == "render":
            # Unchanged: render arm deletes report artifacts only, never the zarr
            # (plots intentionally left in place — render is the surgical
            # "report shell only" path).
            report_html = analysis_dir / "analysis_report.html"
            report_zip = analysis_dir / "analysis_report.zip"
            # EXEMPT-DU: du-handled-by-decrement
            report_html.unlink(missing_ok=True)
            # EXEMPT-DU: du-handled-by-decrement
            report_zip.unlink(missing_ok=True)
            if not dry_run:
                restamp_parent_sentinels(report_html, analysis_dir=analysis_dir)  # PATTERN B
        else:
            raise ValueError(f"start_with must be one of 'process', 'consolidate', 'render'; got {start_with!r}")

    def _delete_processed_outputs_for_reprocess(
        self, start_with: str, *, regenerate_existing: bool, dry_run: bool = False
    ) -> None:
        """Delete per-scenario PROCESSED outputs for an opt-in rebuild-from-raw.

        Invoked from ``reprocess`` ONLY when ``start_with == "process"`` AND
        ``regenerate_existing`` is True. Distinct from ``override_force_rerun``
        (which OVERWRITES processed zarrs in place by clearing the log so
        ``_already_written`` returns False, but never deletes the artifact /
        frees disk). This DELETES so a rebuild-from-raw is clean — required
        because existing zarrs are suspected to carry bugged simulation-duration
        calculations. Reuses ``_invalidate_processing_log_for_force_rerun`` for
        the log half. (Phase 3 — FQ1 single-dir + FQ2.)
        """
        if not (start_with == "process" and regenerate_existing):
            return
        from hhemt.scenario import TRITONSWMM_scenario
        from hhemt.workflow import ResolvedForceRerunSpec

        # 1) Clear per-scenario per-model processing_log.outputs so the runner's
        #    _already_written gate returns False and write paths re-execute.
        #    Scope "all" — a process-stage reprocess rebuilds every scenario.
        self._invalidate_processing_log_for_force_rerun(ResolvedForceRerunSpec(scope="all", tokens=()))
        if dry_run:
            return  # dry-run performs no destructive filesystem mutation

        # 2) Delete the per-scenario PROCESSED artifacts on disk. ALL processed
        #    outputs (summaries + timeseries, every model family) live under one
        #    directory: sims/{event_id}/processed/ (scenario.py:62). The raw
        #    out_triton/out_tritonswmm/out_swmm binaries are SIBLINGS under
        #    sim_folder (scenario.py:76-80), NOT under processed/, so deleting
        #    processed/ preserves the rebuild source. Single-dir deletion is
        #    drift-proof (no ScenarioPaths attr-name maintenance — the prior
        #    16-attr tuple had wrong names) and is exactly the granularity R8's
        #    SLURM-offload wraps.
        analysis_dir = self.analysis_paths.analysis_dir
        for event_iloc in range(len(self.df_sims)):
            scen = TRITONSWMM_scenario(event_iloc, self)
            processed_dir = scen.scen_paths.sim_folder / "processed"
            if processed_dir.exists():
                fast_rmtree(processed_dir, analysis_dir=analysis_dir)  # PATTERN A
        # The consolidated zarr is deleted by _invalidate_downstream_flags'
        # regenerate_existing=True process-arm — no duplicate deletion here.

    def _validate_force_rerun_targets(self, resolved_force_rerun) -> None:
        """Validate that requested ``sa_id`` / ``event_iloc`` values exist in the analysis.

        Per cleanup-rerun-delete-redesign Phase 4, R11 + D-ForceRerunValidatesSaId
        Option 1 (hard error at API entry). Unknown values raise
        ``ConfigurationError`` before any filesystem touch.
        """
        from .exceptions import ConfigurationError

        if resolved_force_rerun in ("all", "none"):
            return
        if not isinstance(resolved_force_rerun, dict):
            raise ValueError(f"Unexpected force_rerun shape: {resolved_force_rerun!r}")
        key = next(iter(resolved_force_rerun))
        # Cross-check against toggle_sensitivity_analysis — mirrors the cfg-load
        # validator in config/analysis.py but applies to the override path too.
        if key == "sa_id" and not self.cfg_analysis.toggle_sensitivity_analysis:
            raise ConfigurationError(
                field="override_force_rerun",
                message=("override_force_rerun.sa_id requires toggle_sensitivity_analysis=True"),
            )
        if key == "event_iloc" and self.cfg_analysis.toggle_sensitivity_analysis:
            raise ConfigurationError(
                field="override_force_rerun",
                message=(
                    "override_force_rerun.event_iloc requires toggle_sensitivity_analysis=False; "
                    "sensitivity-toggled analyses must use override_force_rerun.sa_id instead"
                ),
            )
        requested = set(map(str, resolved_force_rerun[key]))
        if key == "sa_id":
            known = set(self.sensitivity.df_setup.index.astype(str))
        else:  # event_iloc
            known = set(map(str, self.df_sims.index))
        unknown = requested - known
        if unknown:
            raise ConfigurationError(
                field="override_force_rerun",
                message=(
                    f"override_force_rerun.{key} contains unknown values: "
                    f"{sorted(unknown)}. Known {key} values: {sorted(known)}."
                ),
            )

    def _build_force_rerun_spec(self, resolved_force_rerun):
        """Resolve a ``ForceRerunValue`` into a ``ResolvedForceRerunSpec``.

        For the ``event_iloc`` scope, resolves event_iloc integers to event_id
        slugs via ``compute_event_id_slug`` (V0001's stable event-slug
        invariant); the builder helper consumes only slugs/sa_ids.
        """
        from hhemt.scenario import compute_event_id_slug
        from hhemt.workflow import ResolvedForceRerunSpec

        if resolved_force_rerun == "all":
            return ResolvedForceRerunSpec(scope="all", tokens=())
        if resolved_force_rerun == "none":
            return ResolvedForceRerunSpec(scope="none", tokens=())
        assert isinstance(resolved_force_rerun, dict)
        key = next(iter(resolved_force_rerun))
        values = resolved_force_rerun[key]
        if key == "sa_id":
            return ResolvedForceRerunSpec(scope="sa", tokens=tuple(str(v) for v in values))
        # event_iloc → event_id slug per V0001 stable slug invariant.
        slugs = tuple(
            compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(int(iloc))) for iloc in values
        )
        return ResolvedForceRerunSpec(scope="event", tokens=slugs)

    def _apply_force_rerun(self, override_force_rerun) -> None:
        """Resolve, validate, and pre-delete flags + per-scenario log records
        for the force-rerun override.

        Called at workflow-submission boundaries (run / submit_workflow /
        reprocess / submit_reprocess_workflow). Pre-delete happens on the
        login node BEFORE Snakemake plans the DAG so MTIME-input triggers see
        the deleted flags and cascade re-fire automatically. Per cleanup-
        rerun-delete-redesign Phase 4 + R10.

        Two-layer invalidation per the FQ0 trace (post-Phase-4):

        1. ``_delete_flags_for_force_rerun(spec)`` removes ``_status/*.flag``
           markers so Snakemake re-plans the affected rules.
        2. ``_invalidate_processing_log_for_force_rerun(spec)`` clears the
           per-scenario per-model log ``processing_log.outputs`` so each
           runner subprocess's ``_already_written`` gate returns False and
           the write paths actually re-execute. Without (2), step (1) alone
           produces fresh flags but stale outputs.
        """
        resolved = override_force_rerun if override_force_rerun is not None else self.cfg_analysis.force_rerun
        self._validate_force_rerun_targets(resolved)
        spec = self._build_force_rerun_spec(resolved)
        self._workflow_builder._delete_flags_for_force_rerun(spec)
        self._invalidate_processing_log_for_force_rerun(spec)

    def _invalidate_consolidate_flag_on_scenario_set_change(self) -> None:
        """Delete e_consolidate_complete.flag (and orphan per-event flags) when the
        multi_sim scenario set changed since the last prepared run.

        Under the toolkit's --rerun-triggers mtime profile, a present
        e_consolidate_complete.flag prevents Snakemake from re-demanding an ADDED
        scenario's upstream chain (missing-intermediate is treated as a consumed temp).
        Deleting it forces consolidate to re-evaluate its expand(...) over the new
        SIM_IDS, pulling in the added scenario's prepare/run/process rules. Orphan
        per-event flags for removed events are deleted so their stale chain is not
        re-consolidated. No-op when the set is unchanged.
        """
        from hhemt.scenario import compute_event_id_slug

        status_dir = self.analysis_paths.analysis_dir / "_status"
        if not status_dir.exists():
            return
        # Current config event-id set (same enumeration the Snakefile SIM_IDS uses).
        n_sims = len(self.df_sims)
        config_event_ids = {
            compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i)) for i in range(n_sims)
        }
        # On-disk prepared event-id set (from b_prepare flags).
        prepared_event_ids = {
            p.name[len("b_prepare_evt-") : -len("_complete.flag")]
            for p in status_dir.glob("b_prepare_evt-*_complete.flag")
        }
        if config_event_ids == prepared_event_ids:
            return  # set unchanged — no invalidation needed
        # Set changed: drop the analysis-level consolidate flag so the added chain is
        # re-demanded; drop orphan per-event flags for removed events.
        (status_dir / "e_consolidate_complete.flag").unlink(missing_ok=True)  # EXEMPT-DU: status-flag
        (status_dir / "e_consolidate_complete.flag.json").unlink(missing_ok=True)  # EXEMPT-DU: status-flag
        removed = prepared_event_ids - config_event_ids
        for ev in removed:
            for stem in (
                f"b_prepare_evt-{ev}_complete",
                f"f_consolidate_scenario_evt-{ev}_complete",
            ):
                (status_dir / f"{stem}.flag").unlink(missing_ok=True)  # EXEMPT-DU: status-flag
                (status_dir / f"{stem}.flag.json").unlink(missing_ok=True)  # EXEMPT-DU: status-flag
            for model_type in ("triton", "tritonswmm", "swmm"):
                for phase in ("c_run", "d_process"):
                    stem = f"{phase}_{model_type}_evt-{ev}_complete"
                    (status_dir / f"{stem}.flag").unlink(missing_ok=True)  # EXEMPT-DU: status-flag
                    (status_dir / f"{stem}.flag.json").unlink(missing_ok=True)  # EXEMPT-DU: status-flag

    def _reconcile_stale_process_flags_against_summaries(
        self, *, sa_id: str | None = None, master_dir: Path | None = None
    ) -> set[tuple[str, str]]:
        """Self-heal the reprocess divergence state (FIX 2).

        For each (event_id, model_type) in THIS analysis whose ``d_process``
        completion flag exists but whose consolidate-consumed summary outputs
        are absent on disk, delete the flag AND clear the per-model
        ``processing_log.outputs`` so:

        1. the reprocess generator's emit gate (workflow.py:6684,
           ``start_with=='process' and not d_process_path.exists()``) re-emits
           the process rule, and
        2. the runner's ``_already_written`` gate (process_simulation.py:1175,
           keyed on ``processing_log.outputs[...].success`` — NOT ``.exists()``,
           Gotcha 28) returns False so ``_export_*`` actually re-writes.

        Fires UNCONDITIONALLY on the process path regardless of
        ``regenerate_existing`` (D2). Existence-keyed (D3): no-op for any
        (evt, model) whose enabled summary set is fully present, so it is a
        provable no-op on a healthy analysis and cannot delete a present output.

        Works for both non-sensitivity analyses and sub-analyses (sub-analyses
        are full Analysis instances per Gotcha 11). For a sub-analysis the
        sensitivity master passes the BARE ``sa_id`` AND ``master_dir`` (the
        master analysis_dir, whose ``_status/`` holds the per-sa flags); both
        are required because a sub-analysis's own ``analysis_id`` is the prefixed
        ``sa_{bare}`` (sensitivity_analysis.py:1607) and its own ``analysis_dir``
        is ``subanalyses/sa_X/`` (sensitivity_analysis.py:50) — neither matches
        the gate's bare-``sa_id`` flag under the master ``_status/``
        (workflow.py:6652/6678; sensitivity_analysis.py:480-498). Non-sensitivity
        callers pass neither (flags live in this analysis's own ``_status/``).
        The summary set is never narrowed.

        Returns
        -------
        set[tuple[str, str]]
            The (event_id, model_type) pairs reconciled (flag+log cleared).
            Empty on a healthy analysis.
        """
        # Flag-name helpers live in constants, NOT workflow. There is a per-sa
        # helper (process_timeseries_flag_per_sa) but NO non-sa variant — the
        # non-sensitivity d_process flag is built inline by workflow.py:1350 as
        # "_status/d_process_{model_type}_evt-{event_id}_complete.flag". Verified
        # against constants.py:179 + workflow.py:1350 (2026-06-01).
        from hhemt.constants import (
            STATUS_DIR_NAME,
            process_timeseries_flag_per_sa,
        )
        from hhemt.scenario import (
            TRITONSWMM_scenario,
            compute_event_id_slug,
        )
        from hhemt.workflow import ResolvedForceRerunSpec

        _SUMMARY_ATTRS_BY_MODEL = {
            "tritonswmm": (
                "output_tritonswmm_triton_summary",
                "output_tritonswmm_node_summary",
                "output_tritonswmm_link_summary",
                "output_tritonswmm_performance_summary",
            ),
            "triton": (
                "output_triton_only_summary",
                "output_triton_only_performance_summary",
            ),
            "swmm": (
                "output_swmm_only_node_summary",
                "output_swmm_only_link_summary",
            ),
        }

        def _summary_absent(scen, model_type: str) -> bool:
            for attr in _SUMMARY_ATTRS_BY_MODEL.get(model_type, ()):
                p = getattr(scen.scen_paths, attr, None)
                if p is None:
                    continue
                if not p.exists():
                    return True
            return False

        # sub-analyses: sa_id (BARE) and master_dir (the MASTER analysis_dir,
        # whose _status/ holds the per-sa flags — sensitivity_analysis.py:480-498)
        # are threaded from the sensitivity master loop. A sub-analysis's OWN
        # analysis_dir is subanalyses/sa_X/ (sensitivity_analysis.py:50), which
        # does NOT hold the per-sa flags, and its cfg_analysis.analysis_id is the
        # PREFIXED "sa_{bare}" (sensitivity_analysis.py:1607) — deriving the flag
        # path from either would miss (wrong dir and/or doubled "sa-sa_" token),
        # silently breaking the rebuild. None/None => non-sensitivity: flags live
        # in THIS analysis's own _status/.
        assert (sa_id is None) == (master_dir is None), (
            "sa_id and master_dir must be passed together (sensitivity) or both omitted (non-sensitivity)"
        )
        is_sub = sa_id is not None

        reconciled: set[tuple[str, str]] = set()
        reconciled_event_ids: set[str] = set()
        for event_iloc in range(len(self.df_sims)):
            scen = TRITONSWMM_scenario(event_iloc, self)
            evt = compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(event_iloc))
            for model_type in scen.run.model_types_enabled:
                # Flag name shape MUST match the generator gate's token shape
                # (workflow.py:6678 for sub-analyses; the non-sa flag for
                # non-sensitivity) so the unlink hits exactly the flag the gate
                # checks.
                if is_sub:
                    # Per-sa flags live under the MASTER analysis_dir's _status/
                    # (NOT the sub's own). process_timeseries_flag_per_sa returns
                    # a "_status/"-prefixed rel path keyed by the BARE sa_id, so
                    # the base must be master_dir.
                    # Narrow the Optionals for the type checker; the pre-loop
                    # assert guarantees sa_id and master_dir are set together on
                    # the sub-analysis path (is_sub is True iff sa_id is not None).
                    assert sa_id is not None and master_dir is not None
                    flag_rel = process_timeseries_flag_per_sa(model_type, sa_id, evt)
                    flag_path = master_dir / flag_rel
                else:
                    # No non-sa helper exists; mirror workflow.py:1350 inline.
                    # Non-sensitivity flags live in this analysis's own _status/.
                    flag_rel = f"{STATUS_DIR_NAME}/d_process_{model_type}_evt-{evt}_complete.flag"
                    flag_path = self.analysis_paths.analysis_dir / flag_rel
                if not flag_path.exists():
                    continue  # gate already open for this pair — nothing to heal
                if not _summary_absent(scen, model_type):
                    continue  # summary present — healthy, no-op
                # Divergence: flag present, summary absent → heal.
                # EXEMPT-DU: status-flag
                flag_path.unlink(missing_ok=True)
                reconciled.add((evt, model_type))
                reconciled_event_ids.add(evt)

        if reconciled_event_ids:
            # Clear per-model processing_log for the reconciled events so
            # _already_written returns False on the re-emitted rule. Reuse the
            # event-scoped force-rerun log invalidator (cheap per-scenario JSON
            # rewrites; no GPFS tree walk).
            self._invalidate_processing_log_for_force_rerun(
                ResolvedForceRerunSpec(scope="event", tokens=tuple(reconciled_event_ids))
            )
        return reconciled

    def _assert_reprocess_rebuild_sources_present(self, reconciled: set[tuple[str, str]]) -> None:
        """Fail-fast (Option B) — for each (event_id, model_type) the process-path
        self-heal just reconciled (summary absent → flag+log cleared so the rule
        re-emits), verify the RAW rebuild source the summary aggregation actually
        consumes is present: for triton/tritonswmm the top-level raw dir
        (out_triton / out_tritonswmm with its H/QX/QY/MH binaries) AND the
        per-checkpoint ``out_{model}/performance`` subdir (R9 — a clear_raw'd-then-
        reprocess can strip ``performance/`` while leaving the top-level dir
        non-empty); for swmm the swmm_full_out_file. If a consumed source is gone,
        the re-emitted process rule would re-fail deep inside a SLURM job with an
        opaque FileNotFoundError; raise a clear login-node ConfigurationError
        instead. No-op when `reconciled` is empty (the healthy-analysis case)."""
        if not reconciled:
            return
        from hhemt.exceptions import ConfigurationError
        from hhemt.scenario import TRITONSWMM_scenario, compute_event_id_slug

        # triton/tritonswmm raw outputs are DIRECTORIES (ScenarioPaths.out_triton
        # @paths.py:107, .out_tritonswmm @108). swmm raw outputs are FILES
        # (swmm_full_out_file @93 .out binary, swmm_full_rpt_file @92 .rpt) —
        # there is NO ScenarioPaths.out_swmm. Branch the raw-source check on
        # whether the family writes a dir or a file.
        _raw_dir_attr = {
            "triton": "out_triton",
            "tritonswmm": "out_tritonswmm",
        }

        def _raw_source_present(scen, model_type: str) -> bool:
            """True iff the raw rebuild source for this model family is present.
            Directory model for triton/tritonswmm; file model for swmm."""
            if model_type in _raw_dir_attr:
                raw_dir = getattr(scen.scen_paths, _raw_dir_attr[model_type], None)
                if raw_dir is None or not raw_dir.exists() or not any(raw_dir.iterdir()):
                    return False
                # (R9) The summary aggregation consumes the per-checkpoint
                # performance{N}.txt set under out_{model}/performance (the V0008
                # groupby(level='Rank').diff() over the merged checkpoint set —
                # process_simulation._export_performance_tseries ->
                # _aggregate_perf_tseries, which raises if the performance dir is
                # absent; see the `clear raw triton outputs deferred until last
                # allocation` stipulation). A clear_raw'd-then-reprocess can strip
                # performance/ while leaving the top-level raw H/QX/QY/MH dir
                # non-empty, so check the actually-consumed subdir too — fail fast
                # at the login node instead of deep in a SLURM rebuild.
                perf_dir = raw_dir / "performance"
                return perf_dir.exists() and any(perf_dir.iterdir())
            if model_type == "swmm":
                out_file = getattr(scen.scen_paths, "swmm_full_out_file", None)
                return out_file is not None and out_file.exists()
            return False

        missing_sources: list[str] = []
        for event_iloc in range(len(self.df_sims)):
            scen = TRITONSWMM_scenario(event_iloc, self)
            evt = compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(event_iloc))
            for model_type in scen.run.model_types_enabled:
                if (evt, model_type) not in reconciled:
                    continue
                if not _raw_source_present(scen, model_type):
                    missing_sources.append(f"{model_type}@evt-{evt}")
        if missing_sources:
            raise ConfigurationError(
                field="start_with",
                message=(
                    "reprocess(start_with='process') cannot rebuild missing summaries "
                    f"for {sorted(missing_sources)} — the raw simulation output "
                    "(out_triton/out_tritonswmm/out_swmm) is also absent, so there is "
                    "no rebuild source. Re-run the simulations for these scenarios "
                    "before reprocessing, or restore the raw outputs."
                ),
                config_path=str(self.analysis_config_yaml),
            )

    def _invalidate_processing_log_for_force_rerun(self, spec: "ResolvedForceRerunSpec") -> None:
        """Invalidate per-scenario log ``processing_log.outputs`` entries
        that match the force-rerun spec.

        Per cleanup-rerun-delete-redesign Phase 4 + B-mechanism: flag
        deletion triggers Snakemake to re-plan the DAG, but the runner
        subprocess's ``process_simulation.py::_already_written`` gate
        consults each per-model log's ``processing_log.outputs`` dict, NOT
        the flag files. Without log-record invalidation, the re-fired rule
        subprocess executes ``write_timeseries_outputs(...)`` but every
        internal ``_export_*`` early-returns on ``_already_written``
        because the log still records ``success=True`` from the prior run
        — net result is fresh flags but stale zarrs. This helper closes
        that gap: for each scope matched by ``spec``, it clears the
        corresponding per-model log files' ``processing_log.outputs`` (so
        the next runner pass writes the outputs fresh) and resets the
        ``raw_*_outputs_cleared`` markers so the clear-raw step re-runs.

        For the ``"all"`` scope: invalidates every scenario's per-model
        log. For ``"sa"`` scope: dispatches via
        ``sensitivity._invalidate_processing_log_for_sa_ids``. For
        ``"event"`` scope: invalidates only the scenarios whose
        ``event_id`` slug matches the tokens.

        On-disk side effect: the per-scenario log JSON file
        (``log_tritonswmm.json`` / ``log_triton.json`` / ``log_swmm.json``)
        is rewritten with ``processing_log.outputs = {}``. No zarr/nc
        artifact is touched — the runner's ``_write_output`` path
        overwrites the existing zarr on re-execution.
        """
        if spec.scope == "none":
            return

        if spec.scope == "sa":
            # Sensitivity dispatch — sub-analyses own their scenarios.
            self.sensitivity._invalidate_processing_log_for_sa_ids(spec.tokens)
            return

        if spec.scope == "all":
            target_event_ids = set(self._all_event_id_slugs())
        elif spec.scope == "event":
            target_event_ids = set(spec.tokens)
        else:
            raise ValueError(f"Unrecognized spec.scope: {spec.scope!r}")

        for event_iloc in range(len(self.df_sims)):
            scen = TRITONSWMM_scenario(event_iloc, self)
            if scen.event_id not in target_event_ids:
                continue
            for model_type in scen.run.model_types_enabled:
                model_log = scen.get_log(model_type)
                # Clear the processing_log dict and persist.
                model_log.processing_log.outputs.clear()
                # Also reset raw-outputs-cleared markers so the next
                # processing pass re-runs the clear_raw step on top of
                # the re-written outputs.
                if model_log.raw_TRITON_outputs_cleared is not None:
                    model_log.raw_TRITON_outputs_cleared.set(False)
                if model_log.raw_SWMM_outputs_cleared is not None:
                    model_log.raw_SWMM_outputs_cleared.set(False)
                model_log.write()

    def _all_event_id_slugs(self) -> list[str]:
        """Helper: enumerate every scenario's event_id slug for ``"all"`` scope.

        Uses the same V0001 stable-slug derivation as ``_build_force_rerun_spec``
        — no scenarios attribute exists on Analysis (per Phase 2 audit row), so
        slugs are computed from df_sims index via
        ``compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i))``.
        """
        from hhemt.scenario import compute_event_id_slug

        return [
            compute_event_id_slug(self._retrieve_weather_indexer_using_integer_index(i))
            for i in range(len(self.df_sims))
        ]

    # TODO - fix or delete
    # @property
    # def TRITONSWMM_runtimes(self):
    #     return (
    #         self._tritonswmm_TRITON_summary["compute_time_min"]
    #         .to_dataframe()
    #         .dropna()["compute_time_min"]
    #     )

    @property
    def _tritonswmm_performance_analysis_summary_created(self):
        return bool(self.log.tritonswmm_performance_analysis_summary_created.get())

    @property
    def _tritonswmm_triton_analysis_summary_created(self):
        return bool(self.log.tritonswmm_triton_analysis_summary_created.get())

    @property
    def _tritonswmm_node_analysis_summary_created(self):
        return bool(self.log.tritonswmm_node_analysis_summary_created.get())

    @property
    def _tritonswmm_link_analysis_summary_created(self):
        return bool(self.log.tritonswmm_link_analysis_summary_created.get())

    @property
    def _triton_only_analysis_summary_created(self):
        return bool(self.log.triton_only_analysis_summary_created.get())

    @property
    def _swmm_only_node_analysis_summary_created(self):
        return bool(self.log.swmm_only_node_analysis_summary_created.get())

    @property
    def _swmm_only_link_analysis_summary_created(self):
        return bool(self.log.swmm_only_link_analysis_summary_created.get())

    @property
    def _df_snakemake_allocations(self) -> pd.DataFrame:
        enabled_models_untyped = self._get_enabled_model_types()
        enabled_models: list[Literal["triton", "tritonswmm", "swmm"]] = [
            m for m in ("triton", "tritonswmm", "swmm") if m in enabled_models_untyped
        ]

        if self.cfg_analysis.toggle_sensitivity_analysis:
            snakefile_path = self.analysis_paths.analysis_dir / "Snakefile"
            expected_sa_ids = sorted(self.sensitivity.sub_analyses.keys())
            sa_allocations = parse_sensitivity_analysis_workflow_model_allocations(
                snakefile_path=snakefile_path,
                expected_subanalysis_ids=expected_sa_ids,
                strict=False,
            )
            rows: list[dict] = []
            for sa_id, sub_analysis in self.sensitivity.sub_analyses.items():
                if sa_id not in sa_allocations:
                    # Sub-analysis set up on disk but absent from the Snakefile's
                    # simulation_sa_* rules (expected — see bug plan D-b). Emit no
                    # allocation rows here; df_status's left-merge leaves NaN allocation
                    # columns which df_status annotates with the parse-error string (R5).
                    continue
                alloc = sa_allocations[sa_id]
                for event_iloc in sub_analysis.df_sims.index:
                    scen = TRITONSWMM_scenario(event_iloc, sub_analysis)
                    scen.log.refresh()
                    scenario_dir = str(scen.log.logfile.parent)
                    for model_type in enabled_models:
                        row = {
                            "event_iloc": event_iloc,
                            "model_type": model_type,
                            "scenario_directory": scenario_dir,
                            "snakemake_allocation_parse_error": None,
                        }
                        row.update(alloc)
                        rows.append(row)
            return pd.DataFrame(rows)

        model_allocations, parse_error = self._retrieve_snakemake_allocations()
        rows = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            scen.log.refresh()
            scenario_dir = str(scen.log.logfile.parent)

            for model_type in enabled_models:
                row = {
                    "event_iloc": event_iloc,
                    "model_type": model_type,
                    "scenario_directory": scenario_dir,
                    "snakemake_allocation_parse_error": parse_error,
                }

                if model_type not in model_allocations:
                    # parse_error already set on the row above; leave allocation
                    # columns absent (NaN after DataFrame construction) so the
                    # regular branch is tolerant (R2).
                    rows.append(row)
                    continue

                alloc = model_allocations[model_type]
                row.update(alloc)
                rows.append(row)
        return pd.DataFrame(rows)

    def _get_performance_summary_row(
        self,
        event_iloc: int,
        model_type: Literal["triton", "tritonswmm", "swmm"],
    ) -> dict[str, float | None]:
        """
        Extract per-category timing totals from the performance summary dataset for one scenario.

        Returns a dict keyed by ``perf_<VarName>`` for all variables in PERF_VARS.
        Values are ``None`` for SWMM rows (no TRITON performance dataset) and for
        TRITON/TRITONSWMM rows where the performance summary has not been written yet.

        Parameters
        ----------
        event_iloc : int
            The event index for the scenario.
        model_type : Literal["triton", "tritonswmm", "swmm"]
            The model type for this row.

        Returns
        -------
        dict[str, float | None]
            Keyed by ``perf_<VarName>`` for each variable in PERF_VARS.
        """
        null_row: dict[str, float | None] = {f"perf_{v}": None for v in PERF_VARS}

        if model_type == "swmm":
            return null_row

        # Gate on log flag — performance summary only written when processing completes
        scen = TRITONSWMM_scenario(event_iloc, self)
        model_log = scen.get_log(model_type)
        if model_log.performance_summary_written is None:
            return null_row
        if model_log.performance_summary_written.get() is not True:
            return null_row

        proc = self._retrieve_sim_run_processing_object(event_iloc)
        if model_type == "tritonswmm":
            ds = proc.TRITONSWMM_performance_summary
        else:  # triton
            ds = proc.TRITON_only_performance_summary
        return {f"perf_{v}": float(ds[v].values.item()) for v in PERF_VARS}

    @staticmethod
    def _reorder_df_status_columns(df: pd.DataFrame) -> pd.DataFrame:
        """
        Reorder df_status columns into a canonical reader-friendly layout.

        Groups: identity/status → weather/setup → sensitivity params →
        performance breakdown → expected resources → actual resources → snakemake alloc.

        Any columns present in the DataFrame but not in the canonical list are
        appended at the end so no data is silently dropped.

        Parameters
        ----------
        df : pd.DataFrame
            The raw df_status DataFrame.

        Returns
        -------
        pd.DataFrame
            DataFrame with columns in canonical order.
        """
        fixed_identity = [
            "sa_id",
            "sub_analysis_iloc",
            "event_iloc",
            "model_type",
            "scenario_setup",
            "run_completed",
            "n_resumes",
            "scenario_directory",
        ]
        fixed_perf = [f"perf_{v}" for v in PERF_VARS_ORDERED]
        fixed_resources = [
            "run_mode",
            "n_mpi_procs",
            "n_omp_threads",
            "n_gpus",
            "n_nodes",
            "backend_used",
        ]
        fixed_actual = [
            "actual_nTasks",
            "actual_omp_threads",
            "actual_gpus",
            "actual_total_gpus",
            "actual_gpu_backend",
            "actual_build_type",
        ]
        fixed_snakemake = [
            "snakemake_allocated_nTasks",
            "snakemake_allocated_omp_threads",
            "snakemake_allocated_total_cpus",
            "snakemake_allocation_parse_error",
        ]
        all_fixed = fixed_identity + fixed_perf + fixed_resources + fixed_actual + fixed_snakemake
        # Columns not in any fixed group are weather/setup or sensitivity params —
        # place them between identity and performance (groups 2 & 3).
        dynamic_cols = [c for c in df.columns if c not in all_fixed]
        ordered = [c for c in fixed_identity if c in df.columns]
        ordered += dynamic_cols
        ordered += [c for c in fixed_perf if c in df.columns]
        ordered += [c for c in fixed_resources if c in df.columns]
        ordered += [c for c in fixed_actual if c in df.columns]
        ordered += [c for c in fixed_snakemake if c in df.columns]
        # Append any unexpected columns that slipped through
        ordered += [c for c in df.columns if c not in ordered]
        return df[ordered]

    @property
    def disk_utilization_bytes(self) -> int | None:
        """Return the analysis-level DU sentinel value, or None if absent."""
        from hhemt.du_sentinels import read_du_sentinel

        payload = read_du_sentinel(self.analysis_paths.analysis_dir / "_status" / "_du.json")
        if payload is None or "disk_utilization_bytes" not in payload:
            return None
        return int(payload["disk_utilization_bytes"])

    @property
    def df_status(self):
        """
        Get status DataFrame for all scenarios in the analysis.

        Returns
        -------
        pd.DataFrame
            Long-format status table with one row per (event_iloc, model_type),
            including scenario setup status, model run completion status,
            parsed Snakemake allocated resources, and actual runtime details
            (where available from model logs / reports).
        """
        if self.cfg_analysis.toggle_sensitivity_analysis:
            df_status = self.sensitivity.df_status
            df_status_joined = df_status.merge(
                self._df_snakemake_allocations,
                on=["model_type", "scenario_directory", "event_iloc"],
                how="left",
            )
            allocation_columns = [
                col
                for col in df_status_joined.columns
                if col.startswith("snakemake_") and col != "snakemake_allocation_parse_error"
            ]
            if allocation_columns:
                missing_mask = df_status_joined[allocation_columns].isna().any(axis=1)
                if missing_mask.any():
                    # Un-run sub-analyses: present in sensitivity.df_status (full XLSX
                    # definition) but absent from the parsed Snakefile's simulation_sa_*
                    # rules (expected per v2 wait-rule substitution / reprocess filtering /
                    # mid-study XLSX growth — see bug plan D-b). Surface them (R5) instead
                    # of raising (R1): annotate the parse-error column, leave allocation
                    # columns NaN.
                    if "snakemake_allocation_parse_error" not in df_status_joined.columns:
                        df_status_joined["snakemake_allocation_parse_error"] = None
                    df_status_joined.loc[missing_mask, "snakemake_allocation_parse_error"] = (
                        "no simulation_sa_* rule in Snakefile — sub-analysis not run"
                    )
            return self._reorder_df_status_columns(df_status_joined)

        enabled_models_untyped = self._get_enabled_model_types()
        enabled_models: list[Literal["triton", "tritonswmm", "swmm"]] = [
            m for m in ("triton", "tritonswmm", "swmm") if m in enabled_models_untyped
        ]

        rows: list[dict] = []
        for event_iloc in self.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, self)
            scen.log.refresh()
            scenario_setup = scen.log.scenario_creation_complete.get() is True
            scenario_dir = str(scen.log.logfile.parent)
            scenario_du = scen.disk_utilization_bytes

            weather_row = self.df_sims.loc[event_iloc].to_dict()

            for model_type in enabled_models:
                row = dict(weather_row)
                row["event_iloc"] = event_iloc
                row["model_type"] = model_type
                row["scenario_setup"] = scenario_setup
                row["run_completed"] = scen.model_run_completed(model_type)
                row["n_resumes"] = scen.get_log(model_type).n_resumes.get() or 0
                row["scenario_directory"] = scenario_dir
                row["disk_utilization_bytes"] = scenario_du

                # Provide model-specific expected resources to downstream validators.
                if model_type == "swmm":
                    row["run_mode"] = "serial" if self.cfg_analysis.n_omp_threads == 1 else "openmp"
                    row["n_mpi_procs"] = 1
                    row["n_omp_threads"] = self.cfg_analysis.n_omp_threads or 1
                    row["n_gpus"] = 0
                    row["backend_used"] = "cpu"
                else:
                    row["run_mode"] = self.cfg_analysis.run_mode
                    row["n_mpi_procs"] = self.cfg_analysis.n_mpi_procs or 1
                    row["n_omp_threads"] = self.cfg_analysis.n_omp_threads or 1
                    row["n_gpus"] = (self.cfg_analysis.n_gpus or 0) if self.cfg_analysis.run_mode == "gpu" else 0
                    row["backend_used"] = scen.log.triton_backend_used.get()

                if self.in_slurm:
                    row["n_nodes"] = 1 if model_type == "swmm" else self.cfg_analysis.n_nodes or 1

                # Actual resources (model-dependent availability)
                if model_type == "tritonswmm":
                    log_out_path = (scen.scen_paths.out_tritonswmm or scen.scen_paths.sim_folder) / "log.out"
                    log_data = parse_triton_log_file(log_out_path)
                    row["actual_nTasks"] = log_data["nTasks"]
                    row["actual_omp_threads"] = log_data["omp_threads_per_task"]
                    row["actual_gpus"] = log_data["gpus_per_task"]
                    row["actual_total_gpus"] = log_data["total_gpus"]
                    row["actual_gpu_backend"] = log_data["gpu_backend"]
                    row["actual_build_type"] = log_data["build_type"]
                elif model_type == "triton":
                    log_out_path = (scen.scen_paths.out_triton or scen.scen_paths.sim_folder) / "log.out"
                    log_data = parse_triton_log_file(log_out_path)
                    row["actual_nTasks"] = log_data["nTasks"]
                    row["actual_omp_threads"] = log_data["omp_threads_per_task"]
                    row["actual_gpus"] = log_data["gpus_per_task"]
                    row["actual_total_gpus"] = log_data["total_gpus"]
                    row["actual_gpu_backend"] = log_data["gpu_backend"]
                    row["actual_build_type"] = log_data["build_type"]
                else:  # swmm
                    swmm_report_data = retrieve_swmm_performance_stats_from_rpt(scen.scen_paths.swmm_full_rpt_file)
                    row["actual_nTasks"] = 1
                    row["actual_omp_threads"] = swmm_report_data.get("actual_omp_threads")
                    row["actual_gpus"] = None
                    row["actual_total_gpus"] = None
                    row["actual_gpu_backend"] = "none"
                    row["actual_build_type"] = "SWMM"

                # Performance breakdown from processed summary dataset
                row.update(self._get_performance_summary_row(event_iloc, model_type))

                rows.append(row)

        df_status = pd.DataFrame(rows)
        if self.cfg_analysis.is_subanalysis:
            return self._reorder_df_status_columns(df_status)
        else:
            df_status_joined = df_status.merge(
                self._df_snakemake_allocations,
                on=["model_type", "scenario_directory", "event_iloc"],
                how="left",
            )
            allocation_columns = [
                col
                for col in df_status_joined.columns
                if col.startswith("snakemake_") and col != "snakemake_allocation_parse_error"
            ]
            if allocation_columns:
                missing_mask = df_status_joined[allocation_columns].isna().any(axis=1)
                if missing_mask.any():
                    # Regular analysis whose run_* rule was wait-rule-substituted
                    # (v2 graceful-rerun) or otherwise absent: surface (R5) instead
                    # of raising (R1/R9), symmetric with the sensitivity branch.
                    if "snakemake_allocation_parse_error" not in df_status_joined.columns:
                        df_status_joined["snakemake_allocation_parse_error"] = None
                    df_status_joined.loc[missing_mask, "snakemake_allocation_parse_error"] = (
                        "no run_* rule in Snakefile — model not run"
                    )
            return self._reorder_df_status_columns(df_status_joined)

    # TRITON-SWMM model accessors
    @property
    def _tritonswmm_TRITON_summary(self):
        return self.process.tritonswmm_TRITON_summary

    @property
    def _tritonswmm_performance_summary(self):
        return self.process.tritonswmm_performance_summary

    @property
    def _tritonswmm_SWMM_node_summary(self):
        return self.process.tritonswmm_SWMM_node_summary

    @property
    def _tritonswmm_SWMM_link_summary(self):
        return self.process.tritonswmm_SWMM_link_summary

    # TRITON-only model accessors
    @property
    def _triton_only_summary(self):
        return self.process.triton_only_summary

    @property
    def _triton_only_performance_summary(self):
        return self.process.triton_only_performance_summary

    # SWMM-only model accessors
    @property
    def _swmm_only_node_summary(self):
        return self.process.swmm_only_node_summary

    @property
    def _swmm_only_link_summary(self):
        return self.process.swmm_only_link_summary

    def cancel(self, verbose: bool = True, wait_timeout: int = 120, debug: bool = False) -> dict:
        """
        Cancel ongoing tmux workflow for this analysis.

        This method sends SIGINT to the Snakemake process running in the tmux session,
        which triggers Snakemake's built-in cancel_jobs() to cleanly cancel all worker jobs.

        The method uses persistent log data to identify the session, so it works across
        terminal sessions (close terminal, reopen, reinitialize analysis, call cancel).

        **Key features:**
        - Checks if session is actually running before attempting cancellation
        - Sends SIGINT to Snakemake process for clean cancellation
        - Waits for Snakemake to finish canceling worker jobs
        - Verifies all worker jobs are terminated
        - Gracefully handles already-completed workflows

        Parameters
        ----------
        verbose : bool, default=True
            Print progress messages
        wait_timeout : int, default=120
            Maximum seconds to wait for Snakemake process exit
        debug : bool, default=False
            Print detailed per-iteration diagnostics during the wait loop

        Returns
        -------
        dict
            Cancellation status with keys:
            - success: bool (True if cancellation succeeded or no session running)
            - session_canceled: bool
            - workers_canceled: bool
            - jobs_were_running: bool (False if no session found to cancel)
            - message: str
            - session_name: str | None
            - errors: list[str] (any errors encountered)

        Examples
        --------
        Cancel from same session:
        >>> result = analysis.submit_workflow(wait_for_completion=False)
        >>> # ... later decide to cancel ...
        >>> cancel_result = analysis.cancel()

        Cancel from new terminal session:
        >>> analysis = TRITONSWMM_analysis("config.yaml", system)
        >>> cancel_result = analysis.cancel()  # Loads session name from log
        """
        import datetime
        import subprocess

        if verbose:
            print(
                f"[Cancel] Checking workflow status for analysis '{self.cfg_analysis.analysis_id}'",
                flush=True,
            )

        # Load session info from persistent log
        session_name = self.log.tmux_session_name.get()
        snakemake_pid = self.log.snakemake_pid.get()
        analysis_id = self.cfg_analysis.analysis_id

        # Step 0: Check if tmux session exists
        if not session_name:
            if verbose:
                print(
                    f"[Cancel] No tmux session recorded for analysis '{analysis_id}'",
                    flush=True,
                )
            return {
                "success": True,
                "session_canceled": False,
                "workers_canceled": False,
                "jobs_were_running": False,
                "session_name": None,
                "analysis_id": analysis_id,
                "message": f"No workflow session found for analysis '{analysis_id}'",
                "errors": [],
            }

        # Check if session still exists
        session_check = subprocess.run(
            ["tmux", "has-session", "-t", session_name],
            capture_output=True,
            text=True,
        )

        if session_check.returncode != 0:
            if verbose:
                print(
                    f"[Cancel] Tmux session '{session_name}' no longer exists (workflow already completed)",
                    flush=True,
                )
            return {
                "success": True,
                "session_canceled": False,
                "workers_canceled": False,
                "jobs_were_running": False,
                "session_name": session_name,
                "analysis_id": analysis_id,
                "message": f"Tmux session '{session_name}' already completed",
                "errors": [],
            }

        # Session exists, proceed with cancellation
        if verbose:
            print(
                f"[Cancel] Canceling workflow in tmux session '{session_name}'",
                flush=True,
            )

        errors = []

        # Step 1: Get current Snakemake PID (may have changed since submission)
        current_pid = self._workflow_builder._get_snakemake_pid_from_tmux(session_name)
        if current_pid:
            snakemake_pid = current_pid
        elif not snakemake_pid:
            # Could not find PID - try killing session directly
            if verbose:
                print(
                    "[Cancel] WARNING: Could not find Snakemake PID. Killing tmux session directly.",
                    flush=True,
                )
            subprocess.run(
                ["tmux", "kill-session", "-t", session_name],
                capture_output=True,
            )
            self.log.workflow_canceled.set(True)
            self.log.workflow_cancellation_time.set(datetime.datetime.now().isoformat())

            return {
                "success": True,
                "session_canceled": True,
                "workers_canceled": False,  # Unknown
                "jobs_were_running": True,
                "session_name": session_name,
                "analysis_id": analysis_id,
                "message": "Tmux session killed (PID not found - worker cleanup uncertain)",
                "errors": ["Could not find Snakemake PID for clean cancellation"],
            }

        # Step 2: Send SIGINT to Snakemake process
        if verbose:
            print(
                f"[Cancel] Sending SIGINT to Snakemake (PID {snakemake_pid})...",
                flush=True,
            )

        try:
            os.kill(snakemake_pid, signal.SIGINT)
            if verbose:
                print("[Cancel]   ✓ SIGINT sent", flush=True)
        except ProcessLookupError:
            if verbose:
                print(
                    f"[Cancel]   ⚠ Process {snakemake_pid} already exited",
                    flush=True,
                )
        except PermissionError as e:
            error_msg = f"Permission denied sending SIGINT to PID {snakemake_pid}: {e}"
            errors.append(error_msg)
            if verbose:
                print(f"[Cancel]   ✗ {error_msg}", flush=True)

        # Step 3: Wait for Snakemake to finish canceling jobs
        if verbose:
            print(
                f"[Cancel] Waiting for Snakemake to cancel worker jobs (timeout: {wait_timeout}s)...",
                flush=True,
            )

        start_time = time.time()
        process_exited = False

        while time.time() - start_time < wait_timeout:
            time.sleep(2)

            # Check if process still exists using ps (works across permission boundaries)
            ps_check = subprocess.run(
                ["ps", "-p", str(snakemake_pid)],
                capture_output=True,
            )
            elapsed = int(time.time() - start_time)
            if debug:
                print(
                    f"[Cancel]   [debug] ps returncode={ps_check.returncode} at {elapsed}s",
                    flush=True,
                )
            if ps_check.returncode != 0:
                process_exited = True
                if verbose:
                    print(
                        "[Cancel]   ✓ Snakemake process exited",
                        flush=True,
                    )
                break

        if not process_exited:
            error_msg = f"Snakemake process {snakemake_pid} did not exit within {wait_timeout}s"
            errors.append(error_msg)
            if verbose:
                print(f"[Cancel]   ⚠ {error_msg}", flush=True)
                print(
                    "[Cancel]   (Killing tmux session anyway)",
                    flush=True,
                )

        # Step 4: Verify worker jobs are canceled
        if verbose:
            print("[Cancel] Verifying worker jobs are canceled...", flush=True)

        worker_count = 0
        try:
            result = subprocess.run(
                ["squeue", "-u", "$(whoami)", "-o", "%j", "-h"],
                capture_output=True,
                text=True,
                shell=True,
                timeout=5,
            )

            if result.returncode == 0:
                for line in result.stdout.split("\n"):
                    if analysis_id in line:
                        worker_count += 1

            if worker_count > 0:
                error_msg = f"{worker_count} worker jobs still running (Snakemake may not have canceled them)"
                errors.append(error_msg)
                if verbose:
                    print(f"[Cancel]   ⚠ {error_msg}", flush=True)
            elif verbose:
                print("[Cancel]   ✓ All worker jobs canceled", flush=True)

        except (subprocess.TimeoutExpired, FileNotFoundError):
            if verbose:
                print(
                    "[Cancel]   ⚠ Could not verify worker job status",
                    flush=True,
                )

        # Step 5: Kill tmux session
        if verbose:
            print("[Cancel] Cleaning up tmux session...", flush=True)

        kill_result = subprocess.run(
            ["tmux", "kill-session", "-t", session_name],
            capture_output=True,
            text=True,
        )

        if kill_result.returncode == 0:
            if verbose:
                print("[Cancel]   ✓ Tmux session terminated", flush=True)
        else:
            error_msg = f"Failed to kill tmux session: {kill_result.stderr.strip()}"
            errors.append(error_msg)
            if verbose:
                print(f"[Cancel]   ✗ {error_msg}", flush=True)

        # Step 6: Update analysis log
        self.log.workflow_canceled.set(True)
        self.log.workflow_cancellation_time.set(datetime.datetime.now().isoformat())

        success = len(errors) == 0 and worker_count == 0

        if verbose:
            if success:
                print("[Cancel] ✓ Workflow canceled successfully", flush=True)
            else:
                print(
                    "[Cancel] ✗ Cancellation completed with warnings/errors",
                    flush=True,
                )

        return {
            "success": success,
            "session_canceled": True,
            "workers_canceled": worker_count == 0,
            "jobs_were_running": True,
            "session_name": session_name,
            "analysis_id": analysis_id,
            "message": ("Workflow canceled" if success else f"Cancellation issues: {'; '.join(errors)}"),
            "errors": errors,
        }

df_status property

Get status DataFrame for all scenarios in the analysis.

Returns:

Type	Description
DataFrame	Long-format status table with one row per (event_iloc, model_type), including scenario setup status, model run completion status, parsed Snakemake allocated resources, and actual runtime details (where available from model logs / reports).

disk_utilization_bytes property

Return the analysis-level DU sentinel value, or None if absent.

bundle_report_data(output_path=None)

Emit a portable render bundle for local renderer iteration.

Opt-in only — NEVER invoked from analysis.run() or submit_workflow(). The bundle is a self-contained tar including every source path declared via prov.artist().add_channel(...) during the most recent render_report() execution, plus configs with relative paths, the Snakefile, and the HPC-baseline analysis_report.{html,zip} under bundle_baseline/.

Args: output_path: Optional target path for the bundle tar. Defaults to {analysis_dir}/render_bundle/{analysis_id}_{git_sha}_v{schema}.tar.

Returns: Path to the emitted bundle tar.

Raises: FileNotFoundError: If render_report() has not been invoked on this analysis (no *.manifest.json sidecars exist).

Source code in src/hhemt/analysis.py

def bundle_report_data(
    self,
    output_path: "Path | None" = None,
) -> "Path":
    """Emit a portable render bundle for local renderer iteration.

    Opt-in only — NEVER invoked from analysis.run() or
    submit_workflow(). The bundle is a self-contained tar including
    every source path declared via prov.artist().add_channel(...)
    during the most recent render_report() execution, plus configs
    with relative paths, the Snakefile, and the HPC-baseline
    analysis_report.{html,zip} under bundle_baseline/.

    Args:
        output_path: Optional target path for the bundle tar.
            Defaults to
            {analysis_dir}/render_bundle/{analysis_id}_{git_sha}_v{schema}.tar.

    Returns:
        Path to the emitted bundle tar.

    Raises:
        FileNotFoundError: If render_report() has not been invoked
            on this analysis (no *.manifest.json sidecars exist).
    """
    from hhemt.bundle import emit_bundle

    return emit_bundle(self, output_path)

cancel(verbose=True, wait_timeout=120, debug=False)

Cancel ongoing tmux workflow for this analysis.

This method sends SIGINT to the Snakemake process running in the tmux session, which triggers Snakemake's built-in cancel_jobs() to cleanly cancel all worker jobs.

The method uses persistent log data to identify the session, so it works across terminal sessions (close terminal, reopen, reinitialize analysis, call cancel).

Key features: - Checks if session is actually running before attempting cancellation - Sends SIGINT to Snakemake process for clean cancellation - Waits for Snakemake to finish canceling worker jobs - Verifies all worker jobs are terminated - Gracefully handles already-completed workflows

Parameters:

Name	Type	Description	Default
verbose	bool	Print progress messages	True
wait_timeout	int	Maximum seconds to wait for Snakemake process exit	120
debug	bool	Print detailed per-iteration diagnostics during the wait loop	False

Returns:

Type	Description
dict	Cancellation status with keys: - success: bool (True if cancellation succeeded or no session running) - session_canceled: bool - workers_canceled: bool - jobs_were_running: bool (False if no session found to cancel) - message: str - session_name: str | None - errors: list[str] (any errors encountered)

Examples:

Cancel from same session:

>>> result = analysis.submit_workflow(wait_for_completion=False)
>>> # ... later decide to cancel ...
>>> cancel_result = analysis.cancel()

Cancel from new terminal session:

>>> analysis = TRITONSWMM_analysis("config.yaml", system)
>>> cancel_result = analysis.cancel()  # Loads session name from log

Source code in src/hhemt/analysis.py

def cancel(self, verbose: bool = True, wait_timeout: int = 120, debug: bool = False) -> dict:
    """
    Cancel ongoing tmux workflow for this analysis.

    This method sends SIGINT to the Snakemake process running in the tmux session,
    which triggers Snakemake's built-in cancel_jobs() to cleanly cancel all worker jobs.

    The method uses persistent log data to identify the session, so it works across
    terminal sessions (close terminal, reopen, reinitialize analysis, call cancel).

    **Key features:**
    - Checks if session is actually running before attempting cancellation
    - Sends SIGINT to Snakemake process for clean cancellation
    - Waits for Snakemake to finish canceling worker jobs
    - Verifies all worker jobs are terminated
    - Gracefully handles already-completed workflows

    Parameters
    ----------
    verbose : bool, default=True
        Print progress messages
    wait_timeout : int, default=120
        Maximum seconds to wait for Snakemake process exit
    debug : bool, default=False
        Print detailed per-iteration diagnostics during the wait loop

    Returns
    -------
    dict
        Cancellation status with keys:
        - success: bool (True if cancellation succeeded or no session running)
        - session_canceled: bool
        - workers_canceled: bool
        - jobs_were_running: bool (False if no session found to cancel)
        - message: str
        - session_name: str | None
        - errors: list[str] (any errors encountered)

    Examples
    --------
    Cancel from same session:
    >>> result = analysis.submit_workflow(wait_for_completion=False)
    >>> # ... later decide to cancel ...
    >>> cancel_result = analysis.cancel()

    Cancel from new terminal session:
    >>> analysis = TRITONSWMM_analysis("config.yaml", system)
    >>> cancel_result = analysis.cancel()  # Loads session name from log
    """
    import datetime
    import subprocess

    if verbose:
        print(
            f"[Cancel] Checking workflow status for analysis '{self.cfg_analysis.analysis_id}'",
            flush=True,
        )

    # Load session info from persistent log
    session_name = self.log.tmux_session_name.get()
    snakemake_pid = self.log.snakemake_pid.get()
    analysis_id = self.cfg_analysis.analysis_id

    # Step 0: Check if tmux session exists
    if not session_name:
        if verbose:
            print(
                f"[Cancel] No tmux session recorded for analysis '{analysis_id}'",
                flush=True,
            )
        return {
            "success": True,
            "session_canceled": False,
            "workers_canceled": False,
            "jobs_were_running": False,
            "session_name": None,
            "analysis_id": analysis_id,
            "message": f"No workflow session found for analysis '{analysis_id}'",
            "errors": [],
        }

    # Check if session still exists
    session_check = subprocess.run(
        ["tmux", "has-session", "-t", session_name],
        capture_output=True,
        text=True,
    )

    if session_check.returncode != 0:
        if verbose:
            print(
                f"[Cancel] Tmux session '{session_name}' no longer exists (workflow already completed)",
                flush=True,
            )
        return {
            "success": True,
            "session_canceled": False,
            "workers_canceled": False,
            "jobs_were_running": False,
            "session_name": session_name,
            "analysis_id": analysis_id,
            "message": f"Tmux session '{session_name}' already completed",
            "errors": [],
        }

    # Session exists, proceed with cancellation
    if verbose:
        print(
            f"[Cancel] Canceling workflow in tmux session '{session_name}'",
            flush=True,
        )

    errors = []

    # Step 1: Get current Snakemake PID (may have changed since submission)
    current_pid = self._workflow_builder._get_snakemake_pid_from_tmux(session_name)
    if current_pid:
        snakemake_pid = current_pid
    elif not snakemake_pid:
        # Could not find PID - try killing session directly
        if verbose:
            print(
                "[Cancel] WARNING: Could not find Snakemake PID. Killing tmux session directly.",
                flush=True,
            )
        subprocess.run(
            ["tmux", "kill-session", "-t", session_name],
            capture_output=True,
        )
        self.log.workflow_canceled.set(True)
        self.log.workflow_cancellation_time.set(datetime.datetime.now().isoformat())

        return {
            "success": True,
            "session_canceled": True,
            "workers_canceled": False,  # Unknown
            "jobs_were_running": True,
            "session_name": session_name,
            "analysis_id": analysis_id,
            "message": "Tmux session killed (PID not found - worker cleanup uncertain)",
            "errors": ["Could not find Snakemake PID for clean cancellation"],
        }

    # Step 2: Send SIGINT to Snakemake process
    if verbose:
        print(
            f"[Cancel] Sending SIGINT to Snakemake (PID {snakemake_pid})...",
            flush=True,
        )

    try:
        os.kill(snakemake_pid, signal.SIGINT)
        if verbose:
            print("[Cancel]   ✓ SIGINT sent", flush=True)
    except ProcessLookupError:
        if verbose:
            print(
                f"[Cancel]   ⚠ Process {snakemake_pid} already exited",
                flush=True,
            )
    except PermissionError as e:
        error_msg = f"Permission denied sending SIGINT to PID {snakemake_pid}: {e}"
        errors.append(error_msg)
        if verbose:
            print(f"[Cancel]   ✗ {error_msg}", flush=True)

    # Step 3: Wait for Snakemake to finish canceling jobs
    if verbose:
        print(
            f"[Cancel] Waiting for Snakemake to cancel worker jobs (timeout: {wait_timeout}s)...",
            flush=True,
        )

    start_time = time.time()
    process_exited = False

    while time.time() - start_time < wait_timeout:
        time.sleep(2)

        # Check if process still exists using ps (works across permission boundaries)
        ps_check = subprocess.run(
            ["ps", "-p", str(snakemake_pid)],
            capture_output=True,
        )
        elapsed = int(time.time() - start_time)
        if debug:
            print(
                f"[Cancel]   [debug] ps returncode={ps_check.returncode} at {elapsed}s",
                flush=True,
            )
        if ps_check.returncode != 0:
            process_exited = True
            if verbose:
                print(
                    "[Cancel]   ✓ Snakemake process exited",
                    flush=True,
                )
            break

    if not process_exited:
        error_msg = f"Snakemake process {snakemake_pid} did not exit within {wait_timeout}s"
        errors.append(error_msg)
        if verbose:
            print(f"[Cancel]   ⚠ {error_msg}", flush=True)
            print(
                "[Cancel]   (Killing tmux session anyway)",
                flush=True,
            )

    # Step 4: Verify worker jobs are canceled
    if verbose:
        print("[Cancel] Verifying worker jobs are canceled...", flush=True)

    worker_count = 0
    try:
        result = subprocess.run(
            ["squeue", "-u", "$(whoami)", "-o", "%j", "-h"],
            capture_output=True,
            text=True,
            shell=True,
            timeout=5,
        )

        if result.returncode == 0:
            for line in result.stdout.split("\n"):
                if analysis_id in line:
                    worker_count += 1

        if worker_count > 0:
            error_msg = f"{worker_count} worker jobs still running (Snakemake may not have canceled them)"
            errors.append(error_msg)
            if verbose:
                print(f"[Cancel]   ⚠ {error_msg}", flush=True)
        elif verbose:
            print("[Cancel]   ✓ All worker jobs canceled", flush=True)

    except (subprocess.TimeoutExpired, FileNotFoundError):
        if verbose:
            print(
                "[Cancel]   ⚠ Could not verify worker job status",
                flush=True,
            )

    # Step 5: Kill tmux session
    if verbose:
        print("[Cancel] Cleaning up tmux session...", flush=True)

    kill_result = subprocess.run(
        ["tmux", "kill-session", "-t", session_name],
        capture_output=True,
        text=True,
    )

    if kill_result.returncode == 0:
        if verbose:
            print("[Cancel]   ✓ Tmux session terminated", flush=True)
    else:
        error_msg = f"Failed to kill tmux session: {kill_result.stderr.strip()}"
        errors.append(error_msg)
        if verbose:
            print(f"[Cancel]   ✗ {error_msg}", flush=True)

    # Step 6: Update analysis log
    self.log.workflow_canceled.set(True)
    self.log.workflow_cancellation_time.set(datetime.datetime.now().isoformat())

    success = len(errors) == 0 and worker_count == 0

    if verbose:
        if success:
            print("[Cancel] ✓ Workflow canceled successfully", flush=True)
        else:
            print(
                "[Cancel] ✗ Cancellation completed with warnings/errors",
                flush=True,
            )

    return {
        "success": success,
        "session_canceled": True,
        "workers_canceled": worker_count == 0,
        "jobs_were_running": True,
        "session_name": session_name,
        "analysis_id": analysis_id,
        "message": ("Workflow canceled" if success else f"Cancellation issues: {'; '.join(errors)}"),
        "errors": errors,
    }

delete(override_in_flight=False, *, override_multi_sim_run_method=None)

Distributed Snakemake workflow that deletes the entire analysis_dir.

Refuses by default when _status/_submitted/*.json sentinels indicate live SLURM jobs. Pass override_in_flight=True to bypass the guard.

Mirrors the dispatch pattern of :meth:reprocess — sensitivity-toggled analyses dispatch to :meth:TRITONSWMM_sensitivity_analysis.delete.

Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction + D-DeleteBoundary resolutions) and distributed-delete-and-du-recording Phase 3 (SLURM lift; override_multi_sim_run_method mirrors the run-mode override pattern from :meth:submit_workflow).

Source code in src/hhemt/analysis.py

def delete(
    self,
    override_in_flight: bool = False,
    *,
    override_multi_sim_run_method: Literal["local", "batch_job", "1_job_many_srun_tasks"] | None = None,
) -> None:
    """Distributed Snakemake workflow that deletes the entire analysis_dir.

    Refuses by default when ``_status/_submitted/*.json`` sentinels
    indicate live SLURM jobs. Pass ``override_in_flight=True`` to bypass
    the guard.

    Mirrors the dispatch pattern of :meth:`reprocess` — sensitivity-toggled
    analyses dispatch to
    :meth:`TRITONSWMM_sensitivity_analysis.delete`.

    Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction
    + D-DeleteBoundary resolutions) and distributed-delete-and-du-recording
    Phase 3 (SLURM lift; ``override_multi_sim_run_method`` mirrors the
    run-mode override pattern from :meth:`submit_workflow`).
    """
    if self.cfg_analysis.toggle_sensitivity_analysis:
        return self.sensitivity.delete(
            override_in_flight=override_in_flight,
            override_multi_sim_run_method=override_multi_sim_run_method,
        )

    analysis_dir = self.analysis_paths.analysis_dir

    # 1. Clear any stale sentinels from a prior failed delete attempt.
    # Without this, the post-check at step 3 could falsely pass on a
    # half-completed previous delete and fast_rmtree a partially-deleted
    # tree.
    stale_dir = analysis_dir / "_status" / "_deleting"
    if stale_dir.exists():
        # EXEMPT-DU: status-dir-cleanup
        fast_rmtree(stale_dir)

    # 2. Submit the distributed delete workflow. The workflow builder's
    # _pre_delete_guards (live-sentinel refusal + scoped lock-check) runs
    # inside submit_delete_workflow; orchestrator does not invoke it
    # directly.
    self._workflow_builder.submit_delete_workflow(
        override_in_flight=override_in_flight,
        override_multi_sim_run_method=override_multi_sim_run_method,
    )

    # 3. Verify all expected sentinels present; remove analysis_dir atomically.
    expected = self._enumerate_expected_delete_sentinels()
    deleting_dir = analysis_dir / "_status" / "_deleting"
    actual = set(deleting_dir.glob("*.flag")) if deleting_dir.exists() else set()
    missing = expected - actual
    if missing:
        print(
            f"[delete] {len(missing)} per-rule sentinels missing — preserving analysis_dir for debugging.",
            flush=True,
        )
        print(f"[delete] missing: {sorted(p.name for p in missing)}", flush=True)
        return
    print(
        f"[delete] all {len(expected)} per-rule sentinels present — removing analysis_dir.",
        flush=True,
    )
    # EXEMPT-DU: full-analysis-root-wipe
    fast_rmtree(analysis_dir)

eda(*, override_eda_config=None, notebook_filename=None)

Run the in-process EDA loop: calc -> plots -> doc (ADR-10).

A LIGHTER non-Snakemake facade. Resolves the EDA config (override-or-cfg per the override_ convention), runs the calc members, renders the EDA plots under plots/eda/, and assembles eda_report/eda_report.html. Returns an EdaReportResult. Bundle carriage: run this BEFORE bundle_report_data() so the EDA plots' declared eda/.zarr sources are harvested into the bundle (the plots emit under plots/eda/ and declare the zarr as a source); bundling before eda() silently omits EDA content.

Source code in src/hhemt/analysis.py

def eda(
    self,
    *,
    override_eda_config: "Path | None" = None,
    notebook_filename: "str | None" = None,
) -> "EdaReportResult":
    """Run the in-process EDA loop: calc -> plots -> doc (ADR-10).

    A LIGHTER non-Snakemake facade. Resolves the EDA config (override-or-cfg
    per the override_ convention), runs the calc members, renders the EDA
    plots under plots/eda/, and assembles eda_report/eda_report.html. Returns
    an EdaReportResult. Bundle carriage: run this BEFORE bundle_report_data()
    so the EDA plots' declared eda/<plot_id>.zarr sources are harvested into
    the bundle (the plots emit under plots/eda/ and declare the zarr as a
    source); bundling before eda() silently omits EDA content.
    """
    from hhemt.config.eda import eda_config
    from hhemt.config.loaders import yaml_to_model
    from hhemt.eda import (
        EdaReportResult,
        check_cross_sim_identity,
        render_eda_plots,
    )
    from hhemt.eda._html_export import export_eda_html
    from hhemt.eda._local_surface import emit_eda_local_surface
    from hhemt.eda._notebook import emit_eda_notebook

    eda_cfg = (
        yaml_to_model(override_eda_config, eda_config) if override_eda_config is not None else self.cfg_analysis.eda
    )
    root = Path(self.analysis_paths.analysis_dir)
    verdict_result = check_cross_sim_identity(self)
    verdicts = [verdict_result.verdict] if verdict_result.verdict is not None else []
    # F-B Flag 2 (hhemt-specialist review): load_eda_context reads the fixed-name
    # cfg_analysis.yaml / cfg_system.yaml at `root`; the LIVE analysis_dir does NOT
    # carry them by construction (only the bundle staging dir does, via
    # _copy_configs_with_relative_paths). Materialize them here so the loader's
    # fixed-name contract holds for the live root too (without this, the notebook's
    # first executed cell `load_eda_context(root)` raises FileNotFoundError).
    import yaml as _yaml

    (root / "cfg_analysis.yaml").write_text(_yaml.safe_dump(self.cfg_analysis.model_dump(mode="json")))
    (root / "cfg_system.yaml").write_text(_yaml.safe_dump(self._system.cfg_system.model_dump(mode="json")))
    # ADR-12: emit the source-independent eda_local/ package skeleton at root.
    emit_eda_local_surface(root)
    # Non-sensitivity analyses produce no eda/<plot_id>.zarr (the cross-sim check
    # skips), so render_eda_plots would open a non-existent zarr — skip rendering
    # on the figureless branch. The NOTEBOOK is still emitted (ADR-14: primary
    # artifact); its zarr-dependent seed cell is gated at execution.
    figureless = verdict_result.skipped or verdict_result.artifact_path is None
    plot_paths = [] if figureless else render_eda_plots(root, cfg_analysis=self.cfg_analysis, eda_cfg=eda_cfg)
    notebook_path = emit_eda_notebook(
        root,
        cfg_analysis=self.cfg_analysis,
        eda_cfg=eda_cfg,
        is_bundle=False,
        notebook_filename=notebook_filename,
    )
    # ADR-14: HTML is a best-effort nbconvert export of the notebook (the source
    # of truth); a kernel/exec failure degrades to None, never fails the loop.
    report_path = export_eda_html(notebook_path, root=root)
    return EdaReportResult(
        report_path=report_path,
        notebook_path=notebook_path,
        plot_paths=plot_paths,
        verdicts=verdicts,
    )

get_workflow_status()

Generate workflow status report.

Inspects logs and outputs to determine completion state of each phase, providing actionable recommendations for which execution mode to use.

Returns:

Type	Description
WorkflowStatus	Structured status report with phase details and recommendations

Examples:

Check status before running:

>>> status = analysis.get_workflow_status()
>>> print(status)
>>> if not status.simulation.complete:
...     print(f"Retry {len(status.simulation.failed_items)} failed sims")

Use recommended mode:

>>> status = analysis.get_workflow_status()
>>> result = analysis.run(mode=status.recommended_mode)

Notes

This method is read-only and does not modify any state. It provides transparency into workflow progress to help users make informed decisions about execution modes.

See Also

run : High-level workflow execution method

Source code in src/hhemt/analysis.py

def get_workflow_status(self) -> "WorkflowStatus":
    """Generate workflow status report.

    Inspects logs and outputs to determine completion state of each phase,
    providing actionable recommendations for which execution mode to use.

    Returns
    -------
    WorkflowStatus
        Structured status report with phase details and recommendations

    Examples
    --------
    Check status before running:

    >>> status = analysis.get_workflow_status()
    >>> print(status)
    >>> if not status.simulation.complete:
    ...     print(f"Retry {len(status.simulation.failed_items)} failed sims")

    Use recommended mode:

    >>> status = analysis.get_workflow_status()
    >>> result = analysis.run(mode=status.recommended_mode)

    Notes
    -----
    This method is read-only and does not modify any state. It provides
    transparency into workflow progress to help users make informed
    decisions about execution modes.

    See Also
    --------
    run : High-level workflow execution method
    """
    from .orchestration import PhaseStatus, WorkflowStatus

    # Check setup phase
    system_log = self._system.log
    dem_done = system_log.dem_processed.get()
    mannings_done = self._system.cfg_system.toggle_use_constant_mannings or system_log.mannings_processed.get()
    compiled = system_log.compilation_tritonswmm_cpu_successful.get()

    setup_complete = dem_done and mannings_done and compiled
    setup_progress = 1.0 if setup_complete else 0.5 if (dem_done or compiled) else 0.0
    setup_details = {
        "dem": f"{'✓' if dem_done else '✗'} DEM processed",
        "mannings": f"{'✓' if mannings_done else '✗'} Manning's processed",
        "compiled": f"{'✓' if compiled else '✗'} TRITON-SWMM compiled",
    }

    setup_phase = PhaseStatus(
        name="setup",
        complete=setup_complete,
        progress=setup_progress,
        details=setup_details,
    )

    # Check scenario preparation
    all_prepared = self._all_scenarios_created
    not_prepared = self._scenarios_not_created

    n_total = self.n_sims

    n_prepared = n_total - len(not_prepared)

    prep_phase = PhaseStatus(
        name="preparation",
        complete=all_prepared,
        progress=n_prepared / n_total if n_total > 0 else 0.0,
        details={"scenarios": f"{'✓' if all_prepared else '⚠'} {n_prepared}/{n_total} scenarios created"},
        failed_items=[str(p) for p in not_prepared],
    )

    # Check simulations
    all_run = self._all_sims_run
    not_run = self._scenarios_not_run
    n_run = n_total - len(not_run)

    sim_phase = PhaseStatus(
        name="simulation",
        complete=all_run,
        progress=n_run / n_total if n_total > 0 else 0.0,
        details={"sims": f"{'✓' if all_run else '⚠'} {n_run}/{n_total} simulations completed"},
        failed_items=[str(p) for p in not_run],
    )

    # Check processing
    enabled_models = self._get_enabled_model_types()
    triton_enabled = "triton" in enabled_models or "tritonswmm" in enabled_models
    swmm_enabled = "swmm" in enabled_models or "tritonswmm" in enabled_models

    triton_missing = len(self._TRITON_time_series_not_processed) if triton_enabled else 0
    swmm_missing = len(self._SWMM_time_series_not_processed) if swmm_enabled else 0

    triton_total = n_total if triton_enabled else 0
    swmm_total = n_total if swmm_enabled else 0

    triton_processed = max(triton_total - triton_missing, 0)
    swmm_processed = max(swmm_total - swmm_missing, 0)

    processed_total = triton_processed + swmm_processed
    total_needed = triton_total + swmm_total
    proc_progress = processed_total / total_needed if total_needed else 0.0

    triton_proc_complete = triton_missing == 0 if triton_enabled else True
    swmm_proc_complete = swmm_missing == 0 if swmm_enabled else True
    proc_complete = triton_proc_complete and swmm_proc_complete

    proc_phase = PhaseStatus(
        name="processing",
        complete=proc_complete,
        progress=proc_progress,
        details={
            "triton": (
                f"{'✓' if triton_proc_complete else '✗'} TRITON outputs processed: "
                f"{triton_processed}/{triton_total}"
                if triton_enabled
                else "✓ TRITON outputs processed: n/a"
            ),
            "swmm": (
                f"{'✓' if swmm_proc_complete else '✗'} SWMM outputs processed: {swmm_processed}/{swmm_total}"
                if swmm_enabled
                else "✓ SWMM outputs processed: n/a"
            ),
        },
    )

    # Check consolidation: under Option B (render_bundle plan), the
    # canonical master-level artifact is analysis_datatree.zarr; the
    # per-mode flat zarrs no longer exist. The log marker
    # `datatree_consolidation_complete` is the single canonical signal
    # for "consolidation has completed."
    summaries_exist = (
        hasattr(self.log, "datatree_consolidation_complete")
        and self.log.datatree_consolidation_complete.get() is True
    )

    consol_details = {
        "datatree": (
            f"{'✓' if summaries_exist else '✗'} analysis_datatree.zarr "
            f"({'present' if summaries_exist else 'not yet built'})"
        )
    }

    consol_phase = PhaseStatus(
        name="consolidation",
        complete=summaries_exist,
        progress=1.0 if summaries_exist else 0.0,
        details=consol_details,
    )

    # Determine current phase and recommendation
    if not setup_complete:
        current = "setup"
        rec_mode = "fresh"
        rec_text = "Setup incomplete. Use 'fresh' mode to process system inputs."
    elif not all_prepared:
        current = "preparation"
        rec_mode = "resume"
        rec_text = f"Use 'resume' to create {len(not_prepared)} remaining scenarios."
    elif not all_run:
        current = "simulation"
        rec_mode = "resume"
        rec_text = f"Use 'resume' to run {len(not_run)} pending/failed simulations."
    elif not proc_complete:
        current = "processing"
        rec_mode = "resume"
        rec_text = "Use 'resume' to process simulation outputs."
    elif not summaries_exist:
        current = "consolidation"
        rec_mode = "resume"
        rec_text = "Use 'resume' to consolidate analysis summaries."
    else:
        current = "complete"
        # 'fresh' is the only actionable mode for a complete analysis (resume has
        # nothing left to do); it is a valid translate_mode() input, so
        # analysis.run(mode=status.recommended_mode) works. 'n/a' is not a member
        # of the documented {fresh, resume} run-mode set and is not run()-able.
        rec_mode = "fresh"
        rec_text = "All phases complete. Use 'fresh' to redo the analysis from scratch."

    return WorkflowStatus(
        analysis_id=self.cfg_analysis.analysis_id,
        analysis_dir=self.analysis_paths.analysis_dir,
        setup=setup_phase,
        preparation=prep_phase,
        simulation=sim_phase,
        processing=proc_phase,
        consolidation=consol_phase,
        total_simulations=n_total,
        simulations_completed=n_run,
        simulations_failed=len(not_run),
        simulations_pending=0,  # Would need more logic to distinguish failed vs pending
        current_phase=current,
        recommended_mode=rec_mode,
        recommendation=rec_text,
    )

globus_to_hpc(transfer_yaml)

Transfer local inputs to HPC via Globus.

Args: transfer_yaml: Path to a transfer spec YAML in configs/transfers/.

Returns: Globus task ID.

Source code in src/hhemt/analysis.py

def globus_to_hpc(self, transfer_yaml: "Path") -> str:
    """Transfer local inputs to HPC via Globus.

    Args:
        transfer_yaml: Path to a transfer spec YAML in configs/transfers/.

    Returns:
        Globus task ID.
    """
    from pathlib import Path as _Path

    from hhemt.config.loaders import load_transfer_config
    from hhemt.globus_transfer import GlobusTransferManager

    spec = load_transfer_config(_Path(transfer_yaml))
    manager = GlobusTransferManager(collection_uuids=[spec.endpoints.destination_uuid])
    return manager.transfer(spec)

globus_to_local(transfer_yaml)

Transfer HPC results to local machine via Globus.

Args: transfer_yaml: Path to a transfer spec YAML in configs/transfers/. See configs/transfers/template_transfer.yaml.

Returns: Globus task ID. Pass to GlobusTransferManager().wait(task_id) to block until complete, or monitor at app.globus.org.

Example::

task_id = analysis.globus_to_local(
    Path("configs/transfers/my_frontier_run.yaml")
)

Source code in src/hhemt/analysis.py

def globus_to_local(self, transfer_yaml: "Path") -> str:
    """Transfer HPC results to local machine via Globus.

    Args:
        transfer_yaml: Path to a transfer spec YAML in configs/transfers/.
                       See configs/transfers/template_transfer.yaml.

    Returns:
        Globus task ID. Pass to ``GlobusTransferManager().wait(task_id)``
        to block until complete, or monitor at app.globus.org.

    Example::

        task_id = analysis.globus_to_local(
            Path("configs/transfers/my_frontier_run.yaml")
        )
    """
    from pathlib import Path as _Path

    from hhemt.config.loaders import load_transfer_config
    from hhemt.globus_transfer import GlobusTransferManager

    spec = load_transfer_config(_Path(transfer_yaml))
    manager = GlobusTransferManager(collection_uuids=[spec.endpoints.source_uuid])
    return manager.transfer(spec)

print_cfg(which='both')

Print configuration settings in tabular format.

Parameters:

Name	Type	Description	Default
which	Literal['system', 'analysis', 'both']	Which configuration to print (default: "both")	'both'

Source code in src/hhemt/analysis.py

def print_cfg(self, which: Literal["system", "analysis", "both"] = "both"):
    """
    Print configuration settings in tabular format.

    Parameters
    ----------
    which : Literal["system", "analysis", "both"], optional
        Which configuration to print (default: "both")
    """
    if which in ["system", "both"]:
        print("=== System Configuration ===", flush=True)
        self._system.cfg_system.display_tabulate_cfg()
    if which == "both":
        print("\n", flush=True)
    if which in ["analysis", "both"]:
        print("=== analysis Configuration ===", flush=True)
        self.cfg_analysis.display_tabulate_cfg()

process_sim_timeseries(event_iloc, which='both', *, override_clear_raw=None, verbose=False, compression_level=5)

Process and write timeseries outputs for a single simulation.

Converts raw TRITON and/or SWMM outputs into processed timeseries files, optionally clearing raw outputs after processing.

Parameters:

Name	Type	Description	Default
event_iloc	int	Integer index of the scenario in df_sims	required
which	Literal['TRITON', 'SWMM', 'both']	Which outputs to process (default: "both")	'both'
override_clear_raw	ClearRawValue | None	Runtime override for cfg_analysis.clear_raw. None (default) reads the YAML; a concrete value overrides for this invocation.	None
verbose	bool	If True, print progress messages (default: False)	False
compression_level	int	Compression level for output files, 0-9 (default: 5)	5

Source code in src/hhemt/analysis.py

def process_sim_timeseries(
    self,
    event_iloc,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    *,
    override_clear_raw: ClearRawValue | None = None,
    verbose: bool = False,
    compression_level: int = 5,
):
    """
    Process and write timeseries outputs for a single simulation.

    Converts raw TRITON and/or SWMM outputs into processed timeseries files,
    optionally clearing raw outputs after processing.

    Parameters
    ----------
    event_iloc : int
        Integer index of the scenario in df_sims
    which : Literal["TRITON", "SWMM", "both"], optional
        Which outputs to process (default: "both")
    override_clear_raw : ClearRawValue | None, optional
        Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
        reads the YAML; a concrete value overrides for this invocation.
    verbose : bool, optional
        If True, print progress messages (default: False)
    compression_level : int, optional
        Compression level for output files, 0-9 (default: 5)
    """
    proc = self._retrieve_sim_run_processing_object(event_iloc=event_iloc)
    proc.write_timeseries_outputs(
        which=which,
        override_clear_raw=override_clear_raw,
        verbose=verbose,
        compression_level=compression_level,
    )
    proc.write_summary_outputs(
        which=which,
        verbose=verbose,
        compression_level=compression_level,
    )

promote_eda_plot(plot_id, *, target, **kwargs)

Promote an EDA plot into a standard static-plot config OR a named reporting set (ADR-11).

Thin facade over eda/_promote.py. target="static_plot" emits an ADR-4 StaticPlotBaseConfig YAML keyed on the plot-ID (kwargs: output_path, caption); target="reporting_set" records the plot-ID's promotion intent against an ADR-5 ReportingSet (kwargs: set_name). Promotion is a deliberate per-plot user action -- it is NOT folded into eda() and is NOT exposed on Bundle.

Source code in src/hhemt/analysis.py

def promote_eda_plot(self, plot_id: str, *, target: str, **kwargs):
    """Promote an EDA plot into a standard static-plot config OR a named reporting set (ADR-11).

    Thin facade over eda/_promote.py. ``target="static_plot"`` emits an ADR-4
    StaticPlotBaseConfig YAML keyed on the plot-ID (kwargs: ``output_path``,
    ``caption``); ``target="reporting_set"`` records the plot-ID's promotion intent
    against an ADR-5 ReportingSet (kwargs: ``set_name``). Promotion is a deliberate
    per-plot user action -- it is NOT folded into eda() and is NOT exposed on Bundle.
    """
    from hhemt.eda._promote import (
        promote_eda_plot_to_static_config,
        register_eda_plot_in_reporting_set,
    )

    if target == "static_plot":
        return promote_eda_plot_to_static_config(plot_id, **kwargs)
    if target == "reporting_set":
        return register_eda_plot_in_reporting_set(plot_id, **kwargs)
    raise ValueError(f"target must be 'static_plot' or 'reporting_set', got {target!r}.")

publish(target, *, override_dataset_license=None, software_doi=None)

Deposit this analysis to a DOI-minting repository (C6, ADR-11).

Opt-in only — NEVER invoked from analysis.run() or submit_workflow(), mirroring bundle_report_data()/transfer_results(). Requires the analysis to have consolidated (ro-crate-metadata.json + analysis_datatree.zarr present); the license is read from the emitted crate sidecar. override_dataset_license does NOT re-stamp the archived license (baked at consolidation into the immutable crate) — it ASSERTS your expected value against the sidecar and raises PublishError on mismatch, directing you to set analysis_config.dataset_license and reprocess(start_with='consolidate'). Returns {"target","data_doi","software_doi","record_url"}.

Source code in src/hhemt/analysis.py

def publish(
    self,
    target: "Literal['hydroshare', 'zenodo']",
    *,
    override_dataset_license: "Literal['CC0-1.0', 'CC-BY-NC-4.0'] | None" = None,
    software_doi: "str | None" = None,
) -> dict:
    """Deposit this analysis to a DOI-minting repository (C6, ADR-11).

    Opt-in only — NEVER invoked from analysis.run() or submit_workflow(), mirroring
    bundle_report_data()/transfer_results(). Requires the analysis to have consolidated
    (ro-crate-metadata.json + analysis_datatree.zarr present); the license is read from
    the emitted crate sidecar. override_dataset_license does NOT re-stamp the archived
    license (baked at consolidation into the immutable crate) — it ASSERTS your expected
    value against the sidecar and raises PublishError on mismatch, directing you to set
    analysis_config.dataset_license and reprocess(start_with='consolidate'). Returns
    {"target","data_doi","software_doi","record_url"}.
    """
    from hhemt.publishing import publish_analysis

    return publish_analysis(
        self,
        target=target,
        override_dataset_license=override_dataset_license,
        software_doi=software_doi,
    )

render_report(format='zip', *, reprocess=False)

Render the report from already-completed workflow outputs.

Idempotent: invokes snakemake --report against the existing Snakefile without re-executing any rules. Requires the workflow to have completed (so the report() outputs exist) and the Snakefile to be on disk.

Parameters:

Name	Type	Description	Default
format	Literal['html', 'zip']	Output format. "html" produces a single self-contained analysis_report.html with all figures inlined as base64, plus React-bundle post-process surgery. "zip" produces analysis_report.zip containing the unbundled report tree; no post-process surgery is applied.	"zip"
reprocess	bool	When True, render against Snakefile.reprocess (the filtered reprocess DAG) instead of the production Snakefile, so the snakemake --report step only expects the figures the reprocess DAG built. Keyword-only; set by the reprocess render_report rule shell. Default False keeps the production render path byte-identical.	False

Returns:

Type	Description
Path	Path to the rendered analysis_report.{format}.

Source code in src/hhemt/analysis.py

def render_report(self, format: "Literal['html','zip']" = "zip", *, reprocess: bool = False) -> "Path":
    """Render the report from already-completed workflow outputs.

    Idempotent: invokes ``snakemake --report`` against the existing Snakefile
    without re-executing any rules. Requires the workflow to have completed
    (so the report() outputs exist) and the Snakefile to be on disk.

    Parameters
    ----------
    format : Literal["html", "zip"], default "zip"
        Output format. ``"html"`` produces a single self-contained
        ``analysis_report.html`` with all figures inlined as base64, plus
        React-bundle post-process surgery. ``"zip"`` produces
        ``analysis_report.zip`` containing the unbundled report tree;
        no post-process surgery is applied.
    reprocess : bool, default False
        When ``True``, render against ``Snakefile.reprocess`` (the filtered
        reprocess DAG) instead of the production ``Snakefile``, so the
        ``snakemake --report`` step only expects the figures the reprocess
        DAG built. Keyword-only; set by the reprocess ``render_report`` rule
        shell. Default ``False`` keeps the production render path
        byte-identical.

    Returns
    -------
    Path
        Path to the rendered ``analysis_report.{format}``.
    """
    import subprocess
    import sys

    from .exceptions import WorkflowError
    from .workflow import _assert_snakefile_package_current

    snakefile_name = "Snakefile.reprocess" if reprocess else "Snakefile"
    snakefile = self.analysis_paths.analysis_dir / snakefile_name
    _assert_snakefile_package_current(snakefile)
    out = self.analysis_paths.analysis_dir / f"analysis_report.{format}"
    css_path = self.analysis_paths.analysis_dir / "report" / "report.css"
    # Brand-theme resolution (ADR-7 layer 2). The dominant render path is
    # render_report_runner.main() → a FRESH TRITONSWMM_analysis(..., is_main_
    # orchestrator=False) that never had run() called, so self._brand_theme
    # (set only in run()) does NOT exist on it. Resolve once here via
    # getattr-fallback and serve BOTH the CSS emit below and the navbar
    # surgery later (D-6; plan-review SE Flag 1).
    from .config.brand_theme import DEFAULT_BRAND_THEME
    from .config.loaders import load_brand_theme
    from .workflow import _brand_theme_css_map

    _theme = getattr(self, "_brand_theme", None)
    if _theme is None:
        _theme = (
            load_brand_theme(self.cfg_analysis.brand_theme)
            if self.cfg_analysis.brand_theme is not None
            else DEFAULT_BRAND_THEME
        )
    # Re-emit report artifacts (report.css + workflow_description template)
    # from package resources so render_report picks up edits made to the
    # source-tree report_templates/ since the analysis was last run.
    _emit_report_artifacts(
        self.analysis_paths.analysis_dir,
        brand_theme=_brand_theme_css_map(_theme),
    )
    # --cores 1 is required by Snakemake's CLI even though --report is a
    # post-execution render that does not execute rules.
    cmd = [
        sys.executable,
        "-m",
        "snakemake",
        "--snakefile",
        str(snakefile),
        "--directory",
        str(self.analysis_paths.analysis_dir),
        "--report",
        str(out),
        "--report-stylesheet",
        str(css_path),
        "--cores",
        "1",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    if result.returncode != 0:
        tail = "\n".join((result.stdout + "\n" + result.stderr).splitlines()[-50:])
        raise WorkflowError(
            phase="render_report",
            return_code=result.returncode,
            stderr=f"snakemake --report exit {result.returncode}; last 50 lines:\n{tail}",
        )
    # Apply React-bundle post-process surgery (title, navbar, sort order,
    # placeholder category, showCategory auto-pop, row-click delegate).
    # Both formats need the surgery:
    #  - HTML: edit the single rendered file in place.
    #  - Zip: extract, edit `analysis_report/report.html` inside, re-zip.
    # Without surgery in zip mode, the eye-icon-hiding CSS in report.css
    # leaves figure tables with no clickable affordance (the JS click
    # delegate that makes rows clickable lives only in the surgery).
    from .report_renderers._react_surgery import (
        apply_post_process_surgery,
        apply_post_process_surgery_to_zip,
    )

    # Navbar upper-left brand text: brand_theme.upper_left_text (ADR-7),
    # defaulting to analysis_id when None (D-6). _theme is resolved above.
    _navbar = _theme.upper_left_text or self.cfg_analysis.analysis_id
    # Resolve the active set's category_order. render_report() is dominantly
    # invoked from render_report_runner.main() on a FRESH analysis that never
    # called run() (see the _brand_theme getattr-fallback above for the
    # identical hazard), so self._active_reporting_set may not exist. getattr-
    # fallback to a config-only resolution (no CSV cross-validation at render
    # time) mirroring the _theme fallback above. Never let the bare attribute
    # AttributeError be swallowed by the surrounding `except Exception: pass`.
    _active_set = getattr(self, "_active_reporting_set", None)
    if _active_set is None:
        # render-without-run() fallback. Fail SOFT (SE F-I-3): the render path
        # bypasses validate_active_reporting_set, so a stale/unknown
        # reporting_set would raise here and surface as an opaque Snakemake
        # rule failure. Degrade to the historical "default" sidebar order + a
        # one-line warning instead of crashing the render rule.
        import logging

        from .config.report import resolve_active_reporting_set_name
        from .report_renderers._reporting_sets import get_reporting_set

        try:
            _cfg_report = getattr(self, "_cfg_report", None)
            if _cfg_report is None:
                _cfg_report = self.cfg_analysis.report
            _set_name = resolve_active_reporting_set_name(
                _cfg_report,
                is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
            )
            _active_set = get_reporting_set(_set_name)
        except Exception as _e:
            logging.getLogger(__name__).warning(
                "render-path reporting_set resolution failed (%s); falling back to 'default' category order",
                _e,
            )
            _active_set = get_reporting_set("default")
    _category_order = list(_active_set.category_order)
    try:
        if format == "html":
            out.write_text(
                apply_post_process_surgery(
                    out.read_text(),
                    navbar_text=_navbar,
                    category_order=_category_order,
                )
            )
        else:
            apply_post_process_surgery_to_zip(out, navbar_text=_navbar, category_order=_category_order)
    except Exception:
        pass
    if format != "html":
        return out
    out_html = out
    # Snap-confined browsers (Ubuntu Firefox snap) cannot read files under
    # ~/.cache/. If the rendered report lands there, surface a one-line
    # workaround so the user does not hit "Access to the file was denied".
    try:
        if "/.cache/" in str(out_html):
            print(
                f"[render_report] {out_html}\n"
                f"[render_report] Note: snap-confined browsers cannot read ~/.cache; "
                f"copy to ~/Downloads to view: cp {out_html} ~/Downloads/",
                flush=True,
            )
    except Exception:
        pass
    return out_html

reprex_bundle(output_path=None)

Emit a reprex-ready Workflow-Run-Crate bundle and return its extracted directory root (ADR-10, D3).

Peer of bundle_report_data(); opt-in only (never invoked from run()/submit_workflow()). emit_bundle already carries the reprex runnable-template set + WRC crate (Phase 2), so this facade emits the bundle and extracts the zip to a sibling directory so the round-trip consumes a directory root directly (Bundle.from_directory(...).reprex(...)). The HARD emit-time zero-user-info gate is deferred to the emit-hardening follow-up (its prerequisite); Bundle.reprex() runs a consume-side informational scan.

Returns: Path to the extracted reprex-bundle directory.

Source code in src/hhemt/analysis.py

def reprex_bundle(self, output_path: "Path | None" = None) -> "Path":
    """Emit a reprex-ready Workflow-Run-Crate bundle and return its extracted
    directory root (ADR-10, D3).

    Peer of ``bundle_report_data()``; opt-in only (never invoked from
    ``run()``/``submit_workflow()``). ``emit_bundle`` already carries the reprex
    runnable-template set + WRC crate (Phase 2), so this facade emits the bundle and
    extracts the zip to a sibling directory so the round-trip consumes a directory
    root directly (``Bundle.from_directory(...).reprex(...)``). The HARD emit-time
    zero-user-info gate is deferred to the emit-hardening follow-up (its
    prerequisite); ``Bundle.reprex()`` runs a consume-side informational scan.

    Returns:
        Path to the extracted reprex-bundle directory.
    """
    from hhemt.bundle import emit_bundle
    from hhemt.bundle._reprex import extract_reprex_bundle

    return extract_reprex_bundle(emit_bundle(self, output_path))

reprocess(start_with='consolidate', execution_mode='auto', which='both', *, regenerate_existing=False, delete_via_slurm=None, override_clear_raw='none', override_force_rerun=None, verbose=True, dry_run=False, prune_settled_markers=True, report_formats=None)

Re-run downstream stages against existing sim outputs.

Re-runs processing / consolidation / plotting / report rendering without re-running the simulation rules. Runs the Phase-1 reconciliation guard against _status/_submitted/ before submitting, so a parallel live sim driver cannot be double-submitted. Emits a scope-limited Snakefile at {analysis_dir}/Snakefile.reprocess and runs it against the shared .snakemake/ with --nolock; the _status/_orchestrator/ liveness gate (not the Snakemake lock) prevents collision with a live orchestration driver.

Parameters:

Name	Type	Description	Default
start_with	Literal['process', 'consolidate', 'render']	Stage to re-fire from. "consolidate" is the common case — re-aggregates the analysis datatree zarr and re-renders the report against existing sim outputs.	'consolidate'
execution_mode	Literal['auto', 'local', 'slurm']	"auto" (default) detects SLURM context; "local" / "slurm" force the mode.	'auto'
which	Literal['TRITON', 'SWMM', 'both']	"both" (default) / "TRITON" / "SWMM" — passes through to rule consolidate's --which flag.	'both'
regenerate_existing	bool	Default False. When False, reprocess regenerates ONLY report + plot artifacts against the EXISTING consolidated zarr — no zarr deletion, no DU restamp walk. When True, the legacy destructive delete-and-rebuild runs. A plain reprocess-only toggle (NOT override_-prefixed: bridges no config field; meaningless on run()). Matches the prune_settled_markers plain-bool precedent.	False
delete_via_slurm	bool | None	None (default) auto-resolves: route the opt-in deletion through the SLURM analysis.delete() architecture iff multi_sim_run_method is an HPC mode (batch_job / 1_job_many_srun_tasks); local → in-process fast_rmtree. Pass True / False to force. (CLI exposes --delete-via-slurm/--no-delete-via-slurm.)	None
override_clear_raw	ClearRawValue | None	Hard-default "none" to preserve historic reprocess semantics (reprocess never auto-clears unless the caller explicitly opts in). Pass None to read cfg_analysis.clear_raw; pass "all" / "none" / a list of model types to override. When the resolved value is anything other than "none", two guards must both pass: (a) every enabled sim's c_run_* flag must exist (no never-started sims); (b) no _status/_submitted/ sentinel may be present (no in-flight / just-died sims). Cites stipulation clear raw triton outputs deferred until last allocation (under library/docs/stipulations/hhemt/).	'none'
verbose	bool	If True, print progress messages.	True
dry_run	bool	If True, runs snakemake --dry-run only.	False
prune_settled_markers	bool	When True (default), prunes settled _status/_completed / _status/_failed markers (those whose _submitted/ sibling is gone) at the master _status/ level before submitting. Opt-out; mirrors run(). Inert hygiene — does not affect reconcile correctness.	True

Returns:

Type	Description
dict	Status dictionary from :meth:SnakemakeWorkflowBuilder.submit_reprocess_workflow.

Raises:

Type	Description
ConfigurationError	When the resolved clear_raw would clear and either guard fails.

Source code in src/hhemt/analysis.py

def reprocess(
    self,
    start_with: "Literal['process','consolidate','render']" = "consolidate",
    execution_mode: "Literal['auto','local','slurm']" = "auto",
    which: "Literal['TRITON','SWMM','both']" = "both",
    *,
    regenerate_existing: bool = False,
    delete_via_slurm: bool | None = None,
    override_clear_raw: ClearRawValue | None = "none",
    override_force_rerun: ForceRerunValue | None = None,
    verbose: bool = True,
    dry_run: bool = False,
    prune_settled_markers: bool = True,
    report_formats: list[Literal["html", "zip"]] | None = None,
) -> dict:
    """Re-run downstream stages against existing sim outputs.

    Re-runs processing / consolidation / plotting / report rendering
    without re-running the simulation rules. Runs the Phase-1
    reconciliation guard against ``_status/_submitted/`` before
    submitting, so a parallel live sim driver cannot be double-submitted.
    Emits a scope-limited Snakefile at
    ``{analysis_dir}/Snakefile.reprocess`` and runs it against the shared
    ``.snakemake/`` with ``--nolock``; the ``_status/_orchestrator/``
    liveness gate (not the Snakemake lock) prevents collision with a live
    orchestration driver.

    Parameters
    ----------
    start_with
        Stage to re-fire from. ``"consolidate"`` is the common case —
        re-aggregates the analysis datatree zarr and re-renders the
        report against existing sim outputs.
    execution_mode
        ``"auto"`` (default) detects SLURM context; ``"local"`` /
        ``"slurm"`` force the mode.
    which
        ``"both"`` (default) / ``"TRITON"`` / ``"SWMM"`` — passes through
        to ``rule consolidate``'s ``--which`` flag.
    regenerate_existing
        **Default False.** When False, reprocess regenerates ONLY report +
        plot artifacts against the EXISTING consolidated zarr — no zarr
        deletion, no DU restamp walk. When True, the legacy destructive
        delete-and-rebuild runs. A plain reprocess-only toggle (NOT
        ``override_``-prefixed: bridges no config field; meaningless on
        ``run()``). Matches the ``prune_settled_markers`` plain-bool
        precedent.
    delete_via_slurm
        **None (default) auto-resolves**: route the opt-in deletion through
        the SLURM ``analysis.delete()`` architecture iff
        ``multi_sim_run_method`` is an HPC mode (``batch_job`` /
        ``1_job_many_srun_tasks``); ``local`` → in-process ``fast_rmtree``.
        Pass ``True`` / ``False`` to force. (CLI exposes
        ``--delete-via-slurm/--no-delete-via-slurm``.)
    override_clear_raw
        **Hard-default "none"** to preserve historic ``reprocess`` semantics
        (reprocess never auto-clears unless the caller explicitly opts in).
        Pass ``None`` to read ``cfg_analysis.clear_raw``; pass ``"all"`` /
        ``"none"`` / a list of model types to override. When the resolved
        value is anything other than ``"none"``, two guards must both pass:
        (a) every enabled sim's ``c_run_*`` flag must exist (no
        never-started sims); (b) no ``_status/_submitted/`` sentinel
        may be present (no in-flight / just-died sims). Cites
        stipulation ``clear raw triton outputs deferred until last allocation``
        (under ``library/docs/stipulations/hhemt/``).
    verbose
        If True, print progress messages.
    dry_run
        If True, runs ``snakemake --dry-run`` only.
    prune_settled_markers
        When True (default), prunes settled ``_status/_completed`` /
        ``_status/_failed`` markers (those whose ``_submitted/`` sibling is
        gone) at the master ``_status/`` level before submitting. Opt-out;
        mirrors ``run()``. Inert hygiene — does not affect reconcile
        correctness.

    Returns
    -------
    dict
        Status dictionary from
        :meth:`SnakemakeWorkflowBuilder.submit_reprocess_workflow`.

    Raises
    ------
    ConfigurationError
        When the resolved ``clear_raw`` would clear and either guard fails.
    """
    resolved_clear_raw = override_clear_raw if override_clear_raw is not None else self.cfg_analysis.clear_raw
    # True iff the resolved value would trigger any cleanup for any model.
    would_clear = resolved_clear_raw != "none"
    # Lazy-stamp _version.json at LAYOUT_VERSION (PI-1 pattern, mirroring
    # run() and submit_workflow). Idempotent under concurrent writers;
    # if _version.json is missing or stamped at an older version, this
    # writes a fresh stamp at the current LAYOUT_VERSION.
    from hhemt.version_migration import LAYOUT_VERSION
    from hhemt.version_migration.state import stamp_new_target

    from .exceptions import ConfigurationError

    stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

    # Prune settled _status markers (opt-out) — inert hygiene, mirrors run().
    if prune_settled_markers:
        pruned = self._prune_settled_markers()
        if verbose and pruned:
            print(
                f"[prune-settled-markers] Pruned {len(pruned)} settled "
                f"marker(s) from {self.analysis_paths.analysis_dir}/_status/",
                flush=True,
            )

    # Dispatch to the sensitivity-master reprocess path for sensitivity-toggled
    # analyses. The non-sensitivity reprocess generator emits a `rule consolidate`
    # that consumes from `analysis_dir/sims/`, which for sensitivity layouts does
    # not exist — sims live under `subanalyses/sa_*/sims/`. The sensitivity-master
    # generator (SensitivityAnalysisWorkflowBuilder.generate_reprocess_master_snakefile_content)
    # emits per-sa consolidate rules + a master_consolidation rule that consume
    # from the correct paths. Pattern mirrors analysis.py:683-801 property
    # dispatches and the bundle CLI dispatch at cli.py:1026.
    if self.cfg_analysis.toggle_sensitivity_analysis:
        if would_clear:
            raise ConfigurationError(
                field="override_clear_raw",
                message=(
                    "TRITONSWMM_analysis.reprocess does not support clearing raw outputs "
                    "for sensitivity-toggled analyses (resolved clear_raw="
                    f"{resolved_clear_raw!r}). The sensitivity-master reprocess "
                    "path deliberately omits the clear-raw gate (see "
                    "TRITONSWMM_sensitivity_analysis.reprocess docstring). Invoke "
                    "self.sensitivity.reprocess(...) directly with explicit sa_ids if "
                    "raw-output clearing is required."
                ),
                config_path=str(self.analysis_config_yaml),
            )
        # R7 master-level in-flight guard (Phase 3) — refuse processed-output
        # deletion while live workers may be re-writing the master analysis's
        # processed dirs. Mirrors the non-sensitivity guard below. (Per-sub
        # subanalyses/sa_*/_status/_submitted/ recursion is a documented later
        # refinement; the conservative guard refuses on master presence.)
        if start_with == "process" and regenerate_existing and not dry_run:
            submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
            if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
                raise ConfigurationError(
                    field="regenerate_existing",
                    message=(
                        "reprocess refuses processed-output deletion "
                        "(start_with='process', regenerate_existing=True) while "
                        "_submitted/ sentinels are present in the sensitivity master "
                        "_status/ — simulations may still be in flight or recently "
                        "died and could be re-writing the same processed/ directories. "
                        "Run the reconciliation guard or `scancel` outstanding jobs first."
                    ),
                    config_path=str(self.analysis_config_yaml),
                )
        return self.sensitivity.reprocess(
            start_with=start_with,
            execution_mode=execution_mode,
            which=which,
            regenerate_existing=regenerate_existing,
            delete_via_slurm=delete_via_slurm,
            override_force_rerun=override_force_rerun,
            verbose=verbose,
            dry_run=dry_run,
            report_formats=report_formats,
        )

    if would_clear:
        # Guard (a): every enabled sim must have a c_run_* flag.
        if not self._all_sim_flags_present():
            raise ConfigurationError(
                field="override_clear_raw",
                message=(
                    "reprocess refuses raw-output clearing while c_run_* flags are absent "
                    f"(resolved clear_raw={resolved_clear_raw!r}; some sims have not completed). "
                    "See stipulation `clear raw triton outputs deferred until last allocation`."
                ),
                config_path=str(self.analysis_config_yaml),
            )
        # Guard (b): no in-flight or unreconciled _submitted/ sentinel.
        submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
        if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
            raise ConfigurationError(
                field="override_clear_raw",
                message=(
                    "reprocess refuses raw-output clearing while _submitted/ sentinels are present "
                    f"(resolved clear_raw={resolved_clear_raw!r}; simulations may still be in flight "
                    "or recently died). Run the Phase-1 reconciliation guard or `scancel` outstanding "
                    "jobs first."
                ),
                config_path=str(self.analysis_config_yaml),
            )

    # Reprocess overrides 1_job_many_srun_tasks → batch_job at submission
    # time. 1_job_many_srun_tasks reserves an exclusive multi-node SLURM
    # allocation that the downstream-only reprocess does not need, and the
    # method cannot decouple driver-cancel from job-cancel (master plan
    # Assumptions + FQ1 research). The override is local — the analysis's
    # original cfg_analysis.multi_sim_run_method is not mutated.
    effective_method: str | None = None
    if self.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks":
        effective_method = "batch_job"
        if verbose:
            print(
                "[reprocess] NOTE: 1_job_many_srun_tasks reprocess overridden to "
                "batch_job (per-rule sbatch). The original analysis_config is unchanged.",
                flush=True,
            )

    # Invalidate from start_with onward — deletes the upstream flag/artifact
    # that triggers Snakemake's mtime-driven re-fire. Per D-INVALIDATE
    # option 1: delete flags + rely on the generator's baked overwrite.
    # On dry_run, the flag deletion still happens (it is the cheap,
    # rerun-recreated trigger that makes the --dry-run DAG meaningful);
    # only the destructive consolidated-zarr deletion + DU restamp are
    # skipped (see `reprocess dry_run performs no destructive mutation`
    # stipulation).
    # Opt-in processed-output deletion (rebuild-from-raw, FQ2). Refuse while
    # live sim workers may be re-writing the same processed/ dir — mirrors
    # the clear-raw in-flight guard (analysis.py:2607-2619). reprocess
    # coexists with live workers generally, but DELETING the artifact a live
    # worker is writing is unsafe.
    if start_with == "process" and regenerate_existing and not dry_run:
        submitted_dir = self.analysis_paths.analysis_dir / "_status" / "_submitted"
        if submitted_dir.exists() and any(submitted_dir.glob("*.json")):
            raise ConfigurationError(
                field="regenerate_existing",
                message=(
                    "reprocess refuses processed-output deletion "
                    "(start_with='process', regenerate_existing=True) while "
                    "_submitted/ sentinels are present — simulations may still be "
                    "in flight or recently died and could be re-writing the same "
                    "processed/ directories. Run the reconciliation guard or "
                    "`scancel` outstanding jobs first."
                ),
                config_path=str(self.analysis_config_yaml),
            )

    # R8 routing — computed ONCE; shared by both deletion sites. None
    # auto-resolves to slurm-offload on HPC modes (user D6 refinement 1).
    _hpc = self.cfg_analysis.multi_sim_run_method in ("batch_job", "1_job_many_srun_tasks")
    _resolved_delete_via_slurm = _hpc if delete_via_slurm is None else delete_via_slurm
    route_delete_via_slurm = regenerate_existing and _resolved_delete_via_slurm and not dry_run and _hpc
    # Divergence self-heal (FIX 2) — fires on the process path REGARDLESS of
    # regenerate_existing (D2). Reconciles d_process flag + per-model
    # processing_log against on-disk summary presence: where a flag survives
    # but the enabled-model summary set is absent (the May-31 divergence),
    # unlink the flag + clear the log so the workflow.py:6684 emit gate
    # re-emits the process rule and _already_written (Gotcha 28) lets it
    # write. No-op when every enabled summary is present (healthy analysis).
    if start_with == "process" and not dry_run:
        _reconciled = self._reconcile_stale_process_flags_against_summaries()
        self._assert_reprocess_rebuild_sources_present(_reconciled)
    if route_delete_via_slurm:
        # ONE scoped reprocess-delete workflow handles BOTH the consolidated
        # zarr(s) AND (start_with=='process') the per-scenario processed/ dirs.
        self._workflow_builder.submit_reprocess_delete_workflow(
            start_with=start_with,
            override_in_flight=False,
        )
    self._invalidate_downstream_flags(
        start_with,
        regenerate_existing=regenerate_existing,
        dry_run=dry_run,
        skip_destructive_delete=route_delete_via_slurm,
    )

    # Force-rerun pre-delete (login-node responsibility). Resolve +
    # validate + delete BEFORE Snakemake plans the reprocess DAG. Per
    # cleanup-rerun-delete-redesign Phase 4 + R10. Skipped on dry_run —
    # it deletes flags and clears per-scenario processing-log records,
    # both filesystem mutations that the dry-run no-destructive-mutation
    # contract forbids.
    if not dry_run:
        self._apply_force_rerun(override_force_rerun)

    # Processed-output deletion (Phase 3). The per-model PROCESSING-LOG
    # clear (the _already_written invalidation, Gotcha #28) is CHEAP
    # (per-scenario JSON rewrites, no GPFS tree walk) and MUST run on
    # both routes: the SLURM runner deletes processed/ but never clears
    # log_{model_type}.json, so without this the rebuilt process rule
    # would emit but _already_written would skip every _export_* write.
    # The HEAVY processed/+zarr fast_rmtree stays SLURM-routed.
    if route_delete_via_slurm:
        # SLURM deleted the artifacts; clear only the per-model log here.
        if start_with == "process" and regenerate_existing and not dry_run:
            from hhemt.workflow import ResolvedForceRerunSpec

            self._invalidate_processing_log_for_force_rerun(ResolvedForceRerunSpec(scope="all", tokens=()))
    else:
        self._delete_processed_outputs_for_reprocess(
            start_with, regenerate_existing=regenerate_existing, dry_run=dry_run
        )

    # Delegate to the workflow builder. The submit method writes the
    # reprocess Snakefile and orchestrates the snakemake invocation with
    # `--snakefile Snakefile.reprocess --rerun-triggers mtime --nolock`
    # against the shared analysis_dir/.snakemake/; the
    # _status/_orchestrator/ liveness gate is the concurrency authority.
    result = self._workflow_builder.submit_reprocess_workflow(
        start_with=start_with,
        execution_mode=execution_mode,
        multi_sim_run_method_override=effective_method,
        dry_run=dry_run,
        verbose=verbose,
    )
    return result

retrieve_prepare_scenario_launchers(overwrite_scenario_if_already_set_up=False, rerun_swmm_hydro_if_outputs_exist=False, verbose=False)

Create subprocess-based launchers for scenario preparation.

Each launcher runs scenario preparation in an isolated subprocess to avoid PySwmm's MultiSimulationError when preparing multiple scenarios concurrently.

Parameters:

Name	Type	Description	Default
overwrite_scenario_if_already_set_up	bool	If True, overwrite existing scenarios	False
rerun_swmm_hydro_if_outputs_exist	bool	If True, rerun SWMM hydrology model even if outputs exist	False
verbose	bool	If True, print progress messages	False

Returns:

Type	Description
list	List of launcher functions that execute scenario preparation in subprocesses

Source code in src/hhemt/analysis.py

def retrieve_prepare_scenario_launchers(
    self,
    overwrite_scenario_if_already_set_up: bool = False,
    rerun_swmm_hydro_if_outputs_exist: bool = False,
    verbose: bool = False,
):
    """
    Create subprocess-based launchers for scenario preparation.

    Each launcher runs scenario preparation in an isolated subprocess to avoid
    PySwmm's MultiSimulationError when preparing multiple scenarios concurrently.

    Parameters
    ----------
    overwrite_scenario_if_already_set_up : bool
        If True, overwrite existing scenarios
    rerun_swmm_hydro_if_outputs_exist : bool
        If True, rerun SWMM hydrology model even if outputs exist
    verbose : bool
        If True, print progress messages

    Returns
    -------
    list
        List of launcher functions that execute scenario preparation in subprocesses
    """
    prepare_scenario_launchers = []
    for event_iloc in self.df_sims.index:
        scen = TRITONSWMM_scenario(event_iloc, self)

        # Create a subprocess-based launcher
        launcher = scen._create_subprocess_prepare_scenario_launcher(
            overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
            rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
            verbose=verbose,
        )
        prepare_scenario_launchers.append(launcher)

    return prepare_scenario_launchers

retrieve_scenario_timeseries_processing_launchers(which='both', *, override_clear_raw=None, verbose=False, compression_level=5)

Create subprocess-based launchers for scenario timeseries processing.

Each launcher runs timeseries processing in an isolated subprocess to avoid potential conflicts when processing multiple scenarios' outputs concurrently.

Parameters:

Name	Type	Description	Default
which	Literal['TRITON', 'SWMM', 'both']	Which outputs to process: TRITON, SWMM, or both	'both'
override_clear_raw	ClearRawValue | None	Runtime override for cfg_analysis.clear_raw. None (the default) reads from the YAML config; a concrete value overrides for this run.	None
verbose	bool	If True, print progress messages	False
compression_level	int	Compression level for output files (0-9)	5

Returns:

Type	Description
list	List of launcher functions that execute timeseries processing in subprocesses

Source code in src/hhemt/analysis.py

def retrieve_scenario_timeseries_processing_launchers(
    self,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    *,
    override_clear_raw: ClearRawValue | None = None,
    verbose: bool = False,
    compression_level: int = 5,
):
    """
    Create subprocess-based launchers for scenario timeseries processing.

    Each launcher runs timeseries processing in an isolated subprocess to avoid
    potential conflicts when processing multiple scenarios' outputs concurrently.

    Parameters
    ----------
    which : Literal["TRITON", "SWMM", "both"]
        Which outputs to process: TRITON, SWMM, or both
    override_clear_raw : ClearRawValue | None
        Runtime override for ``cfg_analysis.clear_raw``. ``None`` (the default)
        reads from the YAML config; a concrete value overrides for this run.
    verbose : bool
        If True, print progress messages
    compression_level : int
        Compression level for output files (0-9)

    Returns
    -------
    list
        List of launcher functions that execute timeseries processing in subprocesses
    """
    scenario_timeseries_processing_launchers = []
    for event_iloc in self.df_sims.index:
        proc = self._retrieve_sim_run_processing_object(event_iloc=event_iloc)

        # Create a subprocess-based launcher
        launcher = proc._create_subprocess_timeseries_processing_launcher(
            which=which,
            override_clear_raw=override_clear_raw,
            verbose=verbose,
            compression_level=compression_level,
        )
        scenario_timeseries_processing_launchers.append(launcher)

    return scenario_timeseries_processing_launchers

run(from_scratch=False, dry_run=False, events=None, execution_mode='auto', verbose=True, wait_for_job_completion=None, override_clear_raw=None, override_force_rerun=None, override_hpc_total_nodes=None, override_hpc_restart_times_simulate=None, override_hpc_restart_times_other=None, override_pickup_where_leftoff=None, transfer_config=None, report_config=None, override_brand_theme=None, report_formats=None, cleanup_orphans=False, cleanup_stale_metadata=True, prune_settled_markers=True, extra_sbatch_args=None, snakemake_diagnostics=None)

High-level orchestration method for running TRITON-SWMM workflows.

Parameters:

Name	Type	Description	Default
from_scratch	bool	If True, delete all analysis artifacts and start fresh. If False (default), resume from last completed checkpoint.	False
dry_run	bool	If True, validate workflow but don't execute.	False
events	list[int] | None	Subset of event_ilocs to process. If None, processes all events.	None
execution_mode	Literal['auto', 'local', 'slurm']	Where to execute: auto-detect (default), force local, or force SLURM.	'auto'
verbose	bool	If True, print progress messages.	True
override_clear_raw	ClearRawValue | None	Runtime override for cfg_analysis.clear_raw. None (default) reads the YAML; pass "none" / "all" / a list of model types (e.g. ["tritonswmm", "swmm"]) to override for this invocation. Per the override_ prefix convention introduced by cleanup-rerun-delete-redesign Phase 1.	None
wait_for_job_completion	bool | None	If True, block until the SLURM job finishes. Mainly for tests.	None
override_hpc_total_nodes	int | None	Overrides hpc_total_nodes in the SBATCH script without mutating the config. Only valid for multi_sim_run_method="1_job_many_srun_tasks".	None
transfer_config	PostRunTransferConfig | None	If provided, automatically transfer results to the local machine after successful completion (requires wait_for_job_completion=True).	None
cleanup_orphans	bool	When True, deletes orphan sub-analysis artifacts (subanalysis dirs, status flags, sensitivity_datatree.zarr groups) detected when an sa_id is removed from the sensitivity CSV/XLSX. Opt-in because the blast radius is irrecoverable (subanalysis data deleted).	False
cleanup_stale_metadata	bool	When True (default), subprocess-invokes snakemake --cleanup-metadata against orphaned .snakemake/metadata/ records left by past rule-output renames (e.g., Phase 8's .png/.svg → .html flip). When False, skips the cleanup; users may experience a one-shot full plot rebuild on first post-rename invocation per Phase 8 Risks. Asymmetric with cleanup_orphans default (False) because metadata-cleanup blast radius is bounded to records, not data; safe to auto-apply.	True
prune_settled_markers	bool	When True (default), prunes settled _status/_completed and _status/_failed markers (a marker is settled when its sibling _status/_submitted/{token}.json is absent — pure accumulation the reconcile will never re-read). Opt-out, mirroring cleanup_stale_metadata: the blast radius is bounded to inert settled markers, so auto-applying is safe and bounds unbounded marker growth over long resumable campaigns.	True
extra_sbatch_args	list[str] | None	Optional list of additional SBATCH directive strings (e.g., ["--qos=debug"] to route the job to Frontier's debug queue) to append to the generated run_workflow_1job.sh script. Each list element is emitted as one #SBATCH <element> line, after every other source of #SBATCH directives in the script — both the always-emitted directives derived from config fields (--job-name, --partition from cfg_analysis.hpc_ensemble_partition, --account from cfg_analysis.hpc_account, --nodes from cfg_analysis.hpc_total_nodes (or the override_hpc_total_nodes runtime kwarg), --exclusive, --gres from cfg_analysis.hpc_gpus_per_node + cfg_system.gpu_hardware, --time from cfg_analysis.hpc_total_job_duration_min, --output, --error) AND the directives in the cfg_analysis.additional_SBATCH_params config list. Override behavior: any flag in extra_sbatch_args that matches a flag emitted earlier in the script — whether that earlier directive came from a top-level cfg_analysis field (hpc_ensemble_partition, hpc_account, hpc_total_nodes, hpc_total_job_duration_min, hpc_gpus_per_node, etc.) or from the cfg_analysis.additional_SBATCH_params list — WILL OVERRIDE the config-derived value via SLURM's last-directive-wins parser semantics. When such an override is detected, an informational [extra_sbatch_args] OVERRIDE: ... message is printed naming the flag, the origin of the original value (e.g. cfg_analysis.hpc_ensemble_partition), and the new runtime value, so the user can confirm the override took effect as intended. Only valid for multi_sim_run_method="1_job_many_srun_tasks"; raises ConfigurationError otherwise (a fail-fast guard preventing the user from believing they are controlling the experiment when the kwarg would silently no-op in another mode).	None

Returns:

Type	Description
WorkflowResult	Structured result object with success status and execution details.

Examples:

Resume (default):

>>> result = analysis.run()

Fresh start:

>>> result = analysis.run(from_scratch=True)

Dry-run validation:

>>> result = analysis.run(dry_run=True, verbose=True)

Auto-transfer after completion:

>>> from hhemt.config.globus import PostRunTransferConfig
>>> result = analysis.run(
...     wait_for_job_completion=True,
...     transfer_config=PostRunTransferConfig(
...         destination_root=r"D:\Dropbox\results",
...         system="frontier",
...     ),
... )

See Also

submit_workflow : Lower-level workflow submission (15+ parameters) transfer_results : Standalone transfer method

Source code in src/hhemt/analysis.py

def run(
    self,
    from_scratch: bool = False,
    dry_run: bool = False,
    events: list[int] | None = None,
    execution_mode: Literal["auto", "local", "slurm"] = "auto",
    verbose: bool = True,
    wait_for_job_completion: bool | None = None,
    override_clear_raw: ClearRawValue | None = None,
    override_force_rerun: ForceRerunValue | None = None,
    override_hpc_total_nodes: int | None = None,
    override_hpc_restart_times_simulate: int | None = None,
    override_hpc_restart_times_other: int | None = None,
    override_pickup_where_leftoff: bool | None = None,
    transfer_config: "PostRunTransferConfig | None" = None,
    report_config: "Path | None" = None,
    override_brand_theme: "Path | None" = None,
    report_formats: list[Literal["html", "zip"]] | None = None,
    cleanup_orphans: bool = False,
    cleanup_stale_metadata: bool = True,
    prune_settled_markers: bool = True,
    extra_sbatch_args: list[str] | None = None,
    snakemake_diagnostics: SnakemakeDiagnostics | None = None,
) -> "WorkflowResult":
    """
    High-level orchestration method for running TRITON-SWMM workflows.

    Parameters
    ----------
    from_scratch : bool
        If True, delete all analysis artifacts and start fresh.
        If False (default), resume from last completed checkpoint.
    dry_run : bool
        If True, validate workflow but don't execute.
    events : list[int] | None
        Subset of event_ilocs to process. If None, processes all events.
    execution_mode : Literal["auto", "local", "slurm"]
        Where to execute: auto-detect (default), force local, or force SLURM.
    verbose : bool
        If True, print progress messages.
    override_clear_raw : ClearRawValue | None
        Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
        reads the YAML; pass ``"none"`` / ``"all"`` / a list of model types
        (e.g. ``["tritonswmm", "swmm"]``) to override for this invocation.
        Per the ``override_`` prefix convention introduced by
        cleanup-rerun-delete-redesign Phase 1.
    wait_for_job_completion : bool | None
        If True, block until the SLURM job finishes. Mainly for tests.
    override_hpc_total_nodes : int | None
        Overrides `hpc_total_nodes` in the SBATCH script without mutating
        the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.
    transfer_config : PostRunTransferConfig | None
        If provided, automatically transfer results to the local machine
        after successful completion (requires ``wait_for_job_completion=True``).
    cleanup_orphans : bool, default False
        When True, deletes orphan sub-analysis artifacts (subanalysis dirs,
        status flags, sensitivity_datatree.zarr groups) detected when an
        ``sa_id`` is removed from the sensitivity CSV/XLSX. Opt-in because
        the blast radius is irrecoverable (subanalysis data deleted).
    cleanup_stale_metadata : bool, default True
        When True (default), subprocess-invokes ``snakemake --cleanup-metadata``
        against orphaned ``.snakemake/metadata/`` records left by past
        rule-output renames (e.g., Phase 8's ``.png``/``.svg`` → ``.html``
        flip). When False, skips the cleanup; users may experience a
        one-shot full plot rebuild on first post-rename invocation per
        Phase 8 Risks. Asymmetric with ``cleanup_orphans`` default (False)
        because metadata-cleanup blast radius is bounded to records, not
        data; safe to auto-apply.
    prune_settled_markers : bool, default True
        When True (default), prunes settled ``_status/_completed`` and
        ``_status/_failed`` markers (a marker is settled when its sibling
        ``_status/_submitted/{token}.json`` is absent — pure accumulation the
        reconcile will never re-read). Opt-out, mirroring
        ``cleanup_stale_metadata``: the blast radius is bounded to inert
        settled markers, so auto-applying is safe and bounds unbounded marker
        growth over long resumable campaigns.
    extra_sbatch_args : list[str] | None
        Optional list of additional SBATCH directive strings (e.g.,
        ``["--qos=debug"]`` to route the job to Frontier's debug queue) to
        append to the generated ``run_workflow_1job.sh`` script. Each list
        element is emitted as one ``#SBATCH <element>`` line, after every
        other source of ``#SBATCH`` directives in the script — both the
        always-emitted directives derived from config fields
        (``--job-name``, ``--partition`` from
        ``cfg_analysis.hpc_ensemble_partition``, ``--account`` from
        ``cfg_analysis.hpc_account``, ``--nodes`` from
        ``cfg_analysis.hpc_total_nodes`` (or the
        ``override_hpc_total_nodes`` runtime kwarg), ``--exclusive``,
        ``--gres`` from ``cfg_analysis.hpc_gpus_per_node`` +
        ``cfg_system.gpu_hardware``, ``--time`` from
        ``cfg_analysis.hpc_total_job_duration_min``, ``--output``,
        ``--error``) AND the directives in the
        ``cfg_analysis.additional_SBATCH_params`` config list.

        **Override behavior**: any flag in ``extra_sbatch_args`` that
        matches a flag emitted earlier in the script — whether that earlier
        directive came from a top-level ``cfg_analysis`` field
        (``hpc_ensemble_partition``, ``hpc_account``, ``hpc_total_nodes``,
        ``hpc_total_job_duration_min``, ``hpc_gpus_per_node``, etc.) or
        from the ``cfg_analysis.additional_SBATCH_params`` list — WILL
        OVERRIDE the config-derived value via SLURM's last-directive-wins
        parser semantics. When such an override is detected, an
        informational ``[extra_sbatch_args] OVERRIDE: ...`` message is
        printed naming the flag, the origin of the original value
        (e.g. ``cfg_analysis.hpc_ensemble_partition``), and the new
        runtime value, so the user can confirm the override took effect
        as intended.

        Only valid for ``multi_sim_run_method="1_job_many_srun_tasks"``;
        raises ``ConfigurationError`` otherwise (a fail-fast guard
        preventing the user from believing they are controlling the
        experiment when the kwarg would silently no-op in another mode).

    Returns
    -------
    WorkflowResult
        Structured result object with success status and execution details.

    Examples
    --------
    Resume (default):

    >>> result = analysis.run()

    Fresh start:

    >>> result = analysis.run(from_scratch=True)

    Dry-run validation:

    >>> result = analysis.run(dry_run=True, verbose=True)

    Auto-transfer after completion:

    >>> from hhemt.config.globus import PostRunTransferConfig
    >>> result = analysis.run(
    ...     wait_for_job_completion=True,
    ...     transfer_config=PostRunTransferConfig(
    ...         destination_root=r"D:\\Dropbox\\results",
    ...         system="frontier",
    ...     ),
    ... )

    See Also
    --------
    submit_workflow : Lower-level workflow submission (15+ parameters)
    transfer_results : Standalone transfer method
    """
    # TODO - if from_scratch = True, user should be prompted for manual input to
    # type something like 'y' 'yes' or 'proceed' if the status of the
    # analysis shows that some steps have been completed. This should be
    # accompanied by a print statement of the current status.

    import time

    from .config.loaders import load_brand_theme, yaml_to_model
    from .config.report import (
        report_config as ReportConfigModel,
    )
    from .config.report import (
        validate_active_reporting_set,
    )
    from .exceptions import ConfigurationError
    from .orchestration import WorkflowResult, translate_mode, translate_phases
    from .report_renderers._reporting_sets import get_reporting_set

    # _test/ deletion offer (R9): if a leftover smoke-test subtree from a
    # prior analysis.test() exists, offer to delete it before the real run.
    # _test/ is excluded from the analysis _du.json (Phase 2 du change), so
    # size it directly via du_sentinels._walk_root_bytes rather than via
    # self.disk_utilization_bytes. Mirrors the Globus-conflict prompt's
    # sys.stdin.isatty() non-TTY guard (A6); adds a 15 s select-based
    # auto-'no' so an unattended run is never blocked. `select`/`sys` and
    # `du_sentinels` are imported at module top.
    test_dir = self.analysis_paths.analysis_dir / "_test"
    if test_dir.exists():
        size_bytes, _walk_errors = du_sentinels._walk_root_bytes(test_dir)
        size_mb = size_bytes / (1024 * 1024)
        if not sys.stdin.isatty():
            print(
                f"[test] Existing _test/ ({size_mb:.1f} MB) left in place (non-interactive stdin — not prompting).",
                flush=True,
            )
        else:
            print(
                f"[test] An existing _test/ smoke-test subtree ({size_mb:.1f} MB) "
                f"was found at {test_dir}. Delete it? [y/N] (auto-'no' in 15s): ",
                end="",
                flush=True,
            )
            ready, _, _ = select.select([sys.stdin], [], [], 15)
            answer = sys.stdin.readline().strip().lower() if ready else "n"
            if answer in ("y", "yes"):
                fast_rmtree(test_dir, analysis_dir=self.analysis_paths.analysis_dir)
                print(f"[test] Deleted {test_dir}.", flush=True)
            else:
                print("[test] Keeping _test/.", flush=True)

    # Pre-run report_config resolution (post-F2 v2 — 2-step, fail-fast).
    # Resolution order:
    #   (a) explicit `report_config=` argument → load and use
    #   (b) self.cfg_analysis.report (guaranteed non-None by R1 — required
    #       Pydantic field; loading a cfg_analysis.yaml without `report:`
    #       raises ValidationError before this code is reached)
    # No DEFAULT_REPORT_CONFIG fallback — the field is required.
    if report_config is not None:
        report_config = Path(report_config)
        try:
            cfg_report = yaml_to_model(report_config, ReportConfigModel)
        except Exception as e:
            raise ConfigurationError(
                field="report_config",
                message=f"Failed to load/validate {report_config}: {e}",
                config_path=report_config,
            ) from e
    else:
        cfg_report = self.cfg_analysis.report

    sa_csv = self.cfg_analysis.sensitivity_analysis if self.cfg_analysis.toggle_sensitivity_analysis else None
    _resolved_set_name = validate_active_reporting_set(
        cfg_report,
        is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
        sensitivity_csv_path=sa_csv,
    )
    self._active_reporting_set_name = _resolved_set_name
    self._active_reporting_set = get_reporting_set(_resolved_set_name)
    self._cfg_report = cfg_report

    # ADR-17: non-blocking invalidating-fix registry emission at run()-entry.
    # Prints a console warning for any registered calculation-invalidating fix
    # that affects this analysis's ADR-15-stamped scopes, reflecting the CURRENT
    # registry even when the persisted validation_report.json is stale (OE-3).
    # NEVER raises (load-time emission is non-blocking by construction,
    # C-NON-BLOCKING-BUGEMIT); the durable report surface is
    # check_invalidating_fixes in validate_analysis. Graceful-absent: an absent
    # registry or an unstamped/unconsolidated tree prints nothing.
    try:
        from .recompute import match_registry_against_stamps

        _fix_matches = match_registry_against_stamps(self)
        if _fix_matches:
            print(
                f"[hhemt] WARNING: {len(_fix_matches)} registered "
                "calculation-invalidating fix(es) affect this analysis:",
                flush=True,
            )
            for _m in _fix_matches:
                _action = _m.recommended_action.value if _m.recommended_action else "-"
                print(
                    f"  - {_m.commit_id} ({_m.severity}) -> recommended: "
                    f"{_action} [scope {_m.affected_scope}]",
                    flush=True,
                )
            print(
                "  Run `check-invalidating-fixes` for details; this is non-blocking.",
                flush=True,
            )
    except Exception:
        pass  # load-time emission is strictly non-blocking — never crash run()

    # Pre-run brand-theme resolution (ADR-7 layer 2 — 3-step, fail-fast).
    from .config.brand_theme import DEFAULT_BRAND_THEME
    from .config.brand_theme import brand_theme as BrandThemeModel

    if override_brand_theme is not None:
        override_brand_theme = Path(override_brand_theme)
        try:
            resolved_theme = yaml_to_model(override_brand_theme, BrandThemeModel)
        except Exception as e:
            raise ConfigurationError(
                field="brand_theme",
                message=f"Failed to load/validate {override_brand_theme}: {e}",
                config_path=override_brand_theme,
            ) from e
    elif self.cfg_analysis.brand_theme is not None:
        resolved_theme = load_brand_theme(self.cfg_analysis.brand_theme)
    else:
        resolved_theme = DEFAULT_BRAND_THEME
    self._brand_theme = resolved_theme

    # D-5: source the HTML-table brand-derived defaults from the resolved
    # theme via model_validate overlay (NOT setattr — per the per-row-config-
    # overlay stipulation). Semantic pass/fail + th/body text colors stay frozen.
    _t = self._brand_theme
    _table_overlay = {
        "primary_color": _t.primary_color,
        "cell_border_color": _t.neutral_medium,
        "row_alt_bg_color": _t.neutral_light,
        "row_hover_bg_color": _t.accent_color,
    }
    self._cfg_report = type(self._cfg_report).model_validate(
        {
            **self._cfg_report.model_dump(),
            "errors_and_warnings": {
                **self._cfg_report.errors_and_warnings.model_dump(),
                **_table_overlay,
            },
            "scenario_status_appendix": {
                **self._cfg_report.scenario_status_appendix.model_dump(),
                **_table_overlay,
            },
        }
    )

    # Pre-run transfer validation — fail fast before submitting the workflow
    if transfer_config is not None:
        transfer_config.to_transfer_spec(
            analysis_dir=self.analysis_paths.analysis_dir,
            analysis_id=self.cfg_analysis.analysis_id,
        )

    start_time = time.time()

    # Event filtering not yet implemented - validate parameter
    if events is not None:
        raise NotImplementedError(
            "Event filtering via events parameter not yet implemented. "
            "For now, all events in analysis will be processed."
        )

    if from_scratch and not dry_run:
        # remove analysis folder. Use the DERIVED analysis_paths.analysis_dir
        # (never None) — NOT the raw cfg_analysis.analysis_dir Optional field,
        # which defaults None and made fast_rmtree(None) crash here. Every
        # other analysis_dir reference in this module already uses the derived
        # path; this site was the lone holdout.
        # EXEMPT-DU: full-analysis-root-wipe
        fast_rmtree(self.analysis_paths.analysis_dir)

    # Orphan detection gate (sensitivity-only; non-sensitivity covered by
    # follow-up plan per D-EVENT-PARITY).
    if not from_scratch and self.cfg_analysis.toggle_sensitivity_analysis and not self.cfg_analysis.is_subanalysis:
        from hhemt.exceptions import ConfigurationError as _CfgErr

        _dirs = self.sensitivity.find_orphan_subanalysis_dirs()
        _flags = self.sensitivity.find_orphan_status_flags()
        _groups = self.sensitivity.find_orphan_datatree_groups()
        _fingerprints = self.sensitivity.find_orphan_input_fingerprints()
        _has_orphans = bool(_dirs or _flags or _groups or _fingerprints)
        if _has_orphans and not cleanup_orphans:
            raise _CfgErr(
                field="cleanup_orphans",
                message=(
                    "Detected orphan sub-analysis artifacts on disk that are "
                    "absent from the current sensitivity CSV: "
                    f"{len(_dirs)} subanalysis dir(s), "
                    f"{len(_flags)} _status flag(s), "
                    f"{len(_groups)} datatree group(s). "
                    f"{len(_fingerprints)} input-fingerprint(s). "
                    "Re-invoke analysis.run(cleanup_orphans=True) to delete them, "
                    "or run `triton-swmm cleanup-orphans --apply --force` from the CLI."
                ),
                config_path=str(self.analysis_config_yaml),
            )
        if _has_orphans and cleanup_orphans:
            self.sensitivity.cleanup_all_orphans(
                dry_run=dry_run,
                force=True,
                verbose=verbose,
            )

    # Stale-metadata cleanup gate — analysis-level, not sensitivity-specific
    # (per Phase 8.5 of interactive_report_renderers plan). Asymmetric with
    # cleanup_orphans default: cleanup_stale_metadata defaults to True
    # because metadata-cleanup blast radius is bounded to .snakemake/metadata/
    # records — no data is deleted; worst-case auto-apply result is the same
    # one-shot full plot rebuild Phase 8 Risks documents.
    # Precondition for the subprocess invocation: `snakemake
    # --cleanup-metadata` requires BOTH a Snakefile in the working
    # directory (to interpret path arguments) AND a
    # `.snakemake/metadata/` directory (the records to clean). The
    # Snakefile is generated by `submit_workflow()` later in this
    # method, so at this gate site it exists only on resumed analyses
    # (the use case cleanup_stale_metadata targets — fresh analyses
    # have no stale metadata to clean).
    _snakefile = self.analysis_paths.analysis_dir / "Snakefile"
    _metadata_dir = self.analysis_paths.analysis_dir / ".snakemake" / "metadata"
    if cleanup_stale_metadata and not from_scratch and _snakefile.exists() and _metadata_dir.exists():
        orphan_paths = self._enumerate_stale_metadata_paths()
        if orphan_paths:
            if verbose:
                print(
                    f"[cleanup-stale-metadata] Cleaning {len(orphan_paths)} "
                    f"orphan metadata record(s) from "
                    f"{self.analysis_paths.analysis_dir}/.snakemake/metadata/",
                    flush=True,
                )
                for p in orphan_paths:
                    print(f"  orphan: {p}", flush=True)
            self._invoke_snakemake_cleanup_metadata(orphan_paths)

    # Prune settled _status/_completed and _status/_failed markers (opt-out).
    # A settled marker is inert (its _submitted/ sibling is gone, so the
    # reconcile will never re-read it); pruning bounds unbounded marker
    # accumulation over long resumable campaigns. Independent of the reconcile
    # second-pass — settled markers are by definition not in-flight.
    if prune_settled_markers:
        pruned = self._prune_settled_markers()
        if verbose and pruned:
            print(
                f"[prune-settled-markers] Pruned {len(pruned)} settled "
                f"marker(s) from {self.analysis_paths.analysis_dir}/_status/",
                flush=True,
            )

    # Stamp _version.json at LAYOUT_VERSION on first materialization (lazy
    # stamp per version_migration_system master plan PI-1). Idempotent
    # under concurrent writers.
    from hhemt.version_migration import LAYOUT_VERSION
    from hhemt.version_migration.state import stamp_new_target

    stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

    # Translate user-friendly parameters to workflow parameters
    # from_scratch's fast_rmtree(analysis_dir) above is the fresh mechanism for the
    # per-analysis tier; the "fresh" mode params additionally force a re-derive of the
    # SHARED system tier (DEM/Manning's in system_directory/) via overwrite_system_inputs.
    # The other four mode flags are moot post-wipe (no scenarios/checkpoints survive).
    mode_params = translate_mode("fresh" if from_scratch else "resume")
    phase_params = translate_phases(None)  # TODO - hardcoded while troubleshooting

    # Detect system input processing needs

    swmm_used = False
    triton_used = False
    for model_used in self._get_enabled_model_types():
        if "swmm" in model_used.lower():
            swmm_used = True
        if "triton" in model_used.lower():
            triton_used = True
    if swmm_used and triton_used:
        which = "both"
    elif swmm_used and not triton_used:
        which = "SWMM"
    else:
        which = "TRITON"

    # Determine execution mode
    if execution_mode == "auto":
        if (
            self.in_slurm
            or self.cfg_analysis.multi_sim_run_method == "1_job_many_srun_tasks"
            or self.cfg_analysis.multi_sim_run_method == "batch_job"
        ):
            exec_mode = "slurm"
        else:
            exec_mode = "local"
    else:
        exec_mode = execution_mode

    if wait_for_job_completion is None:
        wait_for_job_completion = exec_mode != "slurm"

    # Build complete parameter dict for submit_workflow
    workflow_params = {
        **mode_params,
        **phase_params,
        "mode": exec_mode,
        "which": which,
        "override_clear_raw": override_clear_raw,
        "override_force_rerun": override_force_rerun,
        "compression_level": 5,
        "wait_for_completion": wait_for_job_completion,
        "dry_run": dry_run,
        "verbose": verbose,
        "override_hpc_total_nodes": override_hpc_total_nodes,
        "override_hpc_restart_times_simulate": override_hpc_restart_times_simulate,
        "override_hpc_restart_times_other": override_hpc_restart_times_other,
        "report_formats": report_formats,
        "extra_sbatch_args": extra_sbatch_args,
        "snakemake_diagnostics": snakemake_diagnostics,
    }
    # override_pickup_where_leftoff decouples resume-on-retry from the mode:
    # translate_mode("fresh") (from_scratch=True) sets pickup_where_leftoff=False,
    # but a resume EXPERIMENT wants a fresh wipe AND within-run hotstart-resume (the
    # wipe is once at run-start, before any checkpoint exists; pickup governs the
    # Snakemake-retry behavior AFTER checkpoints are written). None = use the
    # mode-derived value (no behavior change for existing callers).
    if override_pickup_where_leftoff is not None:
        workflow_params["pickup_where_leftoff"] = override_pickup_where_leftoff

    if verbose:
        print("Submitting workflow with args:")
        print(workflow_params)

    # Call underlying submit_workflow
    result_dict = self.submit_workflow(**workflow_params)

    # Calculate execution time
    elapsed = time.time() - start_time

    # Determine which phases were completed based on parameters
    phases_completed = []
    if workflow_params["process_system_level_inputs"] or workflow_params["compile_TRITON_SWMM"]:
        phases_completed.append("setup")
    if workflow_params["prepare_scenarios"]:
        phases_completed.append("prepare")
    if workflow_params["prepare_scenarios"]:  # Simulate always runs if scenarios prepared
        phases_completed.append("simulate")
    if workflow_params["process_timeseries"]:
        phases_completed.append("process")
    if workflow_params["process_timeseries"]:  # Consolidate happens after processing
        phases_completed.append("consolidate")

    # Get event list (all events in analysis)
    events_processed = list(self.df_sims.index)

    # Post-completion auto-transfer
    if transfer_config is not None and wait_for_job_completion and result_dict.get("success", False):
        if verbose:
            print("[Transfer] Workflow succeeded — initiating Globus transfer...", flush=True)
        self.transfer_results(transfer_config)

    # Build WorkflowResult
    return WorkflowResult(
        success=result_dict.get("success", False),
        mode=result_dict.get("mode", exec_mode),
        execution_time=(elapsed if result_dict.get("success") and exec_mode == "local" else None),
        phases_completed=phases_completed if result_dict.get("success") else [],
        events_processed=events_processed if result_dict.get("success") else [],
        snakefile_path=result_dict.get("snakefile_path"),
        job_id=result_dict.get("job_id"),
        message=result_dict.get("message", ""),
        partial_failures=result_dict.get("partial_failures", []),
    )

run_prepare_scenarios_serially(overwrite_scenario_if_already_set_up=False, rerun_swmm_hydro_if_outputs_exist=False, verbose=False)

Prepare all scenarios sequentially.

Executes scenario preparation for all scenarios in serial order, updating logs after each scenario completes.

Parameters:

Name	Type	Description	Default
overwrite_scenario_if_already_set_up	bool	If True, overwrite existing scenarios (default: False)	False
rerun_swmm_hydro_if_outputs_exist	bool	If True, rerun SWMM hydrology model even if outputs exist (default: False)	False
verbose	bool	If True, print progress messages (default: False)	False

Source code in src/hhemt/analysis.py

def run_prepare_scenarios_serially(
    self,
    overwrite_scenario_if_already_set_up: bool = False,
    rerun_swmm_hydro_if_outputs_exist: bool = False,
    verbose: bool = False,
):
    """
    Prepare all scenarios sequentially.

    Executes scenario preparation for all scenarios in serial order, updating
    logs after each scenario completes.

    Parameters
    ----------
    overwrite_scenario_if_already_set_up : bool, optional
        If True, overwrite existing scenarios (default: False)
    rerun_swmm_hydro_if_outputs_exist : bool, optional
        If True, rerun SWMM hydrology model even if outputs exist (default: False)
    verbose : bool, optional
        If True, print progress messages (default: False)
    """
    prepare_scenario_launchers = self.retrieve_prepare_scenario_launchers(
        overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
        rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
        verbose=verbose,
    )
    for launcher in prepare_scenario_launchers:
        launcher()
        self._update_log()  # update logs
    self._update_log()
    return

run_python_functions_concurrently(function_launchers, min_memory_per_function_MiB=1024, max_parallel=None, verbose=True)

Run Python functions concurrently, limiting parallelism by CPU and memory.

Parameters:

Name	Type	Description	Default
function_launchers	List[Callable[[], None]]	Functions to execute concurrently.	required
max_parallel	int | None	Upper bound on parallelism (defaults to CPU count).	None
min_memory_per_function_MiB	int | None	Minimum memory required per function (MiB). If provided, concurrency is reduced to avoid oversubscription.	1024
verbose	bool	Print progress messages.	True

Returns:

Type	Description
List[int]	Indices of functions that completed successfully.

Source code in src/hhemt/analysis.py

def run_python_functions_concurrently(
    self,
    function_launchers: list[Callable[[], None]],
    min_memory_per_function_MiB: int | None = 1024,
    max_parallel: int | None = None,
    verbose: bool = True,
) -> list[int]:
    """
    Run Python functions concurrently, limiting parallelism by CPU and memory.

    Parameters
    ----------
    function_launchers : List[Callable[[], None]]
        Functions to execute concurrently.
    max_parallel : int | None
        Upper bound on parallelism (defaults to CPU count).
    min_memory_per_function_MiB : int | None
        Minimum memory required per function (MiB).
        If provided, concurrency is reduced to avoid oversubscription.
    verbose : bool
        Print progress messages.

    Returns
    -------
    List[int]
        Indices of functions that completed successfully.
    """

    effective_max_parallel = self._calculate_effective_max_parallel(
        min_memory_per_function_MiB=min_memory_per_function_MiB,
        max_concurrent=max_parallel,
        verbose=verbose,
    )

    if verbose:
        print(
            f"Running {len(function_launchers)} functions (max parallel = {effective_max_parallel})",
            flush=True,
        )

    results: list[int] = []
    batch_start = time.time()  # Reference point for all tasks

    def wrapper(idx: int, launcher: Callable[[], None]):
        task_start = time.time()
        launcher()
        task_end = time.time()

        duration = task_end - task_start
        completed_at = task_end - batch_start

        if verbose:
            print(
                f"Function {idx}: duration={duration:.2f}s, completed_at={completed_at:.2f}s",
                flush=True,
            )
        return idx

    # ----------------------------
    # Execute
    # ----------------------------
    with ThreadPoolExecutor(max_workers=effective_max_parallel) as executor:
        futures = {executor.submit(wrapper, idx, launcher): idx for idx, launcher in enumerate(function_launchers)}

        for future in as_completed(futures):
            idx = futures[future]
            try:
                results.append(future.result())
            except Exception as e:
                if verbose:
                    print(f"Function {idx} failed with error: {e}", flush=True)

    self._update_log()
    return results

run_sims_in_sequence(pickup_where_leftoff, process_outputs_after_sim_completion=False, which='both', compression_level=5, *, override_clear_raw=None, verbose=False)

Arguments passed to run: - mode: Mode | Literal["single_core"] - pickup_where_leftoff Arguments passed to processing process_sim_timeseriess (only needed if process_outputs_after_sim_completion=True): - which: Literal["TRITON", "SWMM", "both"] - override_clear_raw: ClearRawValue | None - compression_level: int

Source code in src/hhemt/analysis.py

def run_sims_in_sequence(
    self,
    pickup_where_leftoff,
    process_outputs_after_sim_completion: bool = False,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    compression_level: int = 5,
    *,
    override_clear_raw: ClearRawValue | None = None,
    verbose=False,
):
    """
    Arguments passed to run:
        - mode: Mode | Literal["single_core"]
        - pickup_where_leftoff
    Arguments passed to processing process_sim_timeseriess
    (only needed if process_outputs_after_sim_completion=True):
        - which: Literal["TRITON", "SWMM", "both"]
        - override_clear_raw: ClearRawValue | None
        - compression_level: int
    """
    if verbose:
        print("Running all sims in series...", flush=True)
    enabled_model_types = self._get_enabled_model_types()
    for event_iloc in self.df_sims.index:
        for model_type in enabled_model_types:
            if verbose:
                print(
                    f"Running sim {event_iloc} ({model_type}) and pickup_where_leftoff = {pickup_where_leftoff}",
                    flush=True,
                )
            self._run_sim(
                event_iloc=event_iloc,
                pickup_where_leftoff=pickup_where_leftoff,
                verbose=verbose,
                process_outputs_after_sim_completion=process_outputs_after_sim_completion,
                which=which,
                override_clear_raw=override_clear_raw,
                compression_level=compression_level,
                model_type=model_type,  # type: ignore
            )
    self._update_log()

run_simulations_concurrently(launch_functions, max_concurrent=None, verbose=True)

Run simulations concurrently using the configured execution strategy.

Automatically selects the appropriate executor based on cfg_analysis.multi_sim_run_method: - "1_job_many_srun_tasks": Uses SlurmExecutor for HPC execution - "local": Uses LocalConcurrentExecutor for parallel local execution - Other: Uses SerialExecutor for sequential execution

Parameters:

Name	Type	Description	Default
launch_functions	list[tuple]	List of tuples (launcher, finalize_sim) from _create_subprocess_sim_run_launcher()	required
max_concurrent	Optional[int]	Maximum number of concurrent simulations	None
verbose	bool	If True, print progress messages	True

Returns:

Type	Description
list	List of simulation statuses

Source code in src/hhemt/analysis.py

def run_simulations_concurrently(
    self,
    launch_functions: list[tuple],
    max_concurrent: int | None = None,
    verbose: bool = True,
):
    """
    Run simulations concurrently using the configured execution strategy.

    Automatically selects the appropriate executor based on cfg_analysis.multi_sim_run_method:
    - "1_job_many_srun_tasks": Uses SlurmExecutor for HPC execution
    - "local": Uses LocalConcurrentExecutor for parallel local execution
    - Other: Uses SerialExecutor for sequential execution

    Parameters
    ----------
    launch_functions : list[tuple]
        List of tuples (launcher, finalize_sim) from _create_subprocess_sim_run_launcher()
    max_concurrent : Optional[int]
        Maximum number of concurrent simulations
    verbose : bool
        If True, print progress messages

    Returns
    -------
    list
        List of simulation statuses
    """
    return self._execution_strategy.execute_simulations(launch_functions, max_concurrent, verbose)

static_plots(*, static_config_ids=None, execution_mode='auto', override_static_plot_configs=None, verbose=True, dry_run=False)

Generate publication static figures, one Snakemake rule per static-plot ID, distributed via the existing executor (ADR-8). Adjacent to run()/reprocess()/ bundle_report_data()/eda().

override_static_plot_configs (override-prefixed; default None reads cfg_analysis.static_plot_configs) is the resolved config list, threaded DOWN to the workflow builder so a passed override is honored (anti facade- drift). static_config_ids optionally restricts the render to the named subset. A lazy stamp_new_target performs the 13->14 layout stamp (the static_plots/ output dir is created lazily on first render; V0014 is a no-op against persisted state).

Source code in src/hhemt/analysis.py

def static_plots(
    self,
    *,
    static_config_ids: "list[str] | None" = None,
    execution_mode: "Literal['auto','local','slurm']" = "auto",
    override_static_plot_configs: "list[Path] | None" = None,
    verbose: bool = True,
    dry_run: bool = False,
) -> dict:
    """Generate publication static figures, one Snakemake rule per static-plot ID,
    distributed via the existing executor (ADR-8). Adjacent to run()/reprocess()/
    bundle_report_data()/eda().

    ``override_static_plot_configs`` (override-prefixed; default None reads
    ``cfg_analysis.static_plot_configs``) is the resolved config list, threaded
    DOWN to the workflow builder so a passed override is honored (anti facade-
    drift). ``static_config_ids`` optionally restricts the render to the named
    subset. A lazy ``stamp_new_target`` performs the 13->14 layout stamp (the
    ``static_plots/`` output dir is created lazily on first render; V0014 is a
    no-op against persisted state).
    """
    from hhemt.version_migration import LAYOUT_VERSION
    from hhemt.version_migration.state import stamp_new_target

    resolved = (
        override_static_plot_configs
        if override_static_plot_configs is not None
        else self.cfg_analysis.static_plot_configs
    )
    if not resolved:
        from .exceptions import ConfigurationError

        raise ConfigurationError(
            field="static_plot_configs",
            message=(
                "static_plots() requires >=1 static-plot config; "
                "cfg_analysis.static_plot_configs is empty and no override was passed."
            ),
            config_path=self.analysis_config_yaml,
        )
    stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)
    return self._workflow_builder.submit_static_plots_workflow(
        resolved_static_plot_configs=resolved,
        static_config_ids=static_config_ids,
        execution_mode=execution_mode,
        dry_run=dry_run,
        verbose=verbose,
    )

submit_workflow(mode='auto', process_system_level_inputs=False, overwrite_system_inputs=False, compile_TRITON_SWMM=True, recompile_if_already_done_successfully=False, prepare_scenarios=True, overwrite_scenario_if_already_set_up=False, rerun_swmm_hydro_if_outputs_exist=False, process_timeseries=True, which='both', override_clear_raw=None, override_force_rerun=None, compression_level=5, pickup_where_leftoff=False, wait_for_completion=False, dry_run=False, verbose=True, override_hpc_total_nodes=None, override_hpc_restart_times_simulate=None, override_hpc_restart_times_other=None, report_formats=None, extra_sbatch_args=None, snakemake_diagnostics=None)

Submit workflow using Snakemake (replaces submit_SLURM_job_array).

Automatically detects execution context (local vs. HPC) and submits accordingly.

Delegates to SnakemakeWorkflowBuilder.

Parameters:

Name	Type	Description	Default
mode	Literal['local', 'slurm', 'auto']	Execution mode. If "auto", detects based on SLURM environment variables.	'auto'
process_system_level_inputs	bool	If True, process system-level inputs (DEM, Mannings) in Phase 1	False
overwrite_system_inputs	bool	If True, overwrite existing system input files	False
compile_TRITON_SWMM	bool	If True, compile TRITON-SWMM in Phase 1	True
recompile_if_already_done_successfully	bool	If True, recompile even if already compiled successfully	False
prepare_scenarios	bool	If True, each simulation will prepare its scenario before running	True
overwrite_scenario_if_already_set_up	bool	If True, overwrite existing scenarios	False
rerun_swmm_hydro_if_outputs_exist	bool	If True, rerun SWMM hydrology model even if outputs exist	False
process_timeseries	bool	If True, process timeseries outputs after each simulation	True
which	Literal['TRITON', 'SWMM', 'both']	Which outputs to process (only used if process_timeseries=True)	'both'
override_clear_raw	ClearRawValue | None	Runtime override for cfg_analysis.clear_raw. None (default) reads from YAML; concrete values follow the override-prefix convention.	None
compression_level	int	Compression level for output files (0-9)	5
pickup_where_leftoff	bool	If True, resume simulations from last checkpoint	False
wait_for_completion	bool	If True, wait for workflow completion (relevant for slurm jobs only)	False
dry_run	bool	If True, only perform a dry run and return that result	False
verbose	bool	If True, print progress messages	True
override_hpc_total_nodes	int | None	If set, overrides hpc_total_nodes in the generated SBATCH script without mutating the config. Only valid for multi_sim_run_method="1_job_many_srun_tasks".	None

Returns:

Type	Description
dict	Status dictionary with keys: - success: bool - Whether workflow succeeded - mode: str - "local" or "slurm" - snakefile_path: Path - Path to generated Snakefile - job_id: str | None - Job ID (only for slurm mode) - message: str - Status message

Source code in src/hhemt/analysis.py

def submit_workflow(
    self,
    mode: Literal["local", "slurm", "auto"] = "auto",
    process_system_level_inputs: bool = False,
    overwrite_system_inputs: bool = False,
    compile_TRITON_SWMM: bool = True,
    recompile_if_already_done_successfully: bool = False,
    prepare_scenarios: bool = True,
    overwrite_scenario_if_already_set_up: bool = False,
    rerun_swmm_hydro_if_outputs_exist: bool = False,
    process_timeseries: bool = True,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    override_clear_raw: ClearRawValue | None = None,
    override_force_rerun: ForceRerunValue | None = None,
    compression_level: int = 5,
    pickup_where_leftoff: bool = False,
    wait_for_completion: bool = False,  # relevant for slurm jobs only
    dry_run: bool = False,
    verbose: bool = True,
    override_hpc_total_nodes: int | None = None,
    override_hpc_restart_times_simulate: int | None = None,
    override_hpc_restart_times_other: int | None = None,
    report_formats: list[str] | None = None,
    extra_sbatch_args: list[str] | None = None,
    snakemake_diagnostics: SnakemakeDiagnostics | None = None,
) -> dict:
    """
    Submit workflow using Snakemake (replaces submit_SLURM_job_array).

    Automatically detects execution context (local vs. HPC) and submits accordingly.

    Delegates to SnakemakeWorkflowBuilder.

    Parameters
    ----------
    mode : Literal["local", "slurm", "auto"]
        Execution mode. If "auto", detects based on SLURM environment variables.
    process_system_level_inputs : bool
        If True, process system-level inputs (DEM, Mannings) in Phase 1
    overwrite_system_inputs : bool
        If True, overwrite existing system input files
    compile_TRITON_SWMM : bool
        If True, compile TRITON-SWMM in Phase 1
    recompile_if_already_done_successfully : bool
        If True, recompile even if already compiled successfully
    prepare_scenarios : bool
        If True, each simulation will prepare its scenario before running
    overwrite_scenario_if_already_set_up : bool
        If True, overwrite existing scenarios
    rerun_swmm_hydro_if_outputs_exist : bool
        If True, rerun SWMM hydrology model even if outputs exist
    process_timeseries : bool
        If True, process timeseries outputs after each simulation
    which : Literal["TRITON", "SWMM", "both"]
        Which outputs to process (only used if process_timeseries=True)
    override_clear_raw : ClearRawValue | None
        Runtime override for ``cfg_analysis.clear_raw``. ``None`` (default)
        reads from YAML; concrete values follow the override-prefix convention.
    compression_level : int
        Compression level for output files (0-9)
    pickup_where_leftoff : bool
        If True, resume simulations from last checkpoint
    wait_for_completion : bool
        If True, wait for workflow completion (relevant for slurm jobs only)
    dry_run : bool
        If True, only perform a dry run and return that result
    verbose : bool
        If True, print progress messages
    override_hpc_total_nodes : int | None
        If set, overrides `hpc_total_nodes` in the generated SBATCH script without
        mutating the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.

    Returns
    -------
    dict
        Status dictionary with keys:
        - success: bool - Whether workflow succeeded
        - mode: str - "local" or "slurm"
        - snakefile_path: Path - Path to generated Snakefile
        - job_id: str | None - Job ID (only for slurm mode)
        - message: str - Status message
    """
    # Stamp _version.json at LAYOUT_VERSION on first materialization (lazy
    # stamp per version_migration_system master plan PI-1). Idempotent
    # under concurrent writers.
    from hhemt.version_migration import LAYOUT_VERSION
    from hhemt.version_migration.state import stamp_new_target

    stamp_new_target(self.analysis_paths.analysis_dir, LAYOUT_VERSION)

    # resume-retry-resilience P3 — surface the zero-progress-resume foot-gun
    # before launch (friction Option A). Fires only when actually resuming in an
    # HPC mode with a per-sim walltime set; run() reaches here via delegation,
    # so this single site covers both run() and direct submit_workflow() callers.
    self._warn_resume_zero_progress(pickup_where_leftoff)

    # Driver-start orchestrator-liveness sentinel (Phase 2 of the reprocess
    # concurrency gate). Single-writer per logical driver: a sensitivity
    # run() delegates below to self.sensitivity.submit_workflow, which writes
    # its OWN master-keyed sentinel — writing here too would double-write into
    # the same _status/_orchestrator/ dir for one logical driver, so guard on
    # NOT sensitivity (sensitivity runs leave _driver_id None here).
    _driver_id = None
    _eff_mode = self.cfg_analysis.multi_sim_run_method
    if not self.cfg_analysis.toggle_sensitivity_analysis:
        _driver_id = _osent.new_driver_id()
        _osent.write_orchestrator_sentinel(
            self.analysis_paths.analysis_dir,
            driver_id=_driver_id,
            workflow_submission_mode=_eff_mode,
        )

    try:
        # Force-rerun pre-delete (login-node responsibility per master plan
        # Strategy). Resolve + validate + delete BEFORE Snakemake plans the DAG
        # so MTIME-input triggers cascade re-fire automatically.
        self._apply_force_rerun(override_force_rerun)

        if self.cfg_analysis.toggle_sensitivity_analysis:
            result = self.sensitivity.submit_workflow(
                mode=mode,
                process_system_level_inputs=process_system_level_inputs,
                overwrite_system_inputs=overwrite_system_inputs,
                compile_TRITON_SWMM=compile_TRITON_SWMM,
                recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                prepare_scenarios=prepare_scenarios,
                overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                process_timeseries=process_timeseries,
                which=which,
                override_clear_raw=override_clear_raw,
                override_force_rerun=override_force_rerun,
                compression_level=compression_level,
                pickup_where_leftoff=pickup_where_leftoff,
                wait_for_completion=wait_for_completion,
                dry_run=dry_run,
                verbose=verbose,
                override_hpc_total_nodes=override_hpc_total_nodes,
                override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
                override_hpc_restart_times_other=override_hpc_restart_times_other,
                report_formats=report_formats,
                extra_sbatch_args=extra_sbatch_args,
                snakemake_diagnostics=snakemake_diagnostics,
            )
        else:
            # NOTE: override_force_rerun is NOT threaded into the inner builder
            # — the pre-delete already happened at this layer
            # (self._apply_force_rerun above) and the builder's
            # submit_workflow does not need a runtime force-rerun parameter.
            if pickup_where_leftoff:
                # Scenario-set-change invalidation (multi_sim resume). The toolkit
                # uses --rerun-triggers mtime only (workflow.py), so a present
                # e_consolidate_complete.flag short-circuits the missing-intermediate
                # demand for an ADDED scenario's prepare->run->process->consolidate
                # chain (the per-sim plot rules, enumerated over the full SIM_IDS,
                # would then fire and crash reading the un-prepared hydraulics.inp).
                # Mirror the sensitivity path's scenario-set-change invalidation:
                # when the config event-id set differs from the on-disk prepared set,
                # delete the analysis-level consolidate flag so consolidate re-demands
                # its full SIM_IDS input expand, and delete orphan per-event flags for
                # events no longer in the config.
                self._invalidate_consolidate_flag_on_scenario_set_change()
            result = self._workflow_builder.submit_workflow(
                mode=mode,
                process_system_level_inputs=process_system_level_inputs,
                overwrite_system_inputs=overwrite_system_inputs,
                compile_TRITON_SWMM=compile_TRITON_SWMM,
                recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                prepare_scenarios=prepare_scenarios,
                overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                process_timeseries=process_timeseries,
                which=which,
                override_clear_raw=override_clear_raw,
                compression_level=compression_level,
                pickup_where_leftoff=pickup_where_leftoff,
                wait_for_completion=wait_for_completion,
                dry_run=dry_run,
                verbose=verbose,
                override_hpc_total_nodes=override_hpc_total_nodes,
                override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
                override_hpc_restart_times_other=override_hpc_restart_times_other,
                report_formats=report_formats,
                extra_sbatch_args=extra_sbatch_args,
                snakemake_diagnostics=snakemake_diagnostics,
            )

        if dry_run and result.get("success"):
            snakemake_logfile = result.get("snakemake_logfile")
            if snakemake_logfile is not None:
                report_path = generate_dry_run_report_markdown(
                    snakemake_logfile=Path(snakemake_logfile),
                    analysis_dir=self.analysis_paths.analysis_dir,
                    verbose=verbose,
                )
                result["dry_run_report_markdown"] = report_path
    finally:
        # Blocking-local drivers (this Python process WAS the driver and the
        # builder call blocked to completion) remove the sentinel on return;
        # detached drivers leave a durable sentinel reclaimed by the gate's
        # liveness probes. Only the local arm removes here.
        if _driver_id is not None and _eff_mode == "local":
            _osent.remove_orchestrator_sentinel(self.analysis_paths.analysis_dir, _driver_id)

    # Detached drivers: enrich (persist, do not remove) with the driver's
    # slurm_jobid (single-job) / tmux_session_name (batch_job) so the gate's
    # sacct/tmux arms can probe it. Skipped for sensitivity runs (_driver_id
    # is None — the sensitivity-master submit owns its own sentinel).
    if _driver_id is not None and _eff_mode != "local" and isinstance(result, dict):
        _osent.enrich_orchestrator_sentinel(
            self.analysis_paths.analysis_dir,
            driver_id=_driver_id,
            slurm_jobid=result.get("job_id"),
            tmux_session_name=result.get("session_name"),
        )

    return result

test(*, n_reporting_timesteps=cnst.TEST_N_REPORTING_TSTEPS_PER_SIM, reporting_timestep_s=cnst.TEST_TRITON_REPORTING_TIMESTEP_S, execution_mode='auto', verbose=True, wait_for_job_completion=None, dry_run=False)

Run a strict, least-demanding subset of THIS analysis end-to-end.

Builds one minimum-device representative per unique (enabled-model-toggles x compilation-backend x partition x compute-config) group present in the analysis, materializes each under {analysis_dir}/_test/, truncates its inputs to ~n_reporting_timesteps reporting frames, and runs the full compile->run->process->consolidate-> report path. A strict subset of the user's defined analysis -- no sweeps, no synthetic substitution (PIP O-f requirements 1-7).

Note: This is a user-facing smoke-test ENTRY POINT (ADR-8), not a pytest-collected test function. pytest's default collection (python_functions = test*) collects only module-level functions and Test-prefixed-class methods, so this instance method on TRITONSWMM_analysis is never collected as a test. Invoke it directly (analysis.test() or via the test_analysis_test_end_to_end.py real-data smoke).

Source code in src/hhemt/analysis.py

def test(
    self,
    *,
    n_reporting_timesteps: int = cnst.TEST_N_REPORTING_TSTEPS_PER_SIM,
    reporting_timestep_s: int = cnst.TEST_TRITON_REPORTING_TIMESTEP_S,
    execution_mode: Literal["auto", "local", "slurm"] = "auto",
    verbose: bool = True,
    wait_for_job_completion: bool | None = None,
    dry_run: bool = False,
) -> "TestResult":
    """Run a strict, least-demanding subset of THIS analysis end-to-end.

    Builds one minimum-device representative per unique
    (enabled-model-toggles x compilation-backend x partition x compute-config)
    group present in the analysis, materializes each under
    ``{analysis_dir}/_test/``, truncates its inputs to ~``n_reporting_timesteps``
    reporting frames, and runs the full compile->run->process->consolidate->
    report path. A strict subset of the user's defined analysis -- no sweeps,
    no synthetic substitution (PIP O-f requirements 1-7).

    Note:
        This is a user-facing smoke-test ENTRY POINT (ADR-8), not a
        pytest-collected test function. pytest's default collection
        (``python_functions = test*``) collects only module-level
        functions and ``Test``-prefixed-class methods, so this instance
        method on ``TRITONSWMM_analysis`` is never collected as a test.
        Invoke it directly (``analysis.test()`` or via the
        ``test_analysis_test_end_to_end.py`` real-data smoke).
    """
    reps = self._select_test_representatives()
    # Truncation happens INSIDE _build_test_subanalyses: the sliced-weather path
    # + reporting interval are carried THROUGH the model_validate overlay dict so
    # the on-disk _test YAML the runner subprocess reloads
    # (prepare_scenario_runner.py:139) points at the short weather. An in-memory
    # setattr would be discarded at that reload (Gotcha 15). Scenario-prep
    # regenerates the SWMM .inp window from the sliced weather automatically
    # (swmm_utils.py:105-110), so no separate .inp-edit pass is needed.
    test_subs = self._build_test_subanalyses(
        reps,
        n_reporting_timesteps=n_reporting_timesteps,
        reporting_timestep_s=reporting_timestep_s,
    )
    results: list = []
    # reps and test_subs are index-aligned: _build_test_subanalyses appends
    # one sub per rep in `representatives` order.
    for rep, sub in zip(reps, test_subs, strict=True):
        wf_result = sub.run(
            from_scratch=True,
            execution_mode=execution_mode,
            verbose=verbose,
            wait_for_job_completion=wait_for_job_completion,
            dry_run=dry_run,
        )  # full compile->run->process->consolidate->report
        results.append(
            TestSubResult(
                representative=rep,
                analysis_id=sub.cfg_analysis.analysis_id,
                analysis=sub,
                workflow_result=wf_result,
                binary_provenance=_capture_binary_provenance(sub),
            )
        )
    return TestResult(
        representatives=reps,
        subanalyses=results,
        root=self.analysis_paths.analysis_dir / "_test",
    )

transfer_results(config)

Transfer analysis results to local machine via Globus.

This is a standalone method — it does not require run() to have been called first. Use it for the "submit on HPC, poll squeue, transfer when done" workflow.

Args: config: User-facing transfer configuration. See :class:~hhemt.config.globus.PostRunTransferConfig.

Returns: Globus task ID.

Raises: GlobusTransferError: If the transfer fails or is cancelled (only when config.wait_for_transfer is True).

Example::

from hhemt.config.globus import PostRunTransferConfig

config = PostRunTransferConfig(
    destination_root=r"D:\Dropbox\_GradSchool\repos\hhemt\frontier",
    system="frontier",
)
task_id = analysis.transfer_results(config)

Source code in src/hhemt/analysis.py

def transfer_results(
    self,
    config: "PostRunTransferConfig",
) -> str:
    """Transfer analysis results to local machine via Globus.

    This is a standalone method — it does not require ``run()`` to have
    been called first.  Use it for the "submit on HPC, poll squeue,
    transfer when done" workflow.

    Args:
        config: User-facing transfer configuration.  See
            :class:`~hhemt.config.globus.PostRunTransferConfig`.

    Returns:
        Globus task ID.

    Raises:
        GlobusTransferError: If the transfer fails or is cancelled
            (only when ``config.wait_for_transfer`` is True).

    Example::

        from hhemt.config.globus import PostRunTransferConfig

        config = PostRunTransferConfig(
            destination_root=r"D:\\Dropbox\\_GradSchool\\repos\\hhemt\\frontier",
            system="frontier",
        )
        task_id = analysis.transfer_results(config)
    """
    from hhemt.config.globus import (
        _get_endpoint_uuids,
        _normalize_wsl_path,
    )
    from hhemt.globus_transfer import GlobusTransferManager

    spec = config.to_transfer_spec(
        analysis_dir=self.analysis_paths.analysis_dir,
        analysis_id=self.cfg_analysis.analysis_id,
    )

    # Handle destination conflict
    dest_path = _normalize_wsl_path(config.destination_root).rstrip("/")
    dest_dir = Path(f"{dest_path}/{self.cfg_analysis.analysis_id}")
    if dest_dir.exists():
        self._handle_destination_conflict(dest_dir, config.conflict_policy)

    # Only pass collection_uuids for endpoints that need data_access consent;
    # pass session_required_domains for domain-restricted endpoints (e.g. OLCF).
    _uuid, _base, needs_data_access, session_domain = _get_endpoint_uuids(config.system)
    consent_uuids = [spec.endpoints.source_uuid] if needs_data_access else []
    session_domains = [session_domain] if session_domain else None
    manager = GlobusTransferManager(
        collection_uuids=consent_uuids,
        session_required_domains=session_domains,
    )
    task_id = manager.transfer(spec, exclude_dirs=config.exclude_patterns)

    if config.wait_for_transfer:
        manager.wait(task_id, timeout_minutes=config.timeout_minutes)

    return task_id

validate()

Run preflight validation on system and analysis configurations.

This method performs comprehensive validation of both system and analysis configurations before launching expensive simulation work. It checks:

• System config: paths, toggle dependencies, model selection
• Analysis config: weather data, run-mode consistency, HPC settings
• Data consistency: event alignment, storm tide data, units

Returns:

Type	Description
ValidationResult	Validation result with any errors and warnings. Use result.is_valid to check if validation passed, or result.raise_if_invalid() to raise ConfigurationError if any errors exist.

Examples:

>>> analysis = system.analysis
>>> result = analysis.validate()
>>> if not result.is_valid:
>>>     print(result)  # Show all errors and warnings
>>>     result.raise_if_invalid()  # Raise ConfigurationError

>>> # Or validate and raise in one step:
>>> analysis.validate().raise_if_invalid()

Notes

Validation is NOT automatically called in init to avoid breaking existing workflows. Users should explicitly call validate() before launching simulations, or CLI/API entry points can call it automatically.

Source code in src/hhemt/analysis.py

def validate(self) -> ValidationResult:
    """Run preflight validation on system and analysis configurations.

    This method performs comprehensive validation of both system and analysis
    configurations before launching expensive simulation work. It checks:

    - System config: paths, toggle dependencies, model selection
    - Analysis config: weather data, run-mode consistency, HPC settings
    - Data consistency: event alignment, storm tide data, units

    Returns
    -------
    ValidationResult
        Validation result with any errors and warnings. Use result.is_valid
        to check if validation passed, or result.raise_if_invalid() to raise
        ConfigurationError if any errors exist.

    Examples
    --------
    >>> analysis = system.analysis
    >>> result = analysis.validate()
    >>> if not result.is_valid:
    >>>     print(result)  # Show all errors and warnings
    >>>     result.raise_if_invalid()  # Raise ConfigurationError

    >>> # Or validate and raise in one step:
    >>> analysis.validate().raise_if_invalid()

    Notes
    -----
    Validation is NOT automatically called in __init__ to avoid breaking
    existing workflows. Users should explicitly call validate() before
    launching simulations, or CLI/API entry points can call it automatically.
    """
    return preflight_validate(
        cfg_system=self._system.cfg_system,
        cfg_analysis=self.cfg_analysis,
        cfg_hpc_system=self.cfg_hpc_system,
    )

TestRepresentative dataclass

One minimum-device representative of a unique (model-toggles x compilation-backend x partition x compute-config) group, selected by TRITONSWMM_analysis._select_test_representatives (D-AXES Option A: compute-config is a group KEY so every unique config is represented; device count is the tiebreak). source_analysis is the already-built, already-validated candidate analysis the representative was chosen from (the master itself, or one self.sensitivity.sub_analyses).

Source code in src/hhemt/analysis.py

@dataclass
class TestRepresentative:
    """One minimum-device representative of a unique
    (model-toggles x compilation-backend x partition x compute-config) group,
    selected by ``TRITONSWMM_analysis._select_test_representatives`` (D-AXES
    Option A: compute-config is a group KEY so every unique config is
    represented; device count is the tiebreak). ``source_analysis`` is the
    already-built, already-validated candidate analysis the representative was
    chosen from (the master itself, or one ``self.sensitivity.sub_analyses``)."""

    key: tuple
    source_analysis: "TRITONSWMM_analysis"
    partition: str | None

    @property
    def axes(self) -> dict:
        """Name the declared axes this representative is SELECTED to exercise,
        unpacked from ``key`` = (model_toggles, backend, partition,
        compute_config). ``compilation_backend`` is "CUDA"/"HIP"/None; a None
        backend on an ``n_gpus>=1`` rep is the diagnostic that the GPU-compile
        axis will not resolve (``hpc_system_config`` not threaded)."""
        model_toggles, backend, partition, compute_config = self.key
        run_mode, n_mpi_procs, n_omp_threads, n_gpus, n_nodes = compute_config
        return {
            "model_toggles": model_toggles,
            "compilation_backend": backend,
            "partition": partition,
            "run_mode": run_mode,
            "n_mpi_procs": n_mpi_procs,
            "n_omp_threads": n_omp_threads,
            "n_gpus": n_gpus,
            "n_nodes": n_nodes,
        }

axes property

Name the declared axes this representative is SELECTED to exercise, unpacked from key = (model_toggles, backend, partition, compute_config). compilation_backend is "CUDA"/"HIP"/None; a None backend on an n_gpus>=1 rep is the diagnostic that the GPU-compile axis will not resolve (hpc_system_config not threaded).

TestResult dataclass

Result of TRITONSWMM_analysis.test() -- the selected representatives, the materialized _test/ sub-analyses as TestSubResult records (each carrying the representative's declared axes, the resolved workflow_result, and informational binary_provenance), and the {analysis_dir}/_test root directory.

Source code in src/hhemt/analysis.py

@dataclass
class TestResult:
    """Result of ``TRITONSWMM_analysis.test()`` -- the selected representatives,
    the materialized ``_test/`` sub-analyses as ``TestSubResult`` records (each
    carrying the representative's declared ``axes``, the resolved
    ``workflow_result``, and informational ``binary_provenance``), and the
    ``{analysis_dir}/_test`` root directory."""

    representatives: list
    subanalyses: list
    root: Path

TestSubResult dataclass

One representative's REALIZED _test run: declared axes (via representative.axes) plus the RESOLVED execution the old (analysis_id, sub) tuple discarded, plus informational binary provenance. workflow_result.mode is "slurm" (real allocation) vs "local" (login-node ThreadPool, NO allocation); .job_id the real SLURM job id; .success the per-rep pass/fail. binary_provenance is captured-not-asserted (ldd/readelf text of the compiled triton.exe; None when absent) — NEVER parsed/gated in the engine (empirical judgment only).

Source code in src/hhemt/analysis.py

@dataclass
class TestSubResult:
    """One representative's REALIZED ``_test`` run: declared axes (via
    ``representative.axes``) plus the RESOLVED execution the old
    ``(analysis_id, sub)`` tuple discarded, plus informational binary
    provenance. ``workflow_result.mode`` is "slurm" (real allocation) vs
    "local" (login-node ThreadPool, NO allocation); ``.job_id`` the real SLURM
    job id; ``.success`` the per-rep pass/fail. ``binary_provenance`` is
    captured-not-asserted (ldd/readelf text of the compiled triton.exe; None
    when absent) — NEVER parsed/gated in the engine (empirical judgment only)."""

    representative: "TestRepresentative"
    analysis_id: str
    analysis: "TRITONSWMM_analysis"
    workflow_result: "WorkflowResult"
    binary_provenance: dict | None = None

TRITONSWMM_sensitivity_analysis

Manages sensitivity analysis by creating and orchestrating multiple sub-analyses.

This class creates a separate TRITONSWMM_analysis instance for each row in a sensitivity analysis configuration table (CSV or Excel). Each sub-analysis runs with different parameter values, and results are consolidated at the master level.

The sensitivity analysis workflow: 1. Reads sensitivity configuration (CSV/Excel with parameter combinations) 2. Creates sub-analysis for each configuration row 3. Runs simulations for all sub-analyses 4. Consolidates outputs across all parameter combinations 5. Produces multi-dimensional datasets with sensitivity dimensions

Parameters:

Name	Type	Description	Default
analysis	TRITONSWMM_analysis	Master analysis instance that contains the sensitivity configuration	required

Attributes:

Name	Type	Description
master_analysis	TRITONSWMM_analysis	Reference to the master analysis
sub_analyses	dict	Dictionary mapping sub-analysis index to TRITONSWMM_analysis instances
df_setup	DataFrame	Sensitivity configuration table with parameter combinations
independent_vars	list	List of parameters being varied in the sensitivity analysis

Source code in src/hhemt/sensitivity_analysis.py

class TRITONSWMM_sensitivity_analysis:
    """
    Manages sensitivity analysis by creating and orchestrating multiple sub-analyses.

    This class creates a separate TRITONSWMM_analysis instance for each row in a
    sensitivity analysis configuration table (CSV or Excel). Each sub-analysis runs
    with different parameter values, and results are consolidated at the master level.

    The sensitivity analysis workflow:
    1. Reads sensitivity configuration (CSV/Excel with parameter combinations)
    2. Creates sub-analysis for each configuration row
    3. Runs simulations for all sub-analyses
    4. Consolidates outputs across all parameter combinations
    5. Produces multi-dimensional datasets with sensitivity dimensions

    Parameters
    ----------
    analysis : TRITONSWMM_analysis
        Master analysis instance that contains the sensitivity configuration

    Attributes
    ----------
    master_analysis : TRITONSWMM_analysis
        Reference to the master analysis
    sub_analyses : dict
        Dictionary mapping sub-analysis index to TRITONSWMM_analysis instances
    df_setup : pd.DataFrame
        Sensitivity configuration table with parameter combinations
    independent_vars : list
        List of parameters being varied in the sensitivity analysis
    """

    def __init__(
        self,
        analysis: "TRITONSWMM_analysis",
        is_main_orchestrator: bool = True,
        skip_log_update: bool = False,
    ) -> None:
        """
        Initialize a sensitivity analysis orchestrator.

        Creates sub-analyses for each parameter combination defined in the sensitivity
        configuration file, enabling systematic exploration of parameter space.

        Parameters
        ----------
        analysis : TRITONSWMM_analysis
            Master analysis instance containing sensitivity configuration

        Raises
        ------
        ValueError
            If sensitivity configuration mixes GPU and non-GPU run modes
        """
        self.master_analysis = analysis
        self._system = analysis._system
        self._skip_log_update = skip_log_update
        self.analysis_paths = analysis.analysis_paths
        self.cfg_analysis = analysis.cfg_analysis
        self.sub_analyses_prefix = "sa_"
        self.subanalysis_dir = self.master_analysis.analysis_paths.analysis_dir / "subanalyses"
        df_setup_full = self._retrieve_df_setup()
        self._df_setup_full = df_setup_full
        self._has_per_sa_system_configs = "system_config_yaml" in df_setup_full.columns
        self._has_per_sa_system_overlay_columns = any(_is_system_overlay_column(c) for c in df_setup_full.columns)
        # Phase 6 (DQ7): a per-row ensemble-partition axis (hpc.partition canonical or
        # analysis.hpc_ensemble_partition legacy) that varies across rows resolves to
        # DISTINCT gpu_hardware per row, so it must route through the per-target build
        # path (one UniqueSystemTarget per distinct hardware) — not the master fast
        # path. A single distinct partition collapses to the master target as before.
        _partition_axis_cols = [
            c for c in df_setup_full.columns if c == "hpc.partition" or c == "analysis.hpc_ensemble_partition"
        ]
        _distinct_row_partitions: set = set()
        for _c in _partition_axis_cols:
            _distinct_row_partitions |= {str(v) for v in df_setup_full[_c].dropna().tolist() if str(v).strip() != ""}
        self._has_per_row_partition_variation = len(_distinct_row_partitions) > 1
        if (
            self._has_per_sa_system_overlay_columns
            or self._has_per_sa_system_configs
            or self._has_per_row_partition_variation
        ):
            self.unique_system_targets = self._build_unique_system_targets(
                df_setup_full,
                is_main_orchestrator=is_main_orchestrator,
            )
        else:
            # Fast path: no row varies system_config; reuse master self._system. A
            # same-partition GPU sensitivity suite (e.g. the native/container
            # within-family suite, which shares ONE partition across rows so the
            # per-row-partition gate above is False) lands here. The single target
            # MUST carry the master ensemble partition so `setup_target_N` emits
            # `--target-partition` (-> resolve_gpu_target -> GPU compile backend) AND
            # self._system gets the GPU pair injected (the workflow.py GPU-sensitivity
            # validation reads self.system.gpu_compilation_backend). Guarded on a
            # resolved backend so CPU/null-selector fast paths stay byte-identical
            # (every Snakefile golden builds with hpc_ensemble_partition null ->
            # _fast_backend is None -> no injection, no target_partition).
            _master_partition = self.master_analysis.cfg_analysis.hpc_ensemble_partition
            _fast_hw, _fast_backend = resolve_gpu_target(self.master_analysis.cfg_hpc_system, _master_partition)
            _fast_target_partition = None
            if _fast_backend is not None:
                # synth_cc friction fix (2026-07-08): same invariant as the build-path
                # site — inject the master-ensemble GPU pair into self._system only on
                # the DRIVER. On a setup_target runner (is_main_orchestrator=False)
                # self._system already carries the correct --target-partition pair (the
                # fast path fires only for single-partition suites, so runner partition
                # == master ensemble; this is a no-op-same-value today, gated to keep the
                # "runner never mutates the compile target's gpu fields" invariant explicit).
                if is_main_orchestrator:
                    self._system.gpu_hardware = _fast_hw
                    self._system.gpu_compilation_backend = _fast_backend
                    self._system.additional_modules = resolve_additional_modules(self.master_analysis.cfg_hpc_system)
                _fast_target_partition = _master_partition
            self.unique_system_targets = [
                UniqueSystemTarget(
                    target_id=0,
                    system_config_yaml=self._system.system_config_yaml,
                    system=self._system,
                    sub_analysis_ids=list(df_setup_full.index.astype(str)),
                    target_partition=_fast_target_partition,
                )
            ]
        from hhemt.config.analysis import analysis_config as _analysis_config_for_df_setup

        analysis_cols = [
            c
            for c in df_setup_full.columns
            if c in _analysis_config_for_df_setup.model_fields
            or _is_analysis_overlay_column(c)
            or _is_hpc_overlay_column(c)  # Phase 6 (DQ5): retain hpc.* alias columns so
            # the per-sub overlay application in _create_sub_analyses can resolve them.
        ]
        self.df_setup = df_setup_full.loc[:, analysis_cols]
        self.sub_analyses = self._create_sub_analyses()

        # Initialize workflow builder for sensitivity analysis
        self._workflow_builder = SensitivityAnalysisWorkflowBuilder(self)

    def prepare_scenarios_in_each_subanalysis(
        self,
        overwrite_scenario_if_already_set_up: bool = False,
        rerun_swmm_hydro_if_outputs_exist: bool = False,
        concurrent: bool = True,
        verbose: bool = False,
    ):
        if self.master_analysis.cfg_analysis.multi_sim_run_method in [
            "local",
            "1_job_many_srun_tasks",
        ]:
            prepare_scenario_launchers = []
            for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
                prepare_scenario_launchers += sub_analysis.retrieve_prepare_scenario_launchers(
                    overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                    rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                    verbose=verbose,
                )
            if concurrent:
                self.master_analysis.run_python_functions_concurrently(prepare_scenario_launchers, verbose=verbose)
            else:
                for launcher in prepare_scenario_launchers:
                    launcher()

            if self.all_scenarios_created is not True:
                scens_not_created = "\n\t".join(self.scenarios_not_created)
                raise RuntimeError(f"Preparation failed for the following scenarios:\n{scens_not_created}")
            self._update_master_analysis_log()
        elif self.master_analysis.cfg_analysis.multi_sim_run_method in ["batch_job"]:
            raise ValueError("prepare scenarios is not currently executable as batch_job.")

    def submit_workflow(
        self,
        mode: Literal["local", "slurm", "auto"] = "auto",
        # setup stuff
        process_system_level_inputs: bool = True,
        overwrite_system_inputs: bool = False,
        compile_TRITON_SWMM: bool = True,
        recompile_if_already_done_successfully: bool = False,
        # ensemble run stuff
        prepare_scenarios: bool = True,
        overwrite_scenario_if_already_set_up: bool = False,
        rerun_swmm_hydro_if_outputs_exist: bool = False,
        process_timeseries: bool = True,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        override_clear_raw: ClearRawValue | None = None,
        override_force_rerun: ForceRerunValue | None = None,
        compression_level: int = 5,
        pickup_where_leftoff: bool = True,
        wait_for_completion: bool = False,  # relevant for slurm jobs only
        dry_run: bool = False,
        verbose: bool = True,
        override_hpc_total_nodes: int | None = None,
        override_hpc_restart_times_simulate: int | None = None,
        override_hpc_restart_times_other: int | None = None,
        report_formats: list[str] | None = None,
        extra_sbatch_args: list[str] | None = None,
        snakemake_diagnostics: SnakemakeDiagnostics | None = None,
    ) -> dict:
        """
        Submit sensitivity analysis workflow using Snakemake.

        This orchestrates multiple sub-analysis workflows and a final master
        consolidation step that combines all sub-analysis outputs.

        Parameters
        ----------
        mode : Literal["local", "slurm", "auto"]
            Execution mode. If "auto", detects based on SLURM environment variables.
        process_system_level_inputs : bool
            If True, process system-level inputs (DEM, Mannings)
        overwrite_system_inputs : bool
            If True, overwrite existing system input files
        compile_TRITON_SWMM : bool
            If True, compile TRITON-SWMM
        recompile_if_already_done_successfully : bool
            If True, recompile even if already compiled successfully
        prepare_scenarios : bool
            If True, prepare scenarios before running
        overwrite_scenario_if_already_set_up : bool
            If True, overwrite existing scenarios
        rerun_swmm_hydro_if_outputs_exist : bool
            If True, rerun SWMM hydrology model even if outputs exist
        process_timeseries : bool
            If True, process timeseries outputs after simulations
        which : Literal["TRITON", "SWMM", "both"]
            Which outputs to process
        override_clear_raw : ClearRawValue | None
            Runtime override for ``cfg_analysis.clear_raw`` (None reads YAML).
        compression_level : int
            Compression level for output files (0-9)
        pickup_where_leftoff : bool
            If True, resume simulations from last checkpoint
        dry_run : bool
            If True, only perform a dry run and return that result
        verbose : bool
            If True, print progress messages
        override_hpc_total_nodes : int | None
            If set, overrides `hpc_total_nodes` in the generated SBATCH script without
            mutating the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.

        Returns
        -------
        dict
            Status dictionary with keys:
            - success: bool
            - mode: str
            - snakefile_path: Path
            - message: str
        """
        # Force-rerun pre-delete for direct sensitivity.submit_workflow callers.
        # Idempotent when Analysis.submit_workflow already applied it on the
        # dispatch path (matched flags would be absent by now).
        self.master_analysis._apply_force_rerun(override_force_rerun)

        # Driver-start orchestrator-liveness sentinel (Phase 2), keyed on the
        # MASTER analysis_dir. This is the sensitivity-master submit path and
        # always owns its sentinel (the Analysis.submit_workflow guard leaves
        # _driver_id None there and delegates here). Blocking-local drivers
        # remove on return; detached drivers leave a durable sentinel reclaimed
        # by the gate's liveness probes.
        _master_dir = self.master_analysis.analysis_paths.analysis_dir
        _eff_mode = self.master_analysis.cfg_analysis.multi_sim_run_method
        _driver_id = _osent.new_driver_id()
        _osent.write_orchestrator_sentinel(
            _master_dir,
            driver_id=_driver_id,
            workflow_submission_mode=_eff_mode,
        )
        try:
            result = self._workflow_builder.submit_workflow(
                mode=mode,
                process_system_level_inputs=process_system_level_inputs,
                overwrite_system_inputs=overwrite_system_inputs,
                compile_TRITON_SWMM=compile_TRITON_SWMM,
                recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                prepare_scenarios=prepare_scenarios,
                overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
                rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
                process_timeseries=process_timeseries,
                which=which,
                override_clear_raw=override_clear_raw,
                compression_level=compression_level,
                pickup_where_leftoff=pickup_where_leftoff,
                wait_for_completion=wait_for_completion,
                dry_run=dry_run,
                verbose=verbose,
                override_hpc_total_nodes=override_hpc_total_nodes,
                override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
                override_hpc_restart_times_other=override_hpc_restart_times_other,
                report_formats=report_formats,
                extra_sbatch_args=extra_sbatch_args,
                snakemake_diagnostics=snakemake_diagnostics,
            )
        finally:
            if _eff_mode == "local":
                _osent.remove_orchestrator_sentinel(_master_dir, _driver_id)

        if _eff_mode != "local" and isinstance(result, dict):
            _osent.enrich_orchestrator_sentinel(
                _master_dir,
                driver_id=_driver_id,
                slurm_jobid=result.get("job_id"),
                tmux_session_name=result.get("session_name"),
            )

        return result

    def _invalidate_processing_log_for_sa_ids(self, sa_id_tokens: tuple[str, ...]) -> None:
        """Per-sa_id dispatch for processing-log invalidation under
        ``override_force_rerun={"sa_id": [...]}``.

        For each requested sa_id, looks up its sub-analysis and calls the
        per-sub-analysis ``Analysis._invalidate_processing_log_for_force_rerun``
        with a ``scope="all"`` spec — which invalidates every scenario in
        that sub-analysis. Sub-analyses are full Analysis instances and
        own their own scenario list (cf. CLAUDE.md Gotcha 11: "Sensitivity
        analysis sub-analyses are full TRITONSWMM_analysis instances").

        Per cleanup-rerun-delete-redesign Phase 4 + B-mechanism.
        """
        from hhemt.workflow import ResolvedForceRerunSpec

        all_spec = ResolvedForceRerunSpec(scope="all", tokens=())
        for sa_id in sa_id_tokens:
            sub_analysis = self.sub_analyses.get(sa_id)
            if sub_analysis is None:
                # _validate_force_rerun_targets already filtered unknown
                # sa_ids; reaching here means the sub_analyses dict is
                # out of sync with df_setup — surface loudly.
                raise RuntimeError(
                    f"sub_analyses missing entry for sa_id={sa_id!r} after "
                    f"validation passed; df_setup/sub_analyses are out of sync"
                )
            sub_analysis._invalidate_processing_log_for_force_rerun(all_spec)

    def reprocess(
        self,
        start_with: Literal["process", "consolidate", "render"] = "consolidate",
        sa_ids: list[str] | None = None,
        execution_mode: Literal["auto", "local", "slurm"] = "auto",
        which: Literal["TRITON", "SWMM", "both"] = "both",
        compression_level: int = 5,
        verbose: bool = True,
        dry_run: bool = False,
        report_formats: list[str] | None = None,
        *,
        regenerate_existing: bool = False,
        delete_via_slurm: bool | None = None,
        override_force_rerun: ForceRerunValue | None = None,
    ) -> dict:
        """Master-level reprocess for sensitivity analyses.

        Invalidates per-sub-analysis consolidate flags (subset via ``sa_ids``
        or all sub-analyses by default) plus the master consolidate flag,
        then emits a scoped master Snakefile via
        :meth:`SensitivityAnalysisWorkflowBuilder.generate_reprocess_master_snakefile_content`
        and submits it via
        :meth:`SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow`.

        Unlike :meth:`TRITONSWMM_analysis.reprocess`, this method does NOT
        invoke the ``override_clear_raw`` orphan/abort gate (R12) — sensitivity
        master reprocess is a downstream-only refresh of consolidation +
        plotting + rendering against existing per-sa sim outputs and does
        not need the in-flight reconciliation logic that the analysis-level
        ``override_clear_raw`` flow uses.

        Parameters
        ----------
        start_with
            Stage to re-fire from. ``"consolidate"`` (default) deletes per-sa
            ``e_consolidate_sa-{id}_complete.flag`` files and the master
            ``f_consolidate_master_complete.flag``, then re-runs the consolidate
            + master_consolidation + plot/render rule chain. ``"render"``
            invalidates only the report artifacts. ``"process"`` reconciles
            stale ``d_process`` flags against summary existence and re-emits the
            per-(sa, event) rebuild rules (Gotcha 34/40); it does NOT collapse
            onto the ``"consolidate"`` Snakefile.
        sa_ids
            Optional subset of sub-analysis IDs (string-cast) to invalidate.
            When ``None`` (default), every sub-analysis's per-sa consolidate
            flag is invalidated. IDs not in ``sub_analyses`` are silently
            ignored at the unlink call (``missing_ok=True``).
        execution_mode
            ``"auto"`` detects SLURM context; ``"local"`` / ``"slurm"`` force
            the mode.
        which
            ``"both"`` / ``"TRITON"`` / ``"SWMM"`` — threaded into the
            consolidate rule shells' ``--which`` flag.
        compression_level
            Compression level (0-9) for the consolidate rule shells.
        verbose
            If True, print progress messages.
        dry_run
            If True, runs ``snakemake --dry-run`` only.

        Returns
        -------
        dict
            Status dictionary from
            :meth:`SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow`.
        """
        # Lazy-stamp _version.json at LAYOUT_VERSION (PI-1 pattern). Idempotent.
        from hhemt.version_migration import LAYOUT_VERSION
        from hhemt.version_migration.state import stamp_new_target

        stamp_new_target(self.master_analysis.analysis_paths.analysis_dir, LAYOUT_VERSION)

        # Force-rerun pre-delete (login-node responsibility). Per
        # cleanup-rerun-delete-redesign Phase 4 + R10. Resolves + validates +
        # deletes matched flags before Snakemake plans the reprocess DAG.
        # Skipped on dry_run — it deletes flags and clears per-scenario
        # processing-log records, both filesystem mutations the dry-run
        # no-destructive-mutation contract forbids.
        if not dry_run:
            self.master_analysis._apply_force_rerun(override_force_rerun)

        # Resolve invalidation target set. ``None`` → all sub-analyses; explicit
        # list → subset. String-cast preserves alignment with sub_analyses dict
        # iteration keys regardless of source type (int / str / numpy scalar).
        if sa_ids is None:
            targets = [str(sa_id) for sa_id in self.sub_analyses.keys()]
        else:
            targets = [str(s) for s in sa_ids]

        # Invalidate per-sa consolidate flags + master flag. start_with controls
        # which flags get unlinked; per-sa flag deletion is the entry point for
        # both "consolidate" and "process" (the master generator does not emit
        # process rules, so process invalidation is treated as consolidate
        # invalidation). "render" leaves consolidate flags intact and only
        # invalidates the rendered report artifact.
        from hhemt.du_sentinels import (
            decrement_scope_sentinel,
            restamp_parent_sentinels,
            sum_child_sentinels,
        )
        from hhemt.utils import fast_rmtree as _fast_rmtree

        master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
        status_dir = master_analysis_dir / "_status"
        if start_with in ("consolidate", "process"):
            for sa_id in targets:
                # EXEMPT-DU: status-flag
                (status_dir / f"e_consolidate_sa-{sa_id}_complete.flag").unlink(missing_ok=True)
            # EXEMPT-DU: status-flag
            (status_dir / "f_consolidate_master_complete.flag").unlink(missing_ok=True)
            # R7 (D2 Option a) — consolidate-stage divergence preflight. Login-node
            # fail-fast that converts the SILENT-partial-master-tree hazard into a
            # clear ConfigurationError. On start_with="consolidate" the generator's
            # _sub_included_for_reprocess (Gotcha 37) SILENTLY EXCLUDES any sub whose
            # summaries are absent, and master consolidation's allow_incomplete=True
            # default (Gotcha 36) then assembles the master tree over the COMPLETED
            # subset and returns success — so a sub whose sim completed (c_run
            # present) but whose summary was deleted is silently dropped from
            # sensitivity_datatree.zarr with only a buried log warning. This is the
            # symmetric analogue of the process-stage
            # _assert_reprocess_rebuild_sources_present preflight. Fires ONLY on the
            # divergence signature (c_run present AND summary absent) so Gotcha 36's
            # tolerance for genuinely-never-ran subs is preserved. Read-only (no
            # flag/mtime touch — zero rerun-trigger surface).
            if start_with == "consolidate" and not dry_run:
                from hhemt.constants import sim_run_flag_per_sa
                from hhemt.scenario import compute_event_id_slug
                from hhemt.workflow import _scenario_summaries_present

                _enabled = self.master_analysis._get_enabled_model_types()
                _diverged: list[str] = []
                for sa_id in targets:
                    sub = self.sub_analyses.get(sa_id)
                    if sub is None:
                        continue
                    for _evt_iloc in sub.df_sims.index:
                        _evt = compute_event_id_slug(sub._retrieve_weather_indexer_using_integer_index(_evt_iloc))
                        _c_run = master_analysis_dir / sim_run_flag_per_sa(_enabled[0], str(sa_id), _evt)
                        if _c_run.exists() and not _scenario_summaries_present(sub, _evt, _enabled):
                            _diverged.append(f"{self.sub_analyses_prefix}{sa_id}@evt-{_evt}")
                if _diverged:
                    from hhemt.exceptions import ConfigurationError

                    raise ConfigurationError(
                        field="start_with",
                        message=(
                            "reprocess(start_with='consolidate') cannot consolidate "
                            f"{sorted(_diverged)} — the simulation completed (c_run flag "
                            "present) but the per-scenario summary outputs are absent, so "
                            "the master consolidation would silently drop these "
                            "sub-analyses from sensitivity_datatree.zarr. Re-run with "
                            "start_with='process', regenerate_existing=True to rebuild the "
                            "missing summaries first."
                        ),
                    )
            # FIX 2 — divergence self-heal runs on the process path on EVERY
            # route REGARDLESS of regenerate_existing (D2). Each sub-analysis
            # reconciles its own d_process flags + per-model processing_log
            # against on-disk summary presence (D3): where a flag survives but
            # the enabled-model summary set is absent (the May-31 divergence:
            # 72 d_process flags vs 0 summary zarrs), unlink the flag + clear
            # the log so the master generator's emit gate (workflow.py:6810)
            # re-emits the per-(sa,evt) process rule and _already_written
            # (Gotcha 28) lets it write. No-op for any sub whose summaries are
            # all present (healthy). Sub-analyses are full Analysis instances
            # and own their scenarios (Gotcha 11); the helper resolves the
            # per-sa flag-token shape from each sub's is_subanalysis context.
            if start_with == "process" and not dry_run:
                for sa_id in targets:
                    sub_analysis = self.sub_analyses.get(sa_id)
                    if sub_analysis is None:
                        continue
                    _reconciled = sub_analysis._reconcile_stale_process_flags_against_summaries(
                        sa_id=sa_id, master_dir=master_analysis_dir
                    )
                    sub_analysis._assert_reprocess_rebuild_sources_present(_reconciled)
            # FIX 2b — regenerate_existing rebuild parity (D1 Option A: logic in the
            # extracted free function so the fast unit test exercises it without
            # entering reprocess()'s destructive body). Gated on the
            # regenerate_existing arm; no-op otherwise.
            if start_with == "process" and regenerate_existing and not dry_run:
                _unlink_dprocess_flags_for_regenerate(targets, status_dir)
            # Report+plot deletion ALWAYS runs (toggle-independent) — the report
            # regenerates from the preserved zarr on the default path (FQ1 parity).
            _report_html = master_analysis_dir / "analysis_report.html"
            _report_zip = master_analysis_dir / "analysis_report.zip"
            # EXEMPT-DU: du-handled-by-decrement
            _report_html.unlink(missing_ok=True)
            # EXEMPT-DU: du-handled-by-decrement
            _report_zip.unlink(missing_ok=True)
            # FIX 3 — when regenerate_existing (and not dry_run), a LATER
            # deletion restamps the master _du.json anyway (SLURM route: the
            # reprocess-delete workflow's per-sub + master rules; in-process
            # route: compute_and_write_scope_sentinel(master, scope="analysis")
            # below). The early report-restamp here would otherwise force a
            # full-tree GPFS stat() walk on the login node before any SLURM
            # offload — the observed multi-minute stall. The default
            # (regenerate_existing=False) path still restamps (no later deletion).
            if not dry_run and not regenerate_existing:
                restamp_parent_sentinels(_report_html, analysis_dir=master_analysis_dir)  # PATTERN B (FIX 3 gate)
            # Consolidated-zarr deletion + batched DU restamp are the EXPENSIVE
            # GPFS work — gate behind regenerate_existing. Default path preserves
            # the zarrs (consolidate stays inert) and runs NO restamp walk.
            # R8 routing (D-scope Option C) — computed once from the MASTER
            # multi_sim_run_method. None auto-resolves to slurm-offload on HPC
            # modes (D6 refinement 1). When the SLURM path runs, ONE scoped
            # reprocess-delete workflow fans out per-sub (each sub's processed/
            # across events + its analysis_datatree.zarr) + a master rule for
            # sensitivity_datatree.zarr — replacing BOTH the in-process zarr
            # deletions AND the per-sub processed-output delegation below.
            _hpc = self.master_analysis.cfg_analysis.multi_sim_run_method in (
                "batch_job",
                "1_job_many_srun_tasks",
            )
            _resolved_delete_via_slurm = _hpc if delete_via_slurm is None else delete_via_slurm
            route_delete_via_slurm = regenerate_existing and _resolved_delete_via_slurm and not dry_run and _hpc
            if route_delete_via_slurm:
                self._workflow_builder._base_builder.submit_reprocess_delete_workflow(
                    start_with=start_with,
                    override_in_flight=False,
                )
            elif regenerate_existing and not dry_run:
                # In-process path (local / delete_via_slurm=False) — per-sub +
                # master zarr deletion, plus the Phase-3 per-sub processed-output
                # delegation (FQ2; sub-analyses own their scenarios). The
                # delegated helper internally guards on
                # `start_with == "process"` (analysis.py
                # _delete_processed_outputs_for_reprocess, ~L2979), so a
                # consolidate/render reprocess PRESERVES each sub's processed/
                # (the rebuild source consolidate reads from) — only a
                # process-stage reprocess deletes it. Do NOT inline the
                # processed/ rmtree here without that guard: dropping it makes a
                # consolidate-stage regenerate delete the rebuild source and the
                # consolidate Snakemake step then fails (FIX-1 Phase-1 regression,
                # 2026-05-31). For the process stage the per-model LOG-clear
                # already ran above (FIX 1, hunk 2a) on both routes; the helper's
                # idempotent re-clear here is harmless.
                affected_sub_dirs: set = set()
                for sa_id in targets:
                    sub_analysis = self.sub_analyses.get(sa_id)
                    if sub_analysis is None:
                        continue
                    # FQ2 processed-output deletion (Phase 3) — delegate to the
                    # sub-analysis's own helper (sub-analyses own their scenarios;
                    # the helper's start_with guard protects consolidate/render).
                    sub_analysis._delete_processed_outputs_for_reprocess(
                        start_with, regenerate_existing=regenerate_existing, dry_run=dry_run
                    )
                    _sub_zarr = sub_analysis.analysis_paths.analysis_datatree_zarr
                    if _sub_zarr is not None and _sub_zarr.exists():
                        _fast_rmtree(_sub_zarr, analysis_dir=None)  # batched-restamp
                        affected_sub_dirs.add(sub_analysis.analysis_paths.analysis_dir)
                _master_zarr = self.analysis_paths.sensitivity_datatree_zarr
                if _master_zarr is not None and _master_zarr.exists():
                    _fast_rmtree(_master_zarr, analysis_dir=None)  # batched-restamp
                for _sub_dir in affected_sub_dirs:
                    sum_child_sentinels(_sub_dir, scope="sub_analysis", child_scope_dirs=["sims"])
                sum_child_sentinels(master_analysis_dir, scope="analysis", child_scope_dirs=["subanalyses", "sims"])
        elif start_with == "render":
            # No _status flag for render — re-fire by deleting the report
            # artifacts so Snakemake's mtime trigger sees the output as absent.
            # The report-artifact unlink is the flag-equivalent trigger, so it
            # runs even on dry_run (see D6); only the DU restamp is gated.
            _report_html = master_analysis_dir / "analysis_report.html"
            _report_zip = master_analysis_dir / "analysis_report.zip"
            # D3 — capture sizes BEFORE unlink so the O(1) decrement has the bytes.
            _html_bytes = _report_html.stat().st_size if _report_html.exists() else 0
            _zip_bytes = _report_zip.stat().st_size if _report_zip.exists() else 0
            # EXEMPT-DU: du-handled-by-decrement
            _report_html.unlink(missing_ok=True)
            # EXEMPT-DU: du-handled-by-decrement
            _report_zip.unlink(missing_ok=True)
            if not dry_run:
                # D3 — O(1) decrement of the two report children (no plots on the
                # sensitivity render arm). Mirrors the non-sensitivity render path;
                # routes through write_du_sentinel (compare-and-write mtime invariant).
                _child_deltas: dict[str, int] = {}
                if _html_bytes:
                    _child_deltas["analysis_report.html"] = _html_bytes
                if _zip_bytes:
                    _child_deltas["analysis_report.zip"] = _zip_bytes
                if _child_deltas:
                    decrement_scope_sentinel(master_analysis_dir, scope="analysis", child_deltas=_child_deltas)
        else:
            raise ValueError(f"start_with must be one of 'process', 'consolidate', 'render'; got {start_with!r}")

        # Delegate to the sensitivity workflow builder.
        return self._workflow_builder.submit_reprocess_workflow(
            start_with=start_with,
            execution_mode=execution_mode,
            which=which,
            compression_level=compression_level,
            dry_run=dry_run,
            verbose=verbose,
            report_formats=report_formats,
        )

    def delete(
        self,
        override_in_flight: bool = False,
        *,
        override_multi_sim_run_method: Literal["local", "batch_job", "1_job_many_srun_tasks"] | None = None,
    ) -> None:
        """Distributed delete workflow for the sensitivity master analysis.

        Refuses by default when ``_status/_submitted/*.json`` sentinels
        indicate live SLURM jobs. Pass ``override_in_flight=True`` to bypass
        the guard.

        Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction
        + D-DeleteBoundary resolutions) and distributed-delete-and-du-
        recording Phase 3 (SLURM lift; ``override_multi_sim_run_method``
        mirrors the run-mode override pattern).
        """
        from hhemt.utils import fast_rmtree

        analysis_dir = self.master_analysis.analysis_paths.analysis_dir

        # 1. Clear any stale sentinels from a prior failed delete attempt.
        stale_dir = analysis_dir / "_status" / "_deleting"
        if stale_dir.exists():
            # EXEMPT-DU: status-dir-cleanup
            fast_rmtree(stale_dir)

        # 2. Submit the distributed sensitivity-delete workflow. Guards run
        # inside the builder; orchestrator does not invoke _pre_delete_guards
        # directly.
        self._workflow_builder.submit_delete_workflow_sensitivity(
            override_in_flight=override_in_flight,
            override_multi_sim_run_method=override_multi_sim_run_method,
        )

        # 3. Verify all expected sentinels present; remove analysis_dir atomically.
        expected = self._enumerate_expected_delete_sentinels()
        deleting_dir = analysis_dir / "_status" / "_deleting"
        actual = set(deleting_dir.glob("*.flag")) if deleting_dir.exists() else set()
        missing = expected - actual
        if missing:
            print(
                f"[delete] {len(missing)} per-sa-rule sentinels missing — preserving analysis_dir for debugging.",
                flush=True,
            )
            print(f"[delete] missing: {sorted(p.name for p in missing)}", flush=True)
            return
        print(
            f"[delete] all {len(expected)} per-sa-rule sentinels present — removing analysis_dir.",
            flush=True,
        )
        # EXEMPT-DU: full-analysis-root-wipe
        fast_rmtree(analysis_dir)

    def _enumerate_expected_delete_sentinels(self) -> set[Path]:
        """Compute the set of ``_status/_deleting/*.flag`` paths the
        sensitivity delete workflow will produce on full success.

        One per sub-analysis row in ``self.df_setup.index`` plus one for the
        analysis-level consolidation rule.
        """
        delete_dir = self.master_analysis.analysis_paths.analysis_dir / "_status" / "_deleting"
        expected = {delete_dir / "analysis_consolidation.flag"}
        for sa_id in self.df_setup.index.astype(str):
            expected.add(delete_dir / f"subanalysis_sa-{sa_id}.flag")
        return expected

    def render_report(self, format: Literal["html", "zip"] = "zip", *, reprocess: bool = False) -> "Path":
        """Render the master report for the sensitivity analysis.

        Idempotent: invokes ``snakemake --report`` against the master Snakefile
        without re-executing any rules. Renders only the master-level report;
        per-sub-analysis reports are not generated (R13).

        Parameters
        ----------
        format : Literal["html", "zip"], default "zip"
            Output format. ``"html"`` produces a single self-contained
            ``analysis_report.html`` with all figures inlined as base64, plus
            React-bundle post-process surgery (title, navbar, sidebar order,
            click-to-figure shim). ``"zip"`` produces ``analysis_report.zip``
            containing the unbundled report tree (separate HTML + assets);
            no post-process surgery is applied (the zip layout differs from
            the single-file HTML).
        reprocess : bool, default False
            When ``True``, render against ``Snakefile.reprocess`` (the filtered
            reprocess DAG) instead of the production ``Snakefile``, so the
            ``snakemake --report`` step only expects the figures the reprocess
            DAG built. Keyword-only; set by the reprocess ``render_report`` rule
            shell. Default ``False`` keeps the production render path
            byte-identical.
        """
        import subprocess
        import sys

        from .exceptions import WorkflowError
        from .workflow import _assert_snakefile_package_current

        master_dir = self.master_analysis.analysis_paths.analysis_dir
        snakefile_name = "Snakefile.reprocess" if reprocess else "Snakefile"
        snakefile = master_dir / snakefile_name
        _assert_snakefile_package_current(snakefile)
        out = master_dir / f"analysis_report.{format}"
        css_path = master_dir / "report" / "report.css"
        # Brand-theme resolution (ADR-7 layer 2) — symmetric to
        # analysis.render_report. The render_report_runner path builds a FRESH
        # instance without _brand_theme; resolve once via getattr-fallback to
        # serve BOTH the CSS emit and the navbar surgery (D-6; plan-review SE Flag 1).
        from .config.brand_theme import DEFAULT_BRAND_THEME
        from .config.loaders import load_brand_theme
        from .workflow import _brand_theme_css_map

        _theme = getattr(self, "_brand_theme", None)
        if _theme is None:
            _theme = (
                load_brand_theme(self.cfg_analysis.brand_theme)
                if self.cfg_analysis.brand_theme is not None
                else DEFAULT_BRAND_THEME
            )
        # Re-emit report artifacts from package resources so render_report
        # picks up edits made to the source-tree report_templates/.
        _emit_report_artifacts(master_dir, brand_theme=_brand_theme_css_map(_theme))
        cmd = [
            sys.executable,
            "-m",
            "snakemake",
            "--snakefile",
            str(snakefile),
            "--directory",
            str(master_dir),
            "--report",
            str(out),
            "--report-stylesheet",
            str(css_path),
            "--cores",
            "1",
        ]
        result = subprocess.run(cmd, capture_output=True, text=True)
        if result.returncode != 0:
            tail = "\n".join((result.stdout + "\n" + result.stderr).splitlines()[-50:])
            raise WorkflowError(
                phase="render_report",
                return_code=result.returncode,
                stderr=f"snakemake --report exit {result.returncode}; last 50 lines:\n{tail}",
            )
        # Apply React-bundle post-process surgery (title, navbar, sort order,
        # placeholder category, showCategory auto-pop, row-click delegate).
        # Both formats need the surgery:
        #  - HTML: edit the single rendered file in place.
        #  - Zip: extract, edit `analysis_report/report.html` inside, re-zip.
        # Without surgery in zip mode, the eye-icon-hiding CSS in report.css
        # leaves figure tables with no clickable affordance (the JS click
        # delegate that makes rows clickable lives only in the surgery).
        from .report_renderers._react_surgery import (
            apply_post_process_surgery,
            apply_post_process_surgery_to_zip,
        )

        # Navbar upper-left brand text: brand_theme.upper_left_text (ADR-7),
        # defaulting to analysis_id when None (D-6). _theme is resolved above.
        _navbar = _theme.upper_left_text or self.cfg_analysis.analysis_id
        # Resolve the active set's category_order. render_report() is dominantly
        # invoked from render_report_runner.main() on a FRESH analysis that never
        # called run() (see the _brand_theme getattr-fallback above for the
        # identical hazard), so self._active_reporting_set may not exist. getattr-
        # fallback to a config-only resolution (no CSV cross-validation at render
        # time) mirroring the _theme fallback above. Never let the bare attribute
        # AttributeError be swallowed by the surrounding `except Exception: pass`.
        _active_set = getattr(self, "_active_reporting_set", None)
        if _active_set is None:
            # render-without-run() fallback. Fail SOFT (SE F-I-3): the render path
            # bypasses validate_active_reporting_set, so a stale/unknown
            # reporting_set would raise here and surface as an opaque Snakemake
            # rule failure. Degrade to the historical "default" sidebar order + a
            # one-line warning instead of crashing the render rule.
            import logging

            from .config.report import resolve_active_reporting_set_name
            from .report_renderers._reporting_sets import get_reporting_set

            try:
                _cfg_report = getattr(self, "_cfg_report", None)
                if _cfg_report is None:
                    _cfg_report = self.cfg_analysis.report
                _set_name = resolve_active_reporting_set_name(
                    _cfg_report,
                    is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
                )
                _active_set = get_reporting_set(_set_name)
            except Exception as _e:
                logging.getLogger(__name__).warning(
                    "render-path reporting_set resolution failed (%s); falling back to 'default' category order",
                    _e,
                )
                _active_set = get_reporting_set("default")
        _category_order = list(_active_set.category_order)
        try:
            if format == "html":
                out.write_text(
                    apply_post_process_surgery(
                        out.read_text(),
                        navbar_text=_navbar,
                        category_order=_category_order,
                    )
                )
            else:
                apply_post_process_surgery_to_zip(out, navbar_text=_navbar, category_order=_category_order)
        except Exception:
            pass
        if format != "html":
            return out
        out_html = out
        # Snap-confined browsers (Ubuntu Firefox snap) cannot read files under
        # ~/.cache/. If the rendered report lands there, surface a one-line
        # workaround so the user does not hit "Access to the file was denied".
        try:
            if "/.cache/" in str(out_html):
                print(
                    f"[render_report] {out_html}\n"
                    f"[render_report] Note: snap-confined browsers cannot read ~/.cache; "
                    f"copy to ~/Downloads to view: cp {out_html} ~/Downloads/",
                    flush=True,
                )
        except Exception:
            pass
        return out_html

    # Conforms to hhemt.bundle._protocol.BundleableAnalysis
    # via duck typing — attributes delegated to self.master_analysis in
    # __init__ (lines 91-94).
    def bundle_report_data(
        self,
        output_path: "Path | None" = None,
    ) -> "Path":
        """Emit a portable render bundle for the sensitivity master analysis.

        Opt-in only — NEVER invoked from analysis.run() or
        submit_workflow(). The bundle includes the sensitivity master's
        consolidated outputs plus the union of source paths declared by
        every renderer in the master's render_report(), including per-sim
        renderers wildcarded over (sa_id, event_id).

        Args:
            output_path: Optional target path for the bundle tar.

        Returns:
            Path to the emitted bundle tar.
        """
        from hhemt.bundle import emit_bundle

        return emit_bundle(self, output_path)

    def reprex_bundle(self, output_path: "Path | None" = None) -> "Path":
        """Emit a reprex-ready Workflow-Run-Crate bundle for the sensitivity master and
        return its extracted directory root (ADR-10, D3).

        Parity peer of ``bundle_report_data()`` — the sensitivity master is the PRIMARY
        reprex surface (the ``(sa_id, column)`` problem-pair emission is intrinsically a
        sensitivity concept). ``emit_bundle`` already carries the reprex runnable-template
        set + WRC crate (Phase 2); this facade extracts the emitted zip to a sibling
        directory so the round-trip consumes a directory root directly
        (``Bundle.from_directory(...).reprex(...)``). Opt-in only.

        Returns:
            Path to the extracted reprex-bundle directory.
        """
        from hhemt.bundle import emit_bundle
        from hhemt.bundle._reprex import extract_reprex_bundle

        return extract_reprex_bundle(emit_bundle(self, output_path))

    def publish(
        self,
        target: "Literal['hydroshare', 'zenodo']",
        *,
        override_dataset_license: "Literal['CC0-1.0', 'CC-BY-NC-4.0'] | None" = None,
        software_doi: "str | None" = None,
    ) -> dict:
        """Deposit the sensitivity MASTER tree to a DOI-minting repo (C6, ADR-11).

        Opt-in only — NEVER invoked from run()/submit_workflow(), mirroring
        render_report()/bundle_report_data(). Deposits the master
        sensitivity_datatree.zarr + master-rooted ro-crate sidecar; the license is
        read from the emitted crate. Returns {"target","data_doi","software_doi","record_url"}.
        """
        from hhemt.publishing import publish_analysis

        return publish_analysis(
            self.master_analysis,
            target=target,
            override_dataset_license=override_dataset_license,
            software_doi=software_doi,
            consolidated_zarr_relpath="sensitivity_datatree.zarr",
        )

    def run_all_sims(
        self,
        pickup_where_leftoff,
        concurrent: bool = False,
        process_outputs_after_sim_completion: bool = True,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        compression_level: int = 5,
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose=False,
    ):
        if concurrent:
            raise RuntimeError(
                "Running sensitivity analyses concurrently requires"
                "more intelligent handling of compute resource availability"
                "tracking. Update run_simulations_concurrently function"
                "in analysis.py to enable this."
            )
            launch_functions = []
            for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
                launch_functions += sub_analysis._create_launchable_sims(
                    pickup_where_leftoff=pickup_where_leftoff,
                    verbose=verbose,
                )
            self.master_analysis.run_simulations_concurrently(launch_functions, verbose=verbose)
        else:
            for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
                sub_analysis.run_sims_in_sequence(
                    pickup_where_leftoff=pickup_where_leftoff,
                    process_outputs_after_sim_completion=process_outputs_after_sim_completion,
                    which=which,
                    override_clear_raw=override_clear_raw,
                    compression_level=compression_level,
                    verbose=verbose,
                )
        self._update_master_analysis_log()
        return

    def process_simulation_timeseries_concurrently(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        override_clear_raw: ClearRawValue | None = None,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        scenario_timeseries_processing_launchers = []
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            launchers = sub_analysis.retrieve_scenario_timeseries_processing_launchers(
                which=which,
                override_clear_raw=override_clear_raw,
                verbose=verbose,
                compression_level=compression_level,
            )
            scenario_timeseries_processing_launchers += launchers
        self.master_analysis.run_python_functions_concurrently(scenario_timeseries_processing_launchers)
        return

    def _consolidate_outputs_in_each_subanalysis(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        verbose: bool = False,
        compression_level: int = 5,
    ):
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            sub_analysis._consolidate_analysis_outputs(
                verbose=verbose,
                compression_level=compression_level,
            )
        self._update_master_analysis_log()
        return

    @property
    def TRITON_subanalyses_outputs_consolidated(self):
        cfg_sys = self.master_analysis._system.cfg_system
        success = True
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            if cfg_sys.toggle_tritonswmm_model:
                success = success and sub_analysis._tritonswmm_triton_analysis_summary_created
            elif cfg_sys.toggle_triton_model:
                success = success and sub_analysis._triton_only_analysis_summary_created
        return success

    @property
    def SWMM_subanalyses_outputs_consolidated(self):
        cfg_sys = self.master_analysis._system.cfg_system
        node_success = True
        link_success = True
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            if cfg_sys.toggle_tritonswmm_model:
                node_success = node_success and sub_analysis._tritonswmm_node_analysis_summary_created
                link_success = link_success and sub_analysis._tritonswmm_link_analysis_summary_created
            elif cfg_sys.toggle_swmm_model:
                node_success = node_success and sub_analysis._swmm_only_node_analysis_summary_created
                link_success = link_success and sub_analysis._swmm_only_link_analysis_summary_created
        return node_success and link_success

    def consolidate_outputs(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        verbose: bool = True,
        compression_level: int = 5,
    ):
        self.create_subanalysis_summaries(
            which=which,
            verbose=verbose,
            compression_level=compression_level,
        )
        self.consolidate_subanalysis_outputs(
            which=which,
            verbose=verbose,
            compression_level=compression_level,
        )
        return

    def consolidate_subanalysis_outputs(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        """Consolidate sub-analyses into a hierarchical sensitivity DataTree zarr.

        Replaces the previous per-mode flat ``xr.concat`` path. Each sub-analysis
        first builds its per-analysis DataTree (``analysis_datatree.zarr``); then
        the master assembles all sub-analyses into a single
        ``sensitivity_datatree.zarr`` at the master analysis dir.
        """
        self.consolidate_sensitivity_datatree(
            compression_level=compression_level,
            verbose=verbose,
        )
        return

    def build_sensitivity_datatree(self) -> "xr.DataTree":
        """Assemble the master sensitivity DataTree lazily from sub-analysis trees.

        Each sub-analysis's consolidated DataTree (``analysis_datatree.zarr``) is
        opened lazily and grafted under a ``sa_{sa_id}/`` subtree. Sensitivity
        parameters for each sub-analysis are attached as ``.attrs`` on the
        ``sa_{sa_id}`` node. A parameter-summary Dataset is written at the root
        under ``parameters`` for tabular queries.
        """
        tree_dict: dict[str, xr.Dataset] = {}

        tree_dict["/"] = xr.Dataset(
            attrs={
                "Conventions": "CF-1.13",
                "title": "TRITON-SWMM sensitivity analysis results",
                "analysis_id": str(self.master_analysis.cfg_analysis.analysis_id),
                "output_creation_date": current_datetime_string(),
            }
        )

        tree_dict["parameters"] = xr.Dataset.from_dataframe(self.df_setup)

        for sa_id, sub_analysis in self.sub_analyses.items():
            node_name = f"{self.sub_analyses_prefix}{sa_id}"
            # Refresh the sub-analysis's in-memory log from disk before reading its
            # consolidation state: open_datatree() gates on the in-memory
            # datatree_consolidation_complete flag, which a run (often in another
            # process) sets on disk but not in this long-lived sub object.
            # Previously the sensitivity aggregators' _update_log() refreshed these
            # sub logs as a side effect; that call was dropped in the
            # log-write-race-fix compute-on-read change, so refresh explicitly here
            # at the cross-analysis read site (read-only observer; safe).
            sub_analysis._refresh_log()
            try:
                sub_tree = sub_analysis.process.open_datatree()
            except ValueError:
                continue

            for path, node in sub_tree.subtree_with_keys:
                if not node.has_data:
                    continue
                rel = path.lstrip("/")
                if not rel:
                    continue
                tree_dict[f"{node_name}/{rel}"] = node.dataset

            setup_row = self.df_setup.loc[sa_id]
            attrs = {k: _to_native_attr(v) for k, v in setup_row.to_dict().items()}
            attrs["sa_id"] = str(sa_id)
            tree_dict[node_name] = xr.Dataset(attrs=attrs)

        tree = xr.DataTree.from_dict(tree_dict)
        apply_global_attributes(tree, analysis_id=str(self.master_analysis.cfg_analysis.analysis_id))

        # ADR-15 Phase 1: the per-event hhemt_producing_sha/version coordinates ride
        # up automatically via the verbatim node.dataset graft above (no new
        # transmission code). Re-derive the MASTER-level scalar fast-path here by
        # scanning the grafted sub-nodes for master-wide uniformity (uniform across
        # ALL sub-analyses' events -> scalar; else absent + divergent breadcrumb).
        from hhemt.cf_conventions import apply_producing_stamp

        _sha_vals: list[str] = []
        _semver_vals: list[str] = []
        for _key, _ds in tree_dict.items():
            _coords = getattr(_ds, "coords", {})
            if "hhemt_producing_sha" in _coords:
                _sha_vals.extend(str(v) for v in _ds["hhemt_producing_sha"].values.tolist())
            if "hhemt_producing_version" in _coords:
                _semver_vals.extend(str(v) for v in _ds["hhemt_producing_version"].values.tolist())
        apply_producing_stamp(tree, _sha_vals, _semver_vals)
        return tree

    def consolidate_sensitivity_datatree(
        self,
        compression_level: int = 5,
        verbose: bool = False,
        allow_incomplete: bool = True,
    ) -> Path:
        """Build and write the master sensitivity DataTree zarr.

        Ensures each sub-analysis has its own consolidated ``analysis_datatree.zarr``
        first, then assembles them into a single hierarchical store at
        ``sensitivity_datatree.zarr``.

        When ``allow_incomplete`` is True (the default), sub-analyses whose
        per-scenario summaries are not all present on disk are SKIPPED (with a
        logged warning naming the ``sa_id``) instead of crashing the master
        assembly with ``FileNotFoundError``/``ValueError``. The skip is
        whole-sub-analysis because ``_retrieve_combined_output`` concatenates
        per-scenario summaries along ``event_iloc`` and is all-or-nothing per
        sub-analysis. Pass ``allow_incomplete=False`` to restore fail-fast.

        NOTE (Decision D2, 2026-06-02, "for now"): the default is ``True`` so
        reprocess AND canonical runs tolerate partial-completion sensitivity
        suites by default. To restore strict fail-fast on canonical runs, flip
        this default back to ``False``.
        """
        fname_out = self.analysis_paths.sensitivity_datatree_zarr
        if fname_out is None:
            raise ValueError("sensitivity_datatree_zarr path is not configured on AnalysisPaths.")

        # Per D5/R8: bare .exists() is an unreliable completion signal — a
        # present-but-corrupt sensitivity_datatree.zarr (a write that crashed
        # mid-stream) .exists() as True, and would be returned as a healthy
        # result. Align with the master log's canonical signal: "already
        # consolidated" iff it exists AND sensitivity_datatree_consolidation_complete
        # is True (set only on a successful full write below). Present-but-incomplete
        # falls through to a clean rebuild. This is the master-sensitivity analogue
        # of consolidate_to_datatree's D5 log-keyed early-return; do NOT touch the
        # read-path .exists() guard in open_sensitivity_datatree (a read-path
        # existence check is correct).
        self.master_analysis._refresh_log()
        _log_complete = (
            hasattr(self.master_analysis.log, "sensitivity_datatree_consolidation_complete")
            and self.master_analysis.log.sensitivity_datatree_consolidation_complete.get() is True
        )
        if fname_out.exists() and _log_complete:
            if verbose:
                print(f"Sensitivity DataTree zarr already present at {fname_out} and log complete. Not overwriting.")
            # Ensure the master analysis-scope DU sentinel exists even on the
            # already-consolidated early-return path. This materializes the
            # sentinel on trees consolidated before this write site existed, and
            # is cheap/idempotent via compare-and-write.
            self._write_master_du_sentinel()
            return fname_out
        if fname_out.exists() and not _log_complete:
            from hhemt.utils import fast_rmtree

            fast_rmtree(fname_out, analysis_dir=self.analysis_paths.analysis_dir)
            if verbose:
                print(
                    f"Sensitivity DataTree zarr present at {fname_out} but log incomplete — "
                    "rebuilding (treating as corrupt)."
                )

        # Ensure each sub-analysis has its analysis_datatree.zarr built.
        for sa_id, sub_analysis in self.sub_analyses.items():
            sub_path = sub_analysis.analysis_paths.analysis_datatree_zarr
            if sub_path is None:
                continue
            # Gate on the POSITIVE completion marker, not bare .exists(): a sub
            # whose zarr is on disk but whose datatree_consolidation_complete flag
            # is null/False (set by a consolidate job in another process but not
            # reflected in this long-lived sub's in-memory log) must be
            # (re)consolidated — otherwise build_sensitivity_datatree's
            # open_datatree() raises ValueError and silently drops the sub.
            # consolidate_to_datatree is idempotent on an already-complete sub
            # (its own _log_complete early-return), so this self-heals.
            sub_analysis._refresh_log()
            sub_complete = (
                hasattr(sub_analysis.log, "datatree_consolidation_complete")
                and sub_analysis.log.datatree_consolidation_complete.get() is True
            )
            if not (sub_path.exists() and sub_complete):
                try:
                    sub_analysis.process.consolidate_to_datatree(
                        compression_level=compression_level,
                        verbose=verbose,
                    )
                except (FileNotFoundError, ValueError) as exc:
                    if not allow_incomplete:
                        raise
                    print(
                        f"[sensitivity-consolidate] Skipping incomplete sub-analysis "
                        f"{self.sub_analyses_prefix}{sa_id} under allow_incomplete=True: {exc}",
                        flush=True,
                    )
                    continue

        tree = self.build_sensitivity_datatree()
        from hhemt.cf_conventions import apply_provenance_core
        from hhemt.metadata import write_rocrate_sidecar
        from hhemt.provenance import emit_provenance

        _sub_relpaths = [f"subanalyses/sa_{sa_id}/analysis_datatree.zarr" for sa_id in self.sub_analyses]
        _core_json, _graph_json = emit_provenance(
            self.master_analysis,
            consolidated_zarr_relpath="sensitivity_datatree.zarr",
            sub_dataset_relpaths=_sub_relpaths,
            with_run_units=False,
        )
        apply_provenance_core(tree, core_json_str=_core_json)
        write_datatree_zarr(tree, fname_out, compression_level=compression_level)
        write_rocrate_sidecar(self.master_analysis.analysis_paths.analysis_dir, graph_json=_graph_json)

        self.master_analysis._refresh_log()
        if hasattr(self.master_analysis.log, "sensitivity_datatree_consolidation_complete"):
            self.master_analysis.log.sensitivity_datatree_consolidation_complete.set(True)

        if verbose:
            print(f"Wrote sensitivity DataTree zarr to {fname_out}")
        self._write_master_du_sentinel()
        return fname_out

    def _write_master_du_sentinel(self) -> None:
        """Write the master analysis-scope ``_du.json`` DU sentinel.

        The sensitivity mirror of the multisim analysis-scope write in
        ``processing_analysis.py`` (``consolidate_to_datatree``): the sensitivity
        master-consolidate path is otherwise the ONLY consolidation path that
        never writes an analysis-scope ``_du.json``, leaving the master root
        unsentineled so a ``delete --dry-run`` falls back to a full tree walk.

        Uses ``sum_child_sentinels`` (Gotcha 38 / the DU-rollup decision): the
        master total is the Σ of the per-sub ``_du.json`` sentinels (written by
        the D6 fold) + a bounded own-files walk excluding the child-scope dirs —
        NEVER a full-tree ``compute_and_write_scope_sentinel`` walk on the
        largest tree in the system. Ordering is structurally safe: the
        ``master_consolidation`` rule fans in on every per-sub completion flag,
        so all per-sub sentinels exist before this runs. Compare-and-write keeps
        the call idempotent (mtime preserved on unchanged bytes), so it is safe
        to invoke on the already-consolidated early-return path too.
        """
        from hhemt.du_sentinels import sum_child_sentinels

        sum_child_sentinels(
            self.master_analysis.analysis_paths.analysis_dir,
            scope="analysis",
            child_scope_dirs=["subanalyses", "sims"],
        )

    def open_sensitivity_datatree(self) -> "xr.DataTree":
        """Open the consolidated sensitivity DataTree zarr lazily."""
        path = self.analysis_paths.sensitivity_datatree_zarr
        if path is None or not path.exists():
            raise ValueError("Sensitivity DataTree zarr not found. Run consolidate_sensitivity_datatree() first.")
        return xr.open_datatree(path, engine="zarr", chunks="auto", consolidated=False)

    def create_subanalysis_summaries(
        self,
        which: Literal["TRITON", "SWMM", "both"] = "both",
        *,
        verbose: bool = False,
        compression_level: int = 5,
    ):
        if which in ["TRITON", "both"]:
            self._consolidate_outputs_in_each_subanalysis(
                which="TRITON",
                verbose=verbose,
                compression_level=compression_level,
            )
        if which in ["SWMM", "both"]:
            self._consolidate_outputs_in_each_subanalysis(
                which="SWMM",
                verbose=verbose,
                compression_level=compression_level,
            )
        return

    @property
    def tritonswmm_SWMM_node_summary(self):
        return self.master_analysis._tritonswmm_SWMM_node_summary

    @property
    def tritonswmm_SWMM_link_summary(self):
        return self.master_analysis._tritonswmm_SWMM_link_summary

    @property
    def tritonswmm_TRITON_summary(self):
        return self.master_analysis._tritonswmm_TRITON_summary

    # @property
    # def TRITONSWMM_runtimes(self):
    #     return self.master_analysis.TRITONSWMM_runtimes

    @property
    def analysis_independent_vars(self) -> list[str]:
        """Phase 2 — analysis-config attributes varied across sub-analyses.

        Returns the canonical (stripped) field name for each varied analysis-config
        column. Recognizes both `analysis.{field}` (canonical) and bare `field`
        names (deprecated; emits DeprecationWarning at sub-analysis construction
        time via `_create_sub_analyses`).
        """
        from hhemt.config.analysis import analysis_config

        seen: list[str] = []
        for col in self._df_setup_full.columns:
            if col == "system_config_yaml":
                continue
            if _is_system_overlay_column(col):
                continue
            if _is_hpc_overlay_column(col):
                field_name = _resolve_hpc_alias_to_analysis_field(col)
            elif _is_analysis_overlay_column(col):
                field_name = _strip_analysis_prefix(col)
            elif col in analysis_config.model_fields:
                field_name = col  # bare name; DeprecationWarning fires at sub-analysis construction time
            else:
                continue  # Defensive — should be caught by _retrieve_df_setup allowlist
            if field_name not in seen:
                seen.append(field_name)
        return seen

    @property
    def system_independent_vars(self) -> list[str]:
        """Phase 2 — system-config attributes varied across sub-analyses.

        Recognizes only `system.{field}` columns (no bare names — Phase 1 R1
        rejects bare-name system_config columns at the allowlist gate).
        """
        seen: list[str] = []
        for col in self._df_setup_full.columns:
            if _is_system_overlay_column(col):
                field_name = _strip_system_prefix(col)
                if field_name not in seen:
                    seen.append(field_name)
        return seen

    @property
    def df_setup_with_system_overlays(self) -> pd.DataFrame:
        """`df_setup` unioned with the `system.*` overlay columns from the full frame.

        `df_setup` is filtered to analysis-config columns only (:173-180); the
        `system.*` overlay columns are retained on `_df_setup_full`. Report
        renderers that plot a system-axis independent variable
        (`system.target_dem_resolution`, `system.gpu_hardware`, ...) need both
        sets in one frame. This accessor unions them on the shared `sa_id`
        index, preserving the PREFIXED column names so a renderer's
        `independent_var="system.gpu_hardware"` lookup resolves directly.

        The frame is analysis-columns + system-overlay-columns only — NOT the
        raw `_df_setup_full` (which also carries `system_config_yaml` and
        non-overlay annotation columns), keeping the renderer's column
        membership check scoped to the resolvable independent-var set.
        """
        overlay_cols = [c for c in self._df_setup_full.columns if _is_system_overlay_column(c)]
        if not overlay_cols:
            return self.df_setup
        return pd.concat(
            [self.df_setup, self._df_setup_full.loc[:, overlay_cols]],
            axis=1,
        )

    @property
    def independent_vars(self) -> list[str]:
        """BC alias — Phase 2 retains this name for downstream callers that haven't migrated.

        Returns the union of `analysis_independent_vars` and
        `system_independent_vars` (latter prefixed with `system.` to
        disambiguate). Downstream callers should migrate to the explicit
        `analysis_independent_vars` and `system_independent_vars` properties;
        this alias may be deprecated in a future release.

        Contract for prefixed-name entries: every entry of the returned list is
        an opaque label suitable for Snakemake wildcards (charset
        `^[A-Za-z0-9_.]+$`). Consumers MUST NOT deconstruct entries by `.`-split
        or by Pydantic-field lookup against a single model — entries may name
        either an `analysis_config` field (bare) OR a `system.{field}` overlay
        column. Verified consumers (`analysis.py`, `workflow.py`,
        `report_templates/workflow_description.rst.j2`, `config/report.py`,
        `bundle/snakefile_generator.py`) treat entries as opaque labels and
        tolerate the prefixed form without modification.
        """
        return self.analysis_independent_vars + [f"system.{f}" for f in self.system_independent_vars]

    def _retrieve_df_setup(self) -> pd.DataFrame:
        import re as _re

        snstivity_definition = self.master_analysis.cfg_analysis.sensitivity_analysis
        f_extension = snstivity_definition.name.lower().split(".")[-1]  # type: ignore
        if f_extension == "csv":
            df_setup = pd.read_csv(snstivity_definition)  # type: ignore
        elif f_extension == "xlsx":
            df_setup = pd.read_excel(snstivity_definition)
        else:
            raise ValueError("File extension not recognized for file defining sensitivity analysis.")
        if "sa_id" not in df_setup.columns:
            raise ValueError(
                "sensitivity_analysis file must contain a required 'sa_id' column. "
                "Values may be integer or string but must be unique and match "
                "^[A-Za-z0-9_.]+$ to be safe for Snakemake wildcards."
            )
        df_setup["sa_id"] = df_setup["sa_id"].astype(str)
        if not df_setup["sa_id"].is_unique:
            dupes = df_setup["sa_id"][df_setup["sa_id"].duplicated()].tolist()
            raise ValueError(f"sa_id values must be unique. Duplicates: {dupes}")
        pat = _re.compile(r"^[A-Za-z0-9_.]+$")
        bad = [v for v in df_setup["sa_id"] if not pat.match(v)]
        if bad:
            raise ValueError(
                f"sa_id values must match ^[A-Za-z0-9_.]+$ (Snakemake-wildcard safe). Offending values: {bad}"
            )
        df_setup = df_setup.set_index("sa_id")
        # Phase 1 — column allowlist enforcement (post-set_index so sa_id excluded).
        from hhemt.config.analysis import analysis_config
        from hhemt.config.system import system_config

        KNOWN_BARE_COLS = {"system_config_yaml"}
        valid_columns = (
            KNOWN_BARE_COLS
            | set(analysis_config.model_fields)
            | {_SYSTEM_COLUMN_PREFIX + f for f in system_config.model_fields}
            | {_ANALYSIS_COLUMN_PREFIX + f for f in analysis_config.model_fields}
            | {_HPC_COLUMN_PREFIX + k for k in _HPC_ALIAS_TO_ANALYSIS_FIELD}
        )
        unknown = set(df_setup.columns) - valid_columns
        if unknown:
            # Phase 6 (DQ4): a direct `hpc.gpu_hardware` axis is rejected — gpu_hardware
            # is derived-only (R7); cross-hardware variation is expressed via the
            # `hpc.partition` selector (hardware derives from the partition spec).
            hpc_gpu_hint = (
                " To vary GPU hardware across rows, use `hpc.partition` "
                "(gpu_hardware derives from the partition spec); a direct "
                "`hpc.gpu_hardware` axis is not supported."
                if any(c.startswith(_HPC_COLUMN_PREFIX) for c in unknown)
                else ""
            )
            raise ConfigurationError(
                field="sensitivity_analysis.csv_columns",
                message=(
                    f"Unknown sensitivity-CSV columns: {sorted(unknown)}. "
                    f"Valid columns: sa_id (required, becomes index), system_config_yaml, "
                    f"bare analysis_config field names, `system.{{field}}` for system_config fields, "
                    f"`analysis.{{field}}` for analysis_config fields, and the HPC-resource "
                    f"aliases `hpc.partition` / `hpc.setup_partition` (resolving to the "
                    f"analysis_config partition selectors)." + hpc_gpu_hint
                ),
                config_path=snstivity_definition,
            )
        return df_setup

    def export_sensitivity_definition_csv(self) -> Path:
        """Export sensitivity analysis definition to analysis directory as CSV.

        Exports only the fields that vary across sub-analyses (self.df_setup columns)
        to a standardized 'sensitivity_analysis_definition.csv' file in the analysis directory.
        This allows easier inspection of the sensitivity analysis configuration during debugging.

        Returns:
            Path to the exported CSV file.
        """
        output_path = self.analysis_paths.analysis_dir / "sensitivity_analysis_definition.csv"
        df_export = self.df_setup.copy()
        df_export.to_csv(output_path, index=True)
        return output_path

    def find_orphan_subanalysis_dirs(self) -> list[Path]:
        """Return sub-analysis directories on disk whose sa_id is absent from the current CSV.

        The authoritative set of expected sub-analysis directory names is derived
        from ``self.df_setup.index`` (the sensitivity CSV's ``sa_id`` column) and
        the ``self.sub_analyses_prefix`` constant. This ties orphan detection to
        the CSV directly, so a partially-constructed ``self.sub_analyses`` dict
        cannot cause legitimate directories to be misclassified as orphans.
        On-disk ``sa_*`` directories whose suffix fails the Snakemake-wildcard-safe
        charset ``^[A-Za-z0-9_.]+$`` are skipped — they were not created by this
        toolkit and must not be deleted by it. If ``self.subanalysis_dir`` does
        not exist, returns ``[]``.
        """
        import re as _re

        if not self.subanalysis_dir.exists():
            return []
        expected_names = {f"{self.sub_analyses_prefix}{sa_id}" for sa_id in self.df_setup.index.astype(str)}
        charset = _re.compile(r"^[A-Za-z0-9_.]+$")
        orphans: list[Path] = []
        for entry in self.subanalysis_dir.iterdir():
            if not entry.is_dir():
                continue
            if not entry.name.startswith(self.sub_analyses_prefix):
                continue
            suffix = entry.name[len(self.sub_analyses_prefix) :]
            if not charset.match(suffix):
                continue
            if entry.name not in expected_names:
                orphans.append(entry)
        return sorted(orphans)

    def cleanup_orphan_subanalysis_dirs(
        self,
        dry_run: bool = True,
        force: bool = False,
        verbose: bool = True,
    ) -> list[Path]:
        """Identify and optionally delete orphaned sub-analysis directories.

        Uses :meth:`find_orphan_subanalysis_dirs` to locate directories under
        ``subanalyses/`` whose ``sa_id`` no longer appears in the current CSV.

        Parameters
        ----------
        dry_run : bool
            If True (default), only reports orphans without deleting.
        force : bool
            Required when ``dry_run=False``. Without it, the method raises
            ``ValueError`` to guard against accidental deletion of expensive
            HPC outputs.
        verbose : bool
            If True, prints each orphan path via ``print(..., flush=True)``.

        Returns
        -------
        list[Path]
            The orphan directories (either deleted or proposed for deletion).

        Raises
        ------
        ValueError
            If ``dry_run=False`` and ``force=False``.
        """
        from hhemt.utils import fast_rmtree

        orphans = self.find_orphan_subanalysis_dirs()
        if verbose:
            if orphans:
                print(f"[cleanup-orphans] Found {len(orphans)} orphan sub-analysis directories:", flush=True)
                for p in orphans:
                    print(f"  {p}", flush=True)
            else:
                print("[cleanup-orphans] No orphan sub-analysis directories found.", flush=True)
        if dry_run:
            return orphans
        if not force:
            raise ValueError(
                "cleanup_orphan_subanalysis_dirs called with dry_run=False but "
                "force=False. Pass force=True to perform deletion."
            )
        master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
        deleted: list[Path] = []
        failed: list[tuple[Path, Exception]] = []
        for p in orphans:
            if verbose:
                print(f"[cleanup-orphans] Deleting {p}", flush=True)
            try:
                fast_rmtree(p, analysis_dir=master_analysis_dir)  # PATTERN A
                deleted.append(p)
            except Exception as exc:
                failed.append((p, exc))
                if verbose:
                    print(f"[cleanup-orphans] FAILED to delete {p}: {exc}", flush=True)
        if failed:
            summary = "; ".join(f"{p}: {exc}" for p, exc in failed)
            raise RuntimeError(
                f"cleanup_orphan_subanalysis_dirs deleted {len(deleted)} of {len(orphans)} orphans; failures: {summary}"
            )
        return deleted

    def find_orphan_status_flags(self) -> list[Path]:
        """Return _status/ flag files whose embedded sa_id is absent from df_setup.index.

        Matches against the four Snakemake rule-output flag families that embed
        an sa_id (verified against workflow.py rule generation):

        - ``b_prepare_sa-{sa_id}_evt-{event_id}_complete.flag``
        - ``c_run_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag``
        - ``d_process_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag``
        - ``e_consolidate_sa-{sa_id}_complete.flag``

        The sa_id charset is constrained to ``^[A-Za-z0-9_.]+$`` per the
        project stipulation. Returns an empty list if the ``_status/``
        directory does not exist.
        """
        import re as _re

        status_dir = self.analysis_paths.analysis_dir / "_status"
        if not status_dir.exists():
            return []
        expected_sa_ids = set(self.df_setup.index.astype(str))
        # Anchored to the four known rule-name prefixes so unrelated 'sa-'
        # substrings (or future non-sensitivity rules that happen to contain
        # 'sa-') cannot trigger a false orphan.
        pat = _re.compile(
            r"^(?:b_prepare|c_run_[A-Za-z0-9]+|d_process_[A-Za-z0-9]+|e_consolidate)_sa-([A-Za-z0-9_.]+?)(?:_evt-[A-Za-z0-9_.]+|_complete|)\.flag$"
        )
        orphans: list[Path] = []
        for entry in status_dir.glob("*.flag"):
            m = pat.match(entry.name)
            if m is None:
                continue
            sa_id = m.group(1)
            if sa_id not in expected_sa_ids:
                orphans.append(entry)
        return sorted(orphans)

    def find_orphan_input_fingerprints(self) -> list[Path]:
        """Return _status/sa-{sa_id}_inputs.json fingerprint files whose sa_id is absent from df_setup.index.

        The per-sa_id input-fingerprint files (Gotcha 17) are written by
        ``SensitivityAnalysisWorkflowBuilder`` at Snakefile-build time and named
        ``sa-{sa_id}_inputs.json`` under ``_status/``. When an sa_id is removed
        from the sensitivity CSV its fingerprint is orphaned just like its
        dirs/flags/datatree groups. Returns an empty list if ``_status/`` is absent.
        """
        import re as _re

        status_dir = self.analysis_paths.analysis_dir / "_status"
        if not status_dir.exists():
            return []
        expected_sa_ids = set(self.df_setup.index.astype(str))
        pat = _re.compile(r"^sa-([A-Za-z0-9_.]+?)_inputs\.json$")
        orphans: list[Path] = []
        for entry in status_dir.glob("sa-*_inputs.json"):
            m = pat.match(entry.name)
            if m is None:
                continue
            sa_id = m.group(1)
            if sa_id not in expected_sa_ids:
                orphans.append(entry)
        return sorted(orphans)

    def find_orphan_datatree_groups(self) -> list[str]:
        """Return sa_id strings present as subgroups in sensitivity_datatree.zarr but absent from df_setup.index.

        Inspects on-disk subdirectories of ``sensitivity_datatree.zarr/`` matching
        ``{prefix}{sa_id}`` where ``prefix`` is ``self.sub_analyses_prefix``. Returns
        the sa_id strings (without prefix). Returns an empty list if the zarr
        does not exist.
        """
        zarr_path = self.analysis_paths.sensitivity_datatree_zarr
        if zarr_path is None or not zarr_path.exists():
            return []
        expected_sa_ids = set(self.df_setup.index.astype(str))
        prefix = self.sub_analyses_prefix
        orphans: list[str] = []
        for entry in zarr_path.iterdir():
            if not entry.is_dir():
                continue
            if not entry.name.startswith(prefix):
                continue
            sa_id = entry.name[len(prefix) :]
            if sa_id and sa_id not in expected_sa_ids:
                orphans.append(sa_id)
        return sorted(orphans)

    def cleanup_all_orphans(
        self,
        dry_run: bool = True,
        force: bool = False,
        verbose: bool = True,
    ) -> dict[str, list]:
        """Detect and (optionally) delete orphan subanalysis dirs, flags, datatree groups, and input fingerprints.

        When any orphan is detected and deletion proceeds, the entire
        ``sensitivity_datatree.zarr`` is removed (rebuild approach — see plan
        D-SURGICAL) and the master-consolidation status flag is also removed so
        Snakemake re-runs the master_consolidation rule on the next workflow run.

        Parameters
        ----------
        dry_run : bool
            If True (default), only reports without deleting.
        force : bool
            Required when ``dry_run=False``.
        verbose : bool
            If True, prints each deletion via ``print(..., flush=True)``.

        Returns
        -------
        dict[str, list | bool]
            Keys: ``"dirs"`` (list[Path]), ``"status_flags"`` (list[Path]),
            ``"datatree_groups"`` (list[str]), ``"input_fingerprints"`` (list[Path]),
            and (after deletion only)
            ``"sensitivity_datatree_removed"`` (bool) and
            ``"master_flag_removed"`` (bool) reporting whether the
            rebuild-trigger artifacts were actually removed.

        Raises
        ------
        ValueError
            If ``dry_run=False`` and ``force=False``.
        """
        from hhemt.utils import fast_rmtree

        result = {
            "dirs": self.find_orphan_subanalysis_dirs(),
            "status_flags": self.find_orphan_status_flags(),
            "datatree_groups": self.find_orphan_datatree_groups(),
            "input_fingerprints": self.find_orphan_input_fingerprints(),
        }
        any_orphan = bool(
            result["dirs"] or result["status_flags"] or result["datatree_groups"] or result["input_fingerprints"]
        )
        if verbose:
            if any_orphan:
                print(
                    f"[cleanup-orphans] dirs={len(result['dirs'])} "
                    f"status_flags={len(result['status_flags'])} "
                    f"datatree_groups={len(result['datatree_groups'])}",
                    flush=True,
                )
                for p in result["dirs"]:
                    print(f"  dir: {p}", flush=True)
                for p in result["status_flags"]:
                    print(f"  flag: {p}", flush=True)
                for sa_id in result["datatree_groups"]:
                    print(f"  datatree-group: sa_{sa_id}", flush=True)
                for p in result["input_fingerprints"]:
                    print(f"  input-fingerprint: {p}", flush=True)
            else:
                print("[cleanup-orphans] No orphans detected.", flush=True)
        if dry_run:
            return result
        if not force:
            raise ValueError(
                "cleanup_all_orphans called with dry_run=False but force=False. Pass force=True to perform deletion."
            )
        master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
        for p in result["dirs"]:
            if verbose:
                print(f"[cleanup-orphans] Deleting dir {p}", flush=True)
            fast_rmtree(p, analysis_dir=master_analysis_dir)  # PATTERN A
        for p in result["status_flags"]:
            if verbose:
                print(f"[cleanup-orphans] Unlinking flag {p}", flush=True)
            # EXEMPT-DU: status-flag
            p.unlink()
        for p in result["input_fingerprints"]:
            if verbose:
                print(f"[cleanup-orphans] Unlinking input-fingerprint {p}", flush=True)
            # EXEMPT-DU: status-dir-cleanup
            p.unlink()
        result["sensitivity_datatree_removed"] = False
        result["master_flag_removed"] = False
        if any_orphan:
            zarr_path = self.analysis_paths.sensitivity_datatree_zarr
            if zarr_path is not None and zarr_path.exists():
                if verbose:
                    print(
                        f"[cleanup-orphans] Deleting sensitivity_datatree.zarr (rebuild on next run): {zarr_path}",
                        flush=True,
                    )
                fast_rmtree(zarr_path, analysis_dir=master_analysis_dir)  # PATTERN A
                result["sensitivity_datatree_removed"] = True
            master_flag = self.analysis_paths.analysis_dir / "_status" / "f_consolidate_master_complete.flag"
            if master_flag.exists():
                if verbose:
                    print(
                        f"[cleanup-orphans] Unlinking master-consolidation flag {master_flag}",
                        flush=True,
                    )
                # EXEMPT-DU: status-flag
                master_flag.unlink()
                result["master_flag_removed"] = True
        return result

    def _build_unique_system_targets(
        self,
        df_setup_full: pd.DataFrame,
        is_main_orchestrator: bool = True,
    ) -> list[UniqueSystemTarget]:
        """Resolve per-sa_id system targets and materialize per-target synthesized YAMLs.

        Handles three per-row mechanisms:

        1. ``system_config_yaml`` column (path to a per-sa system YAML).
        2. ``system.{field}`` overlay columns (Phase 1 prefixed-column mechanism).
        3. Neither — fall back to master ``self._system``.

        Mutual exclusion: a single row may use mechanism 1 OR mechanism 2, never both;
        violation raises ``ConfigurationError``.

        ``is_main_orchestrator=True`` purges ``_generated/`` before emission.
        Runner subprocesses pass ``False`` to skip the purge.
        """
        import pydantic

        from hhemt.config.system import system_config
        from hhemt.system import TRITONSWMM_system
        from hhemt.utils import fast_rmtree

        sensitivity_csv = self.master_analysis.cfg_analysis.sensitivity_analysis
        analysis_dir = self.master_analysis.analysis_paths.analysis_dir
        generated_dir = analysis_dir / "_generated"

        if is_main_orchestrator:
            # PATTERN A (_generated is DU-counted; not _status*-prefixed)
            fast_rmtree(generated_dir, missing_ok=True, analysis_dir=analysis_dir)
            generated_dir.mkdir(parents=True, exist_ok=True)

        has_yaml_col = "system_config_yaml" in df_setup_full.columns
        overlay_col_names = sorted(c for c in df_setup_full.columns if _is_system_overlay_column(c))

        # Group sub-analyses by their compile-key tuple.
        groups: dict[tuple, dict] = {}

        for sa_id, row in df_setup_full.iterrows():
            sa_id_str = str(sa_id)
            yaml_cell = row.get("system_config_yaml") if has_yaml_col else None
            yaml_specified = yaml_cell is not None and not pd.isna(yaml_cell) and str(yaml_cell).strip() != ""
            overlay_cells = {_strip_system_prefix(c): row[c] for c in overlay_col_names if not pd.isna(row[c])}
            if overlay_cells and yaml_specified:
                raise ConfigurationError(
                    field=f"sensitivity_analysis.row[{sa_id_str}]",
                    message=(
                        f"sa_id={sa_id_str}: row specifies both system_config_yaml "
                        f"({yaml_cell}) and system.* overlay column(s) "
                        f"{sorted(overlay_cells)}; mutually exclusive — "
                        f"use one mechanism per row."
                    ),
                    config_path=sensitivity_csv,
                )

            if overlay_cells:
                try:
                    # NOTE: the base model_dump() carries TRITONSWMM/SWMM software-dir
                    # Paths that may not yet exist on disk (created by system.py's
                    # clone/build gate at run/setup). They are exempt from
                    # _check_paths_exist via json_schema_extra={"toolkit_owned_output":
                    # True}. Do NOT add an existence assertion here.
                    cfg = system_config.model_validate(
                        {
                            **self._system.cfg_system.model_dump(),
                            **overlay_cells,
                        }
                    )
                except pydantic.ValidationError as exc:
                    raise ConfigurationError(
                        field=f"sensitivity_analysis.row[{sa_id_str}]",
                        message=(
                            f"sa_id={sa_id_str}: system.* overlay-column values failed SystemConfig validation: {exc}"
                        ),
                        config_path=sensitivity_csv,
                    ) from exc
            elif yaml_specified:
                yaml_path = Path(yaml_cell).resolve()
                if not yaml_path.is_file():
                    raise ConfigurationError(
                        field="sensitivity_analysis.system_config_yaml",
                        message=(f"sa_id={sa_id_str}: system_config_yaml does not exist at {yaml_path}."),
                        config_path=sensitivity_csv,
                    )
                cfg = TRITONSWMM_system(yaml_path).cfg_system
            else:
                cfg = self._system.cfg_system

            # Phase 6 (DQ7a): resolve the ensemble partition PER ROW from the
            # overlay (analysis.hpc_ensemble_partition / hpc.partition), falling back
            # to the master. The dedup key STAYS hardware-derived (two same-hardware
            # partitions collapse to one build target) — NOT partition-name-keyed —
            # but the hardware now derives from each row's partition, so a6000 + a100
            # rows produce DISTINCT build targets.
            _row_partition = _resolve_row_ensemble_partition(
                row, self.master_analysis.cfg_analysis.hpc_ensemble_partition
            )
            _gpu_hardware, _gpu_backend = resolve_gpu_target(
                self.master_analysis.cfg_hpc_system,
                _row_partition,
            )
            key = (
                cfg.target_dem_resolution,
                _gpu_hardware,
                _gpu_backend,
            )
            if key not in groups:
                groups[key] = {"cfg": cfg, "sa_ids": [], "partition": _row_partition}
            groups[key]["sa_ids"].append(sa_id_str)

        # Phase-4 (4c): GPU hardware/backend + modules are injected into each target
        # system (retired off system_config), resolved from the master ensemble
        # partition (uniform in 4c). The master self._system represents that same
        # ensemble target, so populate its injected attrs too (so the reuse branch
        # below is GPU-correct, not a None-injected CPU system).
        _gpu_hardware, _gpu_backend = resolve_gpu_target(
            self.master_analysis.cfg_hpc_system,
            self.master_analysis.cfg_analysis.hpc_ensemble_partition,
        )
        _modules = resolve_additional_modules(self.master_analysis.cfg_hpc_system)
        # synth_cc friction fix (2026-07-08): inject the master-ensemble GPU pair into
        # self._system ONLY on the DRIVER (is_main_orchestrator=True). It is a
        # driver-only template feeding the reuse-branch below + the workflow.py GPU-
        # sensitivity validation (`not self.system.gpu_compilation_backend`). In a
        # setup_target_N RUNNER subprocess (is_main_orchestrator=False) self._system IS
        # the compile target that setup_workflow.py already constructed correctly from
        # --target-partition (backend=None for a CPU/standard partition, with
        # sys_paths.compilation_script_gpu frozen to None). Overwriting it here with the
        # master ensemble backend flips the CPU target into the GPU branch of
        # compile_TRITON_SWMM (system.py:522) against a None compile-script path ->
        # AttributeError at system.py:813; it also silently rewrites a non-master GPU
        # target's arch (a100 -> master a6000). sys_paths is frozen at construction and
        # is NOT rebuilt by this assignment, so the runner MUST keep its constructed pair.
        if is_main_orchestrator:
            self._system.gpu_hardware = _gpu_hardware
            self._system.gpu_compilation_backend = _gpu_backend
            self._system.additional_modules = _modules

        targets: list[UniqueSystemTarget] = []
        for target_id, group in enumerate(groups.values()):
            cfg = group["cfg"]
            sa_ids = group["sa_ids"]
            _target_partition = group["partition"]
            # Phase 6 (DQ7a): resolve THIS target's (hw, backend, modules) from its
            # own partition — distinct targets (a6000 vs a100) inject distinct pairs.
            _t_hw, _t_backend = resolve_gpu_target(self.master_analysis.cfg_hpc_system, _target_partition)
            _t_modules = resolve_additional_modules(self.master_analysis.cfg_hpc_system)
            generated_yaml = generated_dir / f"target_{target_id}.yaml"
            if is_main_orchestrator:
                # Temp-file-rename for atomicity (PID-keyed per Gotcha 17 pattern).
                tmp_yaml = generated_dir / f"target_{target_id}.{os.getpid()}.tmp.yaml"
                with tmp_yaml.open("w") as fh:
                    yaml.safe_dump(cfg.model_dump(mode="json"), fh, sort_keys=False)
                tmp_yaml.rename(generated_yaml)
            # Reuse master self._system only when BOTH the resolved cfg AND the
            # injected GPU pair match the master's (else a same-cfg, different-hw
            # target would wrongly reuse the master CPU/hardware system).
            if (
                cfg.model_dump_json() == self._system.cfg_system.model_dump_json()
                and _t_hw == self._system.gpu_hardware
                and _t_backend == self._system.gpu_compilation_backend
            ):
                target_system = self._system
            else:
                target_system = TRITONSWMM_system(
                    generated_yaml,
                    gpu_hardware=_t_hw,
                    gpu_compilation_backend=_t_backend,
                    additional_modules=_t_modules,
                    # ADR-1/M-7: this per-UniqueSystemTarget system DRIVES the
                    # sensitivity compile (compile_TRITON_SWMM below). In container
                    # mode the libstdc++ exact-soname patch must be suppressed (the
                    # SIF carries a self-consistent toolchain); the False default in
                    # native mode keeps the link line byte-identical.
                    execution_container_mode=(self.master_analysis.cfg_analysis.execution_environment == "container"),
                )
            targets.append(
                UniqueSystemTarget(
                    target_id=target_id,
                    system_config_yaml=generated_yaml,
                    system=target_system,
                    sub_analysis_ids=sa_ids,
                    target_partition=_target_partition,
                )
            )

        return targets

    def _materialize_target_yamls(self) -> None:
        """Re-write the per-target ``_generated/target_*.yaml`` files from the
        already-resolved in-memory :attr:`unique_system_targets`.

        ``_build_unique_system_targets`` writes these YAMLs at CONSTRUCTION
        (``is_main_orchestrator=True``), and the ``setup_target_N`` rules emitted by
        :meth:`SensitivityAnalysisWorkflowBuilder.generate_master_snakefile_content`
        reference their ABSOLUTE paths. But ``analysis.run(from_scratch=True)``
        ``fast_rmtree``s the analysis_dir AFTER construction, deleting ``_generated/``
        with nothing re-materializing it, so the setup rules fail with
        ``System config not found: .../_generated/target_0.yaml``. The master-Snakefile
        generator therefore calls this at generation time (post-wipe, before the setup
        rules are written) so the referenced files always exist.

        Only targets whose ``system_config_yaml`` lives under ``analysis_dir/_generated``
        are (re)written — the fast-path target points at the master system config
        (outside analysis_dir, never wiped) and is skipped. Idempotent: a byte-identical
        re-write on the resume path is harmless because the target YAMLs are shell ARGs,
        not declared Snakemake ``input:`` (no rerun trigger).
        """
        analysis_dir = self.master_analysis.analysis_paths.analysis_dir
        generated_dir = (analysis_dir / "_generated").resolve()
        for target in self.unique_system_targets:
            yaml_path = Path(target.system_config_yaml)
            try:
                under_generated = yaml_path.resolve().is_relative_to(generated_dir)
            except (OSError, ValueError):
                under_generated = False
            if not under_generated:
                continue
            generated_dir.mkdir(parents=True, exist_ok=True)
            # Atomic temp-file rename (PID-keyed; mirrors the construction-time write).
            tmp_yaml = generated_dir / f"{yaml_path.stem}.{os.getpid()}.tmp.yaml"
            with tmp_yaml.open("w") as fh:
                yaml.safe_dump(target.system.cfg_system.model_dump(mode="json"), fh, sort_keys=False)
            tmp_yaml.rename(yaml_path)

    def _create_sub_analyses(self):
        sa_id_to_system: dict = {}
        for target in self.unique_system_targets:
            for sa_id in target.sub_analysis_ids:
                sa_id_to_system[sa_id] = target.system

        from hhemt.config.analysis import analysis_config

        dic_sensitivity_analyses = dict()
        for idx, row in self.df_setup.iterrows():
            sa_id = str(idx)
            overlay_cells: dict = {}
            for k, v in row.items():
                if pd.isna(v):
                    continue
                if _is_hpc_overlay_column(k):
                    overlay_cells[_resolve_hpc_alias_to_analysis_field(k)] = v
                elif _is_analysis_overlay_column(k):
                    overlay_cells[_strip_analysis_prefix(k)] = v
                elif k in analysis_config.model_fields:
                    warnings.warn(
                        f"Bare-name analysis-config column `{k}` is deprecated; "
                        f"rename to `analysis.{k}` for the canonical prefixed-column form. "
                        f"Bare-name support will be removed in a future release.",
                        DeprecationWarning,
                        stacklevel=2,
                    )
                    overlay_cells[k] = v
                else:
                    # Defensive — `_retrieve_df_setup`'s column allowlist plus the
                    # analysis-only `self.df_setup` projection should have filtered
                    # `sa_id`/`system_config_yaml`/`system.*` already.
                    raise ConfigurationError(
                        field="sensitivity_analysis.unknown_column",
                        message=f"Column `{k}` is not a recognized analysis-config field.",
                    )
            cfg_snstvty_analysis = analysis_config.model_validate(
                {
                    **self.master_analysis.cfg_analysis.model_dump(),
                    **overlay_cells,
                }
            )
            analysis_id = f"{self.sub_analyses_prefix}{sa_id}"
            cfg_snstvty_analysis.analysis_id = analysis_id  # type: ignore
            sub_analysis_directory = self.subanalysis_dir / str(cfg_snstvty_analysis.analysis_id)
            sub_analysis_directory.mkdir(parents=True, exist_ok=True)
            cfg_snstvty_analysis.toggle_sensitivity_analysis = False
            cfg_snstvty_analysis.is_subanalysis = True

            cfg_anlysys_yaml = sub_analysis_directory / f"{analysis_id}.yaml"

            cfg_snstvty_analysis.analysis_dir = sub_analysis_directory

            cfg_snstvty_analysis.master_analysis_cfg_yaml = self.master_analysis.analysis_config_yaml

            # Atomic write via temp-file + rename. `Path.write_text` truncates the
            # target before writing; concurrent readers in other plot subprocesses
            # would catch the truncated-empty state and fail with
            # `model_validate(None)`. POSIX `rename(2)` (Path.replace) is atomic
            # on the same filesystem, so readers always see a complete file.
            #
            # Temp filename is keyed on PID so concurrent writers do not collide
            # on the same `*.tmp` path (one writer's `replace()` would otherwise
            # move the tmp out from under another, raising FileNotFoundError on
            # the second writer's replace). PID is unique per OS process within
            # a job's lifetime; subprocess A and subprocess B always write to
            # distinct tmp files before swapping into the target.
            #
            # Once the deeper fix lands (sub-analysis yaml materialization lifted
            # out of `__init__` into the setup phase), this temp-file dance can
            # collapse back to a single `cfg_anlysys_yaml.write_text(...)`.
            _tmp = cfg_anlysys_yaml.with_suffix(cfg_anlysys_yaml.suffix + f".{os.getpid()}.tmp")
            _tmp.write_text(
                yaml.safe_dump(
                    cfg_snstvty_analysis.model_dump(mode="json"),
                    sort_keys=False,
                )
            )
            _tmp.replace(cfg_anlysys_yaml)
            anlsys = anlysis.TRITONSWMM_analysis(
                analysis_config_yaml=cfg_anlysys_yaml,
                system=sa_id_to_system[sa_id],
                skip_log_update=self._skip_log_update,
                # Phase 6 (DQ7): thread the master's hpc_system_config to each sub so
                # the per-sub partition -> gpu_hardware / gpus_per_node resolution
                # (resolve_gpu_target / resolve_gpus_per_node, consumed by the per-sa
                # GRES emission) works for cross-hardware sensitivity. Subs previously
                # carried cfg_hpc_system=None, which was latent only because the GRES
                # block is gated on n_gpus>0 (the synth CPU fixtures never tripped it).
                hpc_system_config_yaml=self.master_analysis.hpc_system_config_yaml,
            )
            dic_sensitivity_analyses[sa_id] = anlsys
        return dic_sensitivity_analyses

    def _compute_sa_id_fingerprint_payload(self, sub_analysis: "anlysis.TRITONSWMM_analysis") -> dict[str, object]:
        """Compute the deterministic fingerprint payload for one sub-analysis.

        Projects the sub-analysis's post-Pydantic ``analysis_config.model_dump(mode="json")``
        onto the canonical field names from ``self.analysis_independent_vars``
        (sorted). Adds a ``__schema_version__`` sentinel so future
        serializer-format changes are themselves observable. Excludes ``sa_id``
        (the path already disambiguates).

        Stability contract: every ``analysis_config`` field that may appear in
        ``self.analysis_independent_vars`` must be JSON-stable under ``model_dump(mode="json")``
        — that is, two invocations on the same sub_analysis instance must produce
        byte-identical ``json.dumps(..., sort_keys=True)`` output. The currently-known
        sensitivity-CSV columns (``cpus_per_sim``, ``n_omp_threads``, ``hpc_total_nodes``,
        ``hpc_max_simultaneous_sims``, ``hpc_total_job_duration_min``, ``run_mode``) are
        all native Python int/str/Literal types and meet the contract. Adding a
        new ``analysis_config`` field that may legitimately become a sensitivity-CSV
        column requires re-checking JSON stability and may require bumping
        ``__schema_version__``.

        Returns a plain dict suitable for ``json.dumps`` with ``sort_keys=True``.
        """
        cfg_dump = sub_analysis.cfg_analysis.model_dump(mode="json")
        # KeyError on missing key — surfaces config-schema drift loudly rather than
        # producing fingerprints that silently project None for an absent field.
        # Phase 2 — project against `analysis_independent_vars` (canonical stripped
        # names) rather than the BC alias `independent_vars` (which includes
        # `system.*` entries that have no key in `cfg_dump`).
        payload: dict[str, object] = {
            "__schema_version__": 1,
            "fields": {k: cfg_dump[k] for k in sorted(self.analysis_independent_vars)},
        }
        # When the sensitivity CSV declares a `system_config_yaml` column, bump the
        # schema and attach a SHA-1 of the sub-analysis's resolved cfg_system. This
        # invalidates any sa_id whose system config changes between runs. The
        # schema bump intentionally invalidates every sa_id on the first run that
        # introduces per-sa system configs (Gotcha 17 cascade — see Phase 1 doc).
        if self._has_per_sa_system_configs:
            payload["__schema_version__"] = 2
            cfg_system_json = sub_analysis._system.cfg_system.model_dump_json(by_alias=False, exclude_none=False)
            payload["system_cfg_hash"] = hashlib.sha1(cfg_system_json.encode("utf-8")).hexdigest()

        # Phase 1 — attach system_overlay key when any system.* overlay columns
        # are declared on the master sensitivity df (un-projected).
        from hhemt.config.system import system_config

        df = self.master_analysis.sensitivity._df_setup_full
        overlay_col_names = [c for c in df.columns if _is_system_overlay_column(c)]
        if overlay_col_names:
            sa_id_str = sub_analysis.cfg_analysis.analysis_id.removeprefix(
                self.master_analysis.sensitivity.sub_analyses_prefix
            )
            overlay_cells = {
                _strip_system_prefix(c): df.loc[sa_id_str, c]
                for c in overlay_col_names
                if not pd.isna(df.loc[sa_id_str, c])
            }
            if overlay_cells:
                resolved = system_config.model_validate(
                    {
                        **self.master_analysis._system.cfg_system.model_dump(),
                        **overlay_cells,
                    }
                )
                resolved_overlay = {k: resolved.model_dump(mode="json")[k] for k in overlay_cells}
            else:
                resolved_overlay = {}
            payload["__schema_version__"] = 3
            payload["system_overlay"] = resolved_overlay
        return payload

    def _write_sa_id_fingerprint(
        self,
        sub_analysis: "anlysis.TRITONSWMM_analysis",
        fingerprint_path: Path,
    ) -> bool:
        """Write the per-sa_id fingerprint file via compare-and-write.

        Reads ``fingerprint_path`` if it exists, serializes the new payload with
        ``sort_keys=True`` and stable separators, and only rewrites the file when
        content differs. This preserves mtime when content is unchanged — the
        mechanism on which Snakemake's per-rule rerun gating depends.

        Returns ``True`` if the file was (re)written, ``False`` if skipped because
        content matched the existing file.
        """
        payload = self._compute_sa_id_fingerprint_payload(sub_analysis)
        new_text = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n"
        # Treat unreadable existing content (zero-byte, corrupted, encoding error
        # from a crashed prior workflow) as "not equal to new content" and proceed
        # to overwrite. This preserves the compare-and-write contract under the
        # one failure mode the contract cannot otherwise diagnose.
        try:
            existing = fingerprint_path.read_text() if fingerprint_path.exists() else None
        except (OSError, UnicodeDecodeError):
            existing = None
        if existing == new_text:
            return False
        fingerprint_path.parent.mkdir(parents=True, exist_ok=True)
        fingerprint_path.write_text(new_text)
        return True

    def compile_and_preprocess_all_targets(
        self,
        overwrite_system_inputs: bool = False,
        recompile_if_already_done_successfully: bool = False,
        verbose: bool = True,
    ):
        """Process system-level inputs and compile TRITON-SWMM for each unique target.

        Iterates ``self.unique_system_targets`` (populated in ``__init__``) and runs
        ``process_system_level_inputs()`` + ``compile_TRITON_SWMM()`` once per target.
        In the no-per-sub-analysis-config case the list contains a single target
        wrapping the master system, so this method is the unified entry point for
        non-Snakemake direct execution regardless of whether per-sa configs are used.
        """
        for target in self.unique_system_targets:
            if verbose:
                print(
                    f"[Setup] Processing target {target.target_id} ({len(target.sub_analysis_ids)} sub-analyses)",
                    flush=True,
                )
            target.system.process_system_level_inputs(
                overwrite_outputs_if_already_created=overwrite_system_inputs,
                verbose=verbose,
            )
            target.system.compile_TRITON_SWMM(
                recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                verbose=verbose,
            )
        self._update_master_analysis_log()

    def compile_TRITON_SWMM_for_sensitivity_analysis(
        self,
        verbose: bool = False,
        recompile_if_already_done_successfully: bool = False,
    ):
        for target in self.unique_system_targets:
            target.system.compile_TRITON_SWMM(
                recompile_if_already_done_successfully=recompile_if_already_done_successfully,
                verbose=verbose,
            )
        self._update_master_analysis_log()
        return

    @property
    def scenarios_not_created(self):
        scenarios_not_created = []
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            for event_iloc in sub_analysis.df_sims.index:
                scen = TRITONSWMM_scenario(event_iloc, sub_analysis)
                if scen.log.scenario_creation_complete.get() is not True:
                    scenarios_not_created.append(str(scen.log.logfile.parent))
        return scenarios_not_created

    @property
    def scenarios_not_run(self):
        scens_not_run = []
        for _sub_analysis_iloc, sub_analysis in self.sub_analyses.items():
            for event_iloc in sub_analysis.df_sims.index:
                scen = TRITONSWMM_scenario(event_iloc, sub_analysis)
                # Check if all enabled models completed
                enabled_models = scen.run.model_types_enabled
                all_models_completed = all(scen.model_run_completed(model_type) for model_type in enabled_models)
                if not all_models_completed:
                    scens_not_run.append(str(scen.log.logfile.parent))
        return scens_not_run

    def classify_incomplete_sim_failures(self) -> dict[str, str]:
        """Scan model logs for all incomplete simulations across sub-analyses and classify each failure.

        Aggregates ``_classify_model_log_failure()`` across all sub-analyses.
        Works for both ``"1_job_many_srun_tasks"`` and ``"batch_job"`` execution
        methods — the SLURM cancellation marker appears in the model log in both cases.

        Returns
        -------
        dict[str, str]
            Maps scenario identifier (e.g. ``"sa1_0"``) to failure class:

            - ``"timeout"`` — log contains ``DUE TO TIME LIMIT``
            - ``"unclassified"`` — log exists but no known failure marker found
            - ``"no_log"`` — model log file does not exist
        """
        results: dict[str, str] = {}
        for sa_id, sub_analysis in self.sub_analyses.items():
            for event_iloc in sub_analysis.df_sims.index:
                scen = TRITONSWMM_scenario(event_iloc, sub_analysis)
                enabled_models = scen.run.model_types_enabled
                for model_type in enabled_models:
                    if not scen.model_run_completed(model_type):
                        event_id = scen.event_id
                        key = f"sa-{sa_id}_evt-{event_id}"
                        results[key] = scen.run._classify_model_log_failure(model_type)
        return results

    @property
    def is_timeout_only_failure(self) -> bool:
        """True iff all incomplete simulations across sub-analyses have timeout-classified failures.

        Returns False if there are no incomplete sims (all done), or if any
        incomplete sim has an unclassified or no_log failure.
        """
        failures = self.classify_incomplete_sim_failures()
        if not failures:
            return False
        return all(v == "timeout" for v in failures.values())

    @property
    def df_status(self):
        """
        Get status DataFrame for all scenarios across all sub-analyses.

        Returns
        -------
        pd.DataFrame
            Concatenated status table from all sub-analyses. This includes
            sub-analysis-specific setup columns plus the canonical status
            schema from ``TRITONSWMM_analysis.df_status`` (e.g. ``scenario_setup``
            and ``run_completed``), as well as:

            - sub_analysis_iloc: int - Sub-analysis index
        """
        status_frames = []

        for sa_id, sub_analysis in self.sub_analyses.items():
            assert sub_analysis.cfg_analysis.is_subanalysis, (
                "is_subanalysis attribute not true in sub_analysis.cfg_analysis.is_subanalysis"
            )
            sub_df_status = sub_analysis.df_status.copy()

            setup_row = self.df_setup.loc[sa_id, :]
            for key, val in setup_row.items():
                sub_df_status[key] = val

            sub_df_status["sa_id"] = sa_id
            sub_df_status = sub_df_status[["sa_id"] + [c for c in sub_df_status.columns if c != "sa_id"]]

            status_frames.append(sub_df_status)

        if len(status_frames) == 0:
            return pd.DataFrame()

        return pd.concat(status_frames, ignore_index=True)

    @property
    def all_scenarios_created(self):
        """
        Check if all scenarios across all sub-analyses have been created.

        Returns
        -------
        bool
            True if all scenarios in all sub-analyses are created successfully
        """
        all_scenarios_created = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_scenarios_created = all_scenarios_created and sub_analysis._all_scenarios_created
        return all_scenarios_created is True

    @property
    def all_sims_run(self):
        """
        Check if all simulations across all sub-analyses have completed.

        Returns
        -------
        bool
            True if all simulations in all sub-analyses completed successfully
        """
        all_sims_run = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_sims_run = all_sims_run and sub_analysis._all_sims_run
        return all_sims_run is True

    @property
    def all_TRITON_timeseries_processed(self):
        """
        Check if all TRITON timeseries across all sub-analyses have been processed.

        Returns
        -------
        bool
            True if all TRITON outputs in all sub-analyses are processed
        """
        all_TRITON_timeseries_processed = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_TRITON_timeseries_processed = (
                all_TRITON_timeseries_processed and sub_analysis._all_TRITON_timeseries_processed
            )
        return all_TRITON_timeseries_processed is True

    @property
    def all_SWMM_timeseries_processed(self):
        """
        Check if all SWMM timeseries across all sub-analyses have been processed.

        Returns
        -------
        bool
            True if all SWMM outputs in all sub-analyses are processed
        """
        all_SWMM_timeseries_processed = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_SWMM_timeseries_processed = (
                all_SWMM_timeseries_processed and sub_analysis._all_SWMM_timeseries_processed
            )
        return all_SWMM_timeseries_processed is True

    @property
    def all_TRITONSWMM_performance_timeseries_processed(self):
        """
        Check if all performance timeseries across all sub-analyses have been processed.

        Returns
        -------
        bool
            True if all performance outputs in all sub-analyses are processed
        """
        all_TRITONSWMM_performance_timeseries_processed = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_TRITONSWMM_performance_timeseries_processed = (
                all_TRITONSWMM_performance_timeseries_processed
                and sub_analysis._all_TRITONSWMM_performance_timeseries_processed
            )
        return all_TRITONSWMM_performance_timeseries_processed is True

    @property
    def TRITONSWMM_performance_time_series_not_processed(self):
        lst_scens = []
        for _key, sub_analysis in self.sub_analyses.items():
            lst_scens += sub_analysis._TRITONSWMM_performance_time_series_not_processed
        return lst_scens

    @property
    def TRITON_time_series_not_processed(self):
        lst_scens = []
        for _key, sub_analysis in self.sub_analyses.items():
            lst_scens += sub_analysis._TRITON_time_series_not_processed
        return lst_scens

    @property
    def SWMM_time_series_not_processed(self):
        lst_scens = []
        for _key, sub_analysis in self.sub_analyses.items():
            lst_scens += sub_analysis._SWMM_time_series_not_processed
        return lst_scens

    @property
    def all_raw_TRITON_outputs_cleared(self):
        """
        Check if all raw TRITON outputs across all sub-analyses have been cleared.

        Returns
        -------
        bool
            True if all raw TRITON outputs in all sub-analyses are cleared
        """
        all_raw_TRITON_outputs_cleared = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_raw_TRITON_outputs_cleared = (
                all_raw_TRITON_outputs_cleared and sub_analysis._all_raw_TRITON_outputs_cleared
            )
        return all_raw_TRITON_outputs_cleared is True

    @property
    def all_raw_SWMM_outputs_cleared(self):
        """
        Check if all raw SWMM outputs across all sub-analyses have been cleared.

        Returns
        -------
        bool
            True if all raw SWMM outputs in all sub-analyses are cleared
        """
        all_raw_SWMM_outputs_cleared = True
        for _key, sub_analysis in self.sub_analyses.items():
            all_raw_SWMM_outputs_cleared = all_raw_SWMM_outputs_cleared and sub_analysis._all_raw_SWMM_outputs_cleared
        return all_raw_SWMM_outputs_cleared is True

    def _update_master_analysis_log(self):
        self.master_analysis._update_log()
        return

all_SWMM_timeseries_processed property

Check if all SWMM timeseries across all sub-analyses have been processed.

Returns:

Type	Description
bool	True if all SWMM outputs in all sub-analyses are processed

all_TRITONSWMM_performance_timeseries_processed property

Check if all performance timeseries across all sub-analyses have been processed.

Returns:

Type	Description
bool	True if all performance outputs in all sub-analyses are processed

all_TRITON_timeseries_processed property

Check if all TRITON timeseries across all sub-analyses have been processed.

Returns:

Type	Description
bool	True if all TRITON outputs in all sub-analyses are processed

all_raw_SWMM_outputs_cleared property

Check if all raw SWMM outputs across all sub-analyses have been cleared.

Returns:

Type	Description
bool	True if all raw SWMM outputs in all sub-analyses are cleared

all_raw_TRITON_outputs_cleared property

Check if all raw TRITON outputs across all sub-analyses have been cleared.

Returns:

Type	Description
bool	True if all raw TRITON outputs in all sub-analyses are cleared

all_scenarios_created property

Check if all scenarios across all sub-analyses have been created.

Returns:

Type	Description
bool	True if all scenarios in all sub-analyses are created successfully

all_sims_run property

Check if all simulations across all sub-analyses have completed.

Returns:

Type	Description
bool	True if all simulations in all sub-analyses completed successfully

analysis_independent_vars property

Phase 2 — analysis-config attributes varied across sub-analyses.

Returns the canonical (stripped) field name for each varied analysis-config column. Recognizes both analysis.{field} (canonical) and bare field names (deprecated; emits DeprecationWarning at sub-analysis construction time via _create_sub_analyses).

df_setup_with_system_overlays property

df_setup unioned with the system.* overlay columns from the full frame.

df_setup is filtered to analysis-config columns only (:173-180); the system.* overlay columns are retained on _df_setup_full. Report renderers that plot a system-axis independent variable (system.target_dem_resolution, system.gpu_hardware, ...) need both sets in one frame. This accessor unions them on the shared sa_id index, preserving the PREFIXED column names so a renderer's independent_var="system.gpu_hardware" lookup resolves directly.

The frame is analysis-columns + system-overlay-columns only — NOT the raw _df_setup_full (which also carries system_config_yaml and non-overlay annotation columns), keeping the renderer's column membership check scoped to the resolvable independent-var set.

df_status property

Get status DataFrame for all scenarios across all sub-analyses.

Returns:

Type	Description
DataFrame	Concatenated status table from all sub-analyses. This includes sub-analysis-specific setup columns plus the canonical status schema from TRITONSWMM_analysis.df_status (e.g. scenario_setup and run_completed), as well as: sub_analysis_iloc: int - Sub-analysis index

independent_vars property

BC alias — Phase 2 retains this name for downstream callers that haven't migrated.

Returns the union of analysis_independent_vars and system_independent_vars (latter prefixed with system. to disambiguate). Downstream callers should migrate to the explicit analysis_independent_vars and system_independent_vars properties; this alias may be deprecated in a future release.

Contract for prefixed-name entries: every entry of the returned list is an opaque label suitable for Snakemake wildcards (charset ^[A-Za-z0-9_.]+$). Consumers MUST NOT deconstruct entries by .-split or by Pydantic-field lookup against a single model — entries may name either an analysis_config field (bare) OR a system.{field} overlay column. Verified consumers (analysis.py, workflow.py, report_templates/workflow_description.rst.j2, config/report.py, bundle/snakefile_generator.py) treat entries as opaque labels and tolerate the prefixed form without modification.

is_timeout_only_failure property

True iff all incomplete simulations across sub-analyses have timeout-classified failures.

Returns False if there are no incomplete sims (all done), or if any incomplete sim has an unclassified or no_log failure.

system_independent_vars property

Phase 2 — system-config attributes varied across sub-analyses.

Recognizes only system.{field} columns (no bare names — Phase 1 R1 rejects bare-name system_config columns at the allowlist gate).

build_sensitivity_datatree()

Assemble the master sensitivity DataTree lazily from sub-analysis trees.

Each sub-analysis's consolidated DataTree (analysis_datatree.zarr) is opened lazily and grafted under a sa_{sa_id}/ subtree. Sensitivity parameters for each sub-analysis are attached as .attrs on the sa_{sa_id} node. A parameter-summary Dataset is written at the root under parameters for tabular queries.

Source code in src/hhemt/sensitivity_analysis.py

def build_sensitivity_datatree(self) -> "xr.DataTree":
    """Assemble the master sensitivity DataTree lazily from sub-analysis trees.

    Each sub-analysis's consolidated DataTree (``analysis_datatree.zarr``) is
    opened lazily and grafted under a ``sa_{sa_id}/`` subtree. Sensitivity
    parameters for each sub-analysis are attached as ``.attrs`` on the
    ``sa_{sa_id}`` node. A parameter-summary Dataset is written at the root
    under ``parameters`` for tabular queries.
    """
    tree_dict: dict[str, xr.Dataset] = {}

    tree_dict["/"] = xr.Dataset(
        attrs={
            "Conventions": "CF-1.13",
            "title": "TRITON-SWMM sensitivity analysis results",
            "analysis_id": str(self.master_analysis.cfg_analysis.analysis_id),
            "output_creation_date": current_datetime_string(),
        }
    )

    tree_dict["parameters"] = xr.Dataset.from_dataframe(self.df_setup)

    for sa_id, sub_analysis in self.sub_analyses.items():
        node_name = f"{self.sub_analyses_prefix}{sa_id}"
        # Refresh the sub-analysis's in-memory log from disk before reading its
        # consolidation state: open_datatree() gates on the in-memory
        # datatree_consolidation_complete flag, which a run (often in another
        # process) sets on disk but not in this long-lived sub object.
        # Previously the sensitivity aggregators' _update_log() refreshed these
        # sub logs as a side effect; that call was dropped in the
        # log-write-race-fix compute-on-read change, so refresh explicitly here
        # at the cross-analysis read site (read-only observer; safe).
        sub_analysis._refresh_log()
        try:
            sub_tree = sub_analysis.process.open_datatree()
        except ValueError:
            continue

        for path, node in sub_tree.subtree_with_keys:
            if not node.has_data:
                continue
            rel = path.lstrip("/")
            if not rel:
                continue
            tree_dict[f"{node_name}/{rel}"] = node.dataset

        setup_row = self.df_setup.loc[sa_id]
        attrs = {k: _to_native_attr(v) for k, v in setup_row.to_dict().items()}
        attrs["sa_id"] = str(sa_id)
        tree_dict[node_name] = xr.Dataset(attrs=attrs)

    tree = xr.DataTree.from_dict(tree_dict)
    apply_global_attributes(tree, analysis_id=str(self.master_analysis.cfg_analysis.analysis_id))

    # ADR-15 Phase 1: the per-event hhemt_producing_sha/version coordinates ride
    # up automatically via the verbatim node.dataset graft above (no new
    # transmission code). Re-derive the MASTER-level scalar fast-path here by
    # scanning the grafted sub-nodes for master-wide uniformity (uniform across
    # ALL sub-analyses' events -> scalar; else absent + divergent breadcrumb).
    from hhemt.cf_conventions import apply_producing_stamp

    _sha_vals: list[str] = []
    _semver_vals: list[str] = []
    for _key, _ds in tree_dict.items():
        _coords = getattr(_ds, "coords", {})
        if "hhemt_producing_sha" in _coords:
            _sha_vals.extend(str(v) for v in _ds["hhemt_producing_sha"].values.tolist())
        if "hhemt_producing_version" in _coords:
            _semver_vals.extend(str(v) for v in _ds["hhemt_producing_version"].values.tolist())
    apply_producing_stamp(tree, _sha_vals, _semver_vals)
    return tree

bundle_report_data(output_path=None)

Emit a portable render bundle for the sensitivity master analysis.

Opt-in only — NEVER invoked from analysis.run() or submit_workflow(). The bundle includes the sensitivity master's consolidated outputs plus the union of source paths declared by every renderer in the master's render_report(), including per-sim renderers wildcarded over (sa_id, event_id).

Args: output_path: Optional target path for the bundle tar.

Returns: Path to the emitted bundle tar.

Source code in src/hhemt/sensitivity_analysis.py

def bundle_report_data(
    self,
    output_path: "Path | None" = None,
) -> "Path":
    """Emit a portable render bundle for the sensitivity master analysis.

    Opt-in only — NEVER invoked from analysis.run() or
    submit_workflow(). The bundle includes the sensitivity master's
    consolidated outputs plus the union of source paths declared by
    every renderer in the master's render_report(), including per-sim
    renderers wildcarded over (sa_id, event_id).

    Args:
        output_path: Optional target path for the bundle tar.

    Returns:
        Path to the emitted bundle tar.
    """
    from hhemt.bundle import emit_bundle

    return emit_bundle(self, output_path)

classify_incomplete_sim_failures()

Scan model logs for all incomplete simulations across sub-analyses and classify each failure.

Aggregates _classify_model_log_failure() across all sub-analyses. Works for both "1_job_many_srun_tasks" and "batch_job" execution methods — the SLURM cancellation marker appears in the model log in both cases.

Returns:

Type	Description
dict[str, str]	Maps scenario identifier (e.g. "sa1_0") to failure class: "timeout" — log contains DUE TO TIME LIMIT "unclassified" — log exists but no known failure marker found "no_log" — model log file does not exist

Source code in src/hhemt/sensitivity_analysis.py

def classify_incomplete_sim_failures(self) -> dict[str, str]:
    """Scan model logs for all incomplete simulations across sub-analyses and classify each failure.

    Aggregates ``_classify_model_log_failure()`` across all sub-analyses.
    Works for both ``"1_job_many_srun_tasks"`` and ``"batch_job"`` execution
    methods — the SLURM cancellation marker appears in the model log in both cases.

    Returns
    -------
    dict[str, str]
        Maps scenario identifier (e.g. ``"sa1_0"``) to failure class:

        - ``"timeout"`` — log contains ``DUE TO TIME LIMIT``
        - ``"unclassified"`` — log exists but no known failure marker found
        - ``"no_log"`` — model log file does not exist
    """
    results: dict[str, str] = {}
    for sa_id, sub_analysis in self.sub_analyses.items():
        for event_iloc in sub_analysis.df_sims.index:
            scen = TRITONSWMM_scenario(event_iloc, sub_analysis)
            enabled_models = scen.run.model_types_enabled
            for model_type in enabled_models:
                if not scen.model_run_completed(model_type):
                    event_id = scen.event_id
                    key = f"sa-{sa_id}_evt-{event_id}"
                    results[key] = scen.run._classify_model_log_failure(model_type)
    return results

cleanup_all_orphans(dry_run=True, force=False, verbose=True)

Detect and (optionally) delete orphan subanalysis dirs, flags, datatree groups, and input fingerprints.

When any orphan is detected and deletion proceeds, the entire sensitivity_datatree.zarr is removed (rebuild approach — see plan D-SURGICAL) and the master-consolidation status flag is also removed so Snakemake re-runs the master_consolidation rule on the next workflow run.

Parameters:

Name	Type	Description	Default
dry_run	bool	If True (default), only reports without deleting.	True
force	bool	Required when dry_run=False.	False
verbose	bool	If True, prints each deletion via print(..., flush=True).	True

Returns:

Type	Description
dict[str, list | bool]	Keys: "dirs" (list[Path]), "status_flags" (list[Path]), "datatree_groups" (list[str]), "input_fingerprints" (list[Path]), and (after deletion only) "sensitivity_datatree_removed" (bool) and "master_flag_removed" (bool) reporting whether the rebuild-trigger artifacts were actually removed.

Raises:

Type	Description
ValueError	If dry_run=False and force=False.

Source code in src/hhemt/sensitivity_analysis.py

def cleanup_all_orphans(
    self,
    dry_run: bool = True,
    force: bool = False,
    verbose: bool = True,
) -> dict[str, list]:
    """Detect and (optionally) delete orphan subanalysis dirs, flags, datatree groups, and input fingerprints.

    When any orphan is detected and deletion proceeds, the entire
    ``sensitivity_datatree.zarr`` is removed (rebuild approach — see plan
    D-SURGICAL) and the master-consolidation status flag is also removed so
    Snakemake re-runs the master_consolidation rule on the next workflow run.

    Parameters
    ----------
    dry_run : bool
        If True (default), only reports without deleting.
    force : bool
        Required when ``dry_run=False``.
    verbose : bool
        If True, prints each deletion via ``print(..., flush=True)``.

    Returns
    -------
    dict[str, list | bool]
        Keys: ``"dirs"`` (list[Path]), ``"status_flags"`` (list[Path]),
        ``"datatree_groups"`` (list[str]), ``"input_fingerprints"`` (list[Path]),
        and (after deletion only)
        ``"sensitivity_datatree_removed"`` (bool) and
        ``"master_flag_removed"`` (bool) reporting whether the
        rebuild-trigger artifacts were actually removed.

    Raises
    ------
    ValueError
        If ``dry_run=False`` and ``force=False``.
    """
    from hhemt.utils import fast_rmtree

    result = {
        "dirs": self.find_orphan_subanalysis_dirs(),
        "status_flags": self.find_orphan_status_flags(),
        "datatree_groups": self.find_orphan_datatree_groups(),
        "input_fingerprints": self.find_orphan_input_fingerprints(),
    }
    any_orphan = bool(
        result["dirs"] or result["status_flags"] or result["datatree_groups"] or result["input_fingerprints"]
    )
    if verbose:
        if any_orphan:
            print(
                f"[cleanup-orphans] dirs={len(result['dirs'])} "
                f"status_flags={len(result['status_flags'])} "
                f"datatree_groups={len(result['datatree_groups'])}",
                flush=True,
            )
            for p in result["dirs"]:
                print(f"  dir: {p}", flush=True)
            for p in result["status_flags"]:
                print(f"  flag: {p}", flush=True)
            for sa_id in result["datatree_groups"]:
                print(f"  datatree-group: sa_{sa_id}", flush=True)
            for p in result["input_fingerprints"]:
                print(f"  input-fingerprint: {p}", flush=True)
        else:
            print("[cleanup-orphans] No orphans detected.", flush=True)
    if dry_run:
        return result
    if not force:
        raise ValueError(
            "cleanup_all_orphans called with dry_run=False but force=False. Pass force=True to perform deletion."
        )
    master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
    for p in result["dirs"]:
        if verbose:
            print(f"[cleanup-orphans] Deleting dir {p}", flush=True)
        fast_rmtree(p, analysis_dir=master_analysis_dir)  # PATTERN A
    for p in result["status_flags"]:
        if verbose:
            print(f"[cleanup-orphans] Unlinking flag {p}", flush=True)
        # EXEMPT-DU: status-flag
        p.unlink()
    for p in result["input_fingerprints"]:
        if verbose:
            print(f"[cleanup-orphans] Unlinking input-fingerprint {p}", flush=True)
        # EXEMPT-DU: status-dir-cleanup
        p.unlink()
    result["sensitivity_datatree_removed"] = False
    result["master_flag_removed"] = False
    if any_orphan:
        zarr_path = self.analysis_paths.sensitivity_datatree_zarr
        if zarr_path is not None and zarr_path.exists():
            if verbose:
                print(
                    f"[cleanup-orphans] Deleting sensitivity_datatree.zarr (rebuild on next run): {zarr_path}",
                    flush=True,
                )
            fast_rmtree(zarr_path, analysis_dir=master_analysis_dir)  # PATTERN A
            result["sensitivity_datatree_removed"] = True
        master_flag = self.analysis_paths.analysis_dir / "_status" / "f_consolidate_master_complete.flag"
        if master_flag.exists():
            if verbose:
                print(
                    f"[cleanup-orphans] Unlinking master-consolidation flag {master_flag}",
                    flush=True,
                )
            # EXEMPT-DU: status-flag
            master_flag.unlink()
            result["master_flag_removed"] = True
    return result

cleanup_orphan_subanalysis_dirs(dry_run=True, force=False, verbose=True)

Identify and optionally delete orphaned sub-analysis directories.

Uses :meth:find_orphan_subanalysis_dirs to locate directories under subanalyses/ whose sa_id no longer appears in the current CSV.

Parameters:

Name	Type	Description	Default
dry_run	bool	If True (default), only reports orphans without deleting.	True
force	bool	Required when dry_run=False. Without it, the method raises ValueError to guard against accidental deletion of expensive HPC outputs.	False
verbose	bool	If True, prints each orphan path via print(..., flush=True).	True

Returns:

Type	Description
list[Path]	The orphan directories (either deleted or proposed for deletion).

Raises:

Type	Description
ValueError	If dry_run=False and force=False.

Source code in src/hhemt/sensitivity_analysis.py

def cleanup_orphan_subanalysis_dirs(
    self,
    dry_run: bool = True,
    force: bool = False,
    verbose: bool = True,
) -> list[Path]:
    """Identify and optionally delete orphaned sub-analysis directories.

    Uses :meth:`find_orphan_subanalysis_dirs` to locate directories under
    ``subanalyses/`` whose ``sa_id`` no longer appears in the current CSV.

    Parameters
    ----------
    dry_run : bool
        If True (default), only reports orphans without deleting.
    force : bool
        Required when ``dry_run=False``. Without it, the method raises
        ``ValueError`` to guard against accidental deletion of expensive
        HPC outputs.
    verbose : bool
        If True, prints each orphan path via ``print(..., flush=True)``.

    Returns
    -------
    list[Path]
        The orphan directories (either deleted or proposed for deletion).

    Raises
    ------
    ValueError
        If ``dry_run=False`` and ``force=False``.
    """
    from hhemt.utils import fast_rmtree

    orphans = self.find_orphan_subanalysis_dirs()
    if verbose:
        if orphans:
            print(f"[cleanup-orphans] Found {len(orphans)} orphan sub-analysis directories:", flush=True)
            for p in orphans:
                print(f"  {p}", flush=True)
        else:
            print("[cleanup-orphans] No orphan sub-analysis directories found.", flush=True)
    if dry_run:
        return orphans
    if not force:
        raise ValueError(
            "cleanup_orphan_subanalysis_dirs called with dry_run=False but "
            "force=False. Pass force=True to perform deletion."
        )
    master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
    deleted: list[Path] = []
    failed: list[tuple[Path, Exception]] = []
    for p in orphans:
        if verbose:
            print(f"[cleanup-orphans] Deleting {p}", flush=True)
        try:
            fast_rmtree(p, analysis_dir=master_analysis_dir)  # PATTERN A
            deleted.append(p)
        except Exception as exc:
            failed.append((p, exc))
            if verbose:
                print(f"[cleanup-orphans] FAILED to delete {p}: {exc}", flush=True)
    if failed:
        summary = "; ".join(f"{p}: {exc}" for p, exc in failed)
        raise RuntimeError(
            f"cleanup_orphan_subanalysis_dirs deleted {len(deleted)} of {len(orphans)} orphans; failures: {summary}"
        )
    return deleted

compile_and_preprocess_all_targets(overwrite_system_inputs=False, recompile_if_already_done_successfully=False, verbose=True)

Process system-level inputs and compile TRITON-SWMM for each unique target.

Iterates self.unique_system_targets (populated in __init__) and runs process_system_level_inputs() + compile_TRITON_SWMM() once per target. In the no-per-sub-analysis-config case the list contains a single target wrapping the master system, so this method is the unified entry point for non-Snakemake direct execution regardless of whether per-sa configs are used.

Source code in src/hhemt/sensitivity_analysis.py

def compile_and_preprocess_all_targets(
    self,
    overwrite_system_inputs: bool = False,
    recompile_if_already_done_successfully: bool = False,
    verbose: bool = True,
):
    """Process system-level inputs and compile TRITON-SWMM for each unique target.

    Iterates ``self.unique_system_targets`` (populated in ``__init__``) and runs
    ``process_system_level_inputs()`` + ``compile_TRITON_SWMM()`` once per target.
    In the no-per-sub-analysis-config case the list contains a single target
    wrapping the master system, so this method is the unified entry point for
    non-Snakemake direct execution regardless of whether per-sa configs are used.
    """
    for target in self.unique_system_targets:
        if verbose:
            print(
                f"[Setup] Processing target {target.target_id} ({len(target.sub_analysis_ids)} sub-analyses)",
                flush=True,
            )
        target.system.process_system_level_inputs(
            overwrite_outputs_if_already_created=overwrite_system_inputs,
            verbose=verbose,
        )
        target.system.compile_TRITON_SWMM(
            recompile_if_already_done_successfully=recompile_if_already_done_successfully,
            verbose=verbose,
        )
    self._update_master_analysis_log()

consolidate_sensitivity_datatree(compression_level=5, verbose=False, allow_incomplete=True)

Build and write the master sensitivity DataTree zarr.

Ensures each sub-analysis has its own consolidated analysis_datatree.zarr first, then assembles them into a single hierarchical store at sensitivity_datatree.zarr.

When allow_incomplete is True (the default), sub-analyses whose per-scenario summaries are not all present on disk are SKIPPED (with a logged warning naming the sa_id) instead of crashing the master assembly with FileNotFoundError/ValueError. The skip is whole-sub-analysis because _retrieve_combined_output concatenates per-scenario summaries along event_iloc and is all-or-nothing per sub-analysis. Pass allow_incomplete=False to restore fail-fast.

NOTE (Decision D2, 2026-06-02, "for now"): the default is True so reprocess AND canonical runs tolerate partial-completion sensitivity suites by default. To restore strict fail-fast on canonical runs, flip this default back to False.

Source code in src/hhemt/sensitivity_analysis.py

def consolidate_sensitivity_datatree(
    self,
    compression_level: int = 5,
    verbose: bool = False,
    allow_incomplete: bool = True,
) -> Path:
    """Build and write the master sensitivity DataTree zarr.

    Ensures each sub-analysis has its own consolidated ``analysis_datatree.zarr``
    first, then assembles them into a single hierarchical store at
    ``sensitivity_datatree.zarr``.

    When ``allow_incomplete`` is True (the default), sub-analyses whose
    per-scenario summaries are not all present on disk are SKIPPED (with a
    logged warning naming the ``sa_id``) instead of crashing the master
    assembly with ``FileNotFoundError``/``ValueError``. The skip is
    whole-sub-analysis because ``_retrieve_combined_output`` concatenates
    per-scenario summaries along ``event_iloc`` and is all-or-nothing per
    sub-analysis. Pass ``allow_incomplete=False`` to restore fail-fast.

    NOTE (Decision D2, 2026-06-02, "for now"): the default is ``True`` so
    reprocess AND canonical runs tolerate partial-completion sensitivity
    suites by default. To restore strict fail-fast on canonical runs, flip
    this default back to ``False``.
    """
    fname_out = self.analysis_paths.sensitivity_datatree_zarr
    if fname_out is None:
        raise ValueError("sensitivity_datatree_zarr path is not configured on AnalysisPaths.")

    # Per D5/R8: bare .exists() is an unreliable completion signal — a
    # present-but-corrupt sensitivity_datatree.zarr (a write that crashed
    # mid-stream) .exists() as True, and would be returned as a healthy
    # result. Align with the master log's canonical signal: "already
    # consolidated" iff it exists AND sensitivity_datatree_consolidation_complete
    # is True (set only on a successful full write below). Present-but-incomplete
    # falls through to a clean rebuild. This is the master-sensitivity analogue
    # of consolidate_to_datatree's D5 log-keyed early-return; do NOT touch the
    # read-path .exists() guard in open_sensitivity_datatree (a read-path
    # existence check is correct).
    self.master_analysis._refresh_log()
    _log_complete = (
        hasattr(self.master_analysis.log, "sensitivity_datatree_consolidation_complete")
        and self.master_analysis.log.sensitivity_datatree_consolidation_complete.get() is True
    )
    if fname_out.exists() and _log_complete:
        if verbose:
            print(f"Sensitivity DataTree zarr already present at {fname_out} and log complete. Not overwriting.")
        # Ensure the master analysis-scope DU sentinel exists even on the
        # already-consolidated early-return path. This materializes the
        # sentinel on trees consolidated before this write site existed, and
        # is cheap/idempotent via compare-and-write.
        self._write_master_du_sentinel()
        return fname_out
    if fname_out.exists() and not _log_complete:
        from hhemt.utils import fast_rmtree

        fast_rmtree(fname_out, analysis_dir=self.analysis_paths.analysis_dir)
        if verbose:
            print(
                f"Sensitivity DataTree zarr present at {fname_out} but log incomplete — "
                "rebuilding (treating as corrupt)."
            )

    # Ensure each sub-analysis has its analysis_datatree.zarr built.
    for sa_id, sub_analysis in self.sub_analyses.items():
        sub_path = sub_analysis.analysis_paths.analysis_datatree_zarr
        if sub_path is None:
            continue
        # Gate on the POSITIVE completion marker, not bare .exists(): a sub
        # whose zarr is on disk but whose datatree_consolidation_complete flag
        # is null/False (set by a consolidate job in another process but not
        # reflected in this long-lived sub's in-memory log) must be
        # (re)consolidated — otherwise build_sensitivity_datatree's
        # open_datatree() raises ValueError and silently drops the sub.
        # consolidate_to_datatree is idempotent on an already-complete sub
        # (its own _log_complete early-return), so this self-heals.
        sub_analysis._refresh_log()
        sub_complete = (
            hasattr(sub_analysis.log, "datatree_consolidation_complete")
            and sub_analysis.log.datatree_consolidation_complete.get() is True
        )
        if not (sub_path.exists() and sub_complete):
            try:
                sub_analysis.process.consolidate_to_datatree(
                    compression_level=compression_level,
                    verbose=verbose,
                )
            except (FileNotFoundError, ValueError) as exc:
                if not allow_incomplete:
                    raise
                print(
                    f"[sensitivity-consolidate] Skipping incomplete sub-analysis "
                    f"{self.sub_analyses_prefix}{sa_id} under allow_incomplete=True: {exc}",
                    flush=True,
                )
                continue

    tree = self.build_sensitivity_datatree()
    from hhemt.cf_conventions import apply_provenance_core
    from hhemt.metadata import write_rocrate_sidecar
    from hhemt.provenance import emit_provenance

    _sub_relpaths = [f"subanalyses/sa_{sa_id}/analysis_datatree.zarr" for sa_id in self.sub_analyses]
    _core_json, _graph_json = emit_provenance(
        self.master_analysis,
        consolidated_zarr_relpath="sensitivity_datatree.zarr",
        sub_dataset_relpaths=_sub_relpaths,
        with_run_units=False,
    )
    apply_provenance_core(tree, core_json_str=_core_json)
    write_datatree_zarr(tree, fname_out, compression_level=compression_level)
    write_rocrate_sidecar(self.master_analysis.analysis_paths.analysis_dir, graph_json=_graph_json)

    self.master_analysis._refresh_log()
    if hasattr(self.master_analysis.log, "sensitivity_datatree_consolidation_complete"):
        self.master_analysis.log.sensitivity_datatree_consolidation_complete.set(True)

    if verbose:
        print(f"Wrote sensitivity DataTree zarr to {fname_out}")
    self._write_master_du_sentinel()
    return fname_out

consolidate_subanalysis_outputs(which='both', *, verbose=False, compression_level=5)

Consolidate sub-analyses into a hierarchical sensitivity DataTree zarr.

Replaces the previous per-mode flat xr.concat path. Each sub-analysis first builds its per-analysis DataTree (analysis_datatree.zarr); then the master assembles all sub-analyses into a single sensitivity_datatree.zarr at the master analysis dir.

Source code in src/hhemt/sensitivity_analysis.py

def consolidate_subanalysis_outputs(
    self,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    *,
    verbose: bool = False,
    compression_level: int = 5,
):
    """Consolidate sub-analyses into a hierarchical sensitivity DataTree zarr.

    Replaces the previous per-mode flat ``xr.concat`` path. Each sub-analysis
    first builds its per-analysis DataTree (``analysis_datatree.zarr``); then
    the master assembles all sub-analyses into a single
    ``sensitivity_datatree.zarr`` at the master analysis dir.
    """
    self.consolidate_sensitivity_datatree(
        compression_level=compression_level,
        verbose=verbose,
    )
    return

delete(override_in_flight=False, *, override_multi_sim_run_method=None)

Distributed delete workflow for the sensitivity master analysis.

Refuses by default when _status/_submitted/*.json sentinels indicate live SLURM jobs. Pass override_in_flight=True to bypass the guard.

Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction + D-DeleteBoundary resolutions) and distributed-delete-and-du- recording Phase 3 (SLURM lift; override_multi_sim_run_method mirrors the run-mode override pattern).

Source code in src/hhemt/sensitivity_analysis.py

def delete(
    self,
    override_in_flight: bool = False,
    *,
    override_multi_sim_run_method: Literal["local", "batch_job", "1_job_many_srun_tasks"] | None = None,
) -> None:
    """Distributed delete workflow for the sensitivity master analysis.

    Refuses by default when ``_status/_submitted/*.json`` sentinels
    indicate live SLURM jobs. Pass ``override_in_flight=True`` to bypass
    the guard.

    Per cleanup-rerun-delete-redesign Phase 2 (D-DeleteSentinelInteraction
    + D-DeleteBoundary resolutions) and distributed-delete-and-du-
    recording Phase 3 (SLURM lift; ``override_multi_sim_run_method``
    mirrors the run-mode override pattern).
    """
    from hhemt.utils import fast_rmtree

    analysis_dir = self.master_analysis.analysis_paths.analysis_dir

    # 1. Clear any stale sentinels from a prior failed delete attempt.
    stale_dir = analysis_dir / "_status" / "_deleting"
    if stale_dir.exists():
        # EXEMPT-DU: status-dir-cleanup
        fast_rmtree(stale_dir)

    # 2. Submit the distributed sensitivity-delete workflow. Guards run
    # inside the builder; orchestrator does not invoke _pre_delete_guards
    # directly.
    self._workflow_builder.submit_delete_workflow_sensitivity(
        override_in_flight=override_in_flight,
        override_multi_sim_run_method=override_multi_sim_run_method,
    )

    # 3. Verify all expected sentinels present; remove analysis_dir atomically.
    expected = self._enumerate_expected_delete_sentinels()
    deleting_dir = analysis_dir / "_status" / "_deleting"
    actual = set(deleting_dir.glob("*.flag")) if deleting_dir.exists() else set()
    missing = expected - actual
    if missing:
        print(
            f"[delete] {len(missing)} per-sa-rule sentinels missing — preserving analysis_dir for debugging.",
            flush=True,
        )
        print(f"[delete] missing: {sorted(p.name for p in missing)}", flush=True)
        return
    print(
        f"[delete] all {len(expected)} per-sa-rule sentinels present — removing analysis_dir.",
        flush=True,
    )
    # EXEMPT-DU: full-analysis-root-wipe
    fast_rmtree(analysis_dir)

export_sensitivity_definition_csv()

Export sensitivity analysis definition to analysis directory as CSV.

Exports only the fields that vary across sub-analyses (self.df_setup columns) to a standardized 'sensitivity_analysis_definition.csv' file in the analysis directory. This allows easier inspection of the sensitivity analysis configuration during debugging.

Returns: Path to the exported CSV file.

Source code in src/hhemt/sensitivity_analysis.py

def export_sensitivity_definition_csv(self) -> Path:
    """Export sensitivity analysis definition to analysis directory as CSV.

    Exports only the fields that vary across sub-analyses (self.df_setup columns)
    to a standardized 'sensitivity_analysis_definition.csv' file in the analysis directory.
    This allows easier inspection of the sensitivity analysis configuration during debugging.

    Returns:
        Path to the exported CSV file.
    """
    output_path = self.analysis_paths.analysis_dir / "sensitivity_analysis_definition.csv"
    df_export = self.df_setup.copy()
    df_export.to_csv(output_path, index=True)
    return output_path

find_orphan_datatree_groups()

Return sa_id strings present as subgroups in sensitivity_datatree.zarr but absent from df_setup.index.

Inspects on-disk subdirectories of sensitivity_datatree.zarr/ matching {prefix}{sa_id} where prefix is self.sub_analyses_prefix. Returns the sa_id strings (without prefix). Returns an empty list if the zarr does not exist.

Source code in src/hhemt/sensitivity_analysis.py

def find_orphan_datatree_groups(self) -> list[str]:
    """Return sa_id strings present as subgroups in sensitivity_datatree.zarr but absent from df_setup.index.

    Inspects on-disk subdirectories of ``sensitivity_datatree.zarr/`` matching
    ``{prefix}{sa_id}`` where ``prefix`` is ``self.sub_analyses_prefix``. Returns
    the sa_id strings (without prefix). Returns an empty list if the zarr
    does not exist.
    """
    zarr_path = self.analysis_paths.sensitivity_datatree_zarr
    if zarr_path is None or not zarr_path.exists():
        return []
    expected_sa_ids = set(self.df_setup.index.astype(str))
    prefix = self.sub_analyses_prefix
    orphans: list[str] = []
    for entry in zarr_path.iterdir():
        if not entry.is_dir():
            continue
        if not entry.name.startswith(prefix):
            continue
        sa_id = entry.name[len(prefix) :]
        if sa_id and sa_id not in expected_sa_ids:
            orphans.append(sa_id)
    return sorted(orphans)

find_orphan_input_fingerprints()

Return _status/sa-{sa_id}_inputs.json fingerprint files whose sa_id is absent from df_setup.index.

The per-sa_id input-fingerprint files (Gotcha 17) are written by SensitivityAnalysisWorkflowBuilder at Snakefile-build time and named sa-{sa_id}_inputs.json under _status/. When an sa_id is removed from the sensitivity CSV its fingerprint is orphaned just like its dirs/flags/datatree groups. Returns an empty list if _status/ is absent.

Source code in src/hhemt/sensitivity_analysis.py

def find_orphan_input_fingerprints(self) -> list[Path]:
    """Return _status/sa-{sa_id}_inputs.json fingerprint files whose sa_id is absent from df_setup.index.

    The per-sa_id input-fingerprint files (Gotcha 17) are written by
    ``SensitivityAnalysisWorkflowBuilder`` at Snakefile-build time and named
    ``sa-{sa_id}_inputs.json`` under ``_status/``. When an sa_id is removed
    from the sensitivity CSV its fingerprint is orphaned just like its
    dirs/flags/datatree groups. Returns an empty list if ``_status/`` is absent.
    """
    import re as _re

    status_dir = self.analysis_paths.analysis_dir / "_status"
    if not status_dir.exists():
        return []
    expected_sa_ids = set(self.df_setup.index.astype(str))
    pat = _re.compile(r"^sa-([A-Za-z0-9_.]+?)_inputs\.json$")
    orphans: list[Path] = []
    for entry in status_dir.glob("sa-*_inputs.json"):
        m = pat.match(entry.name)
        if m is None:
            continue
        sa_id = m.group(1)
        if sa_id not in expected_sa_ids:
            orphans.append(entry)
    return sorted(orphans)

find_orphan_status_flags()

Return _status/ flag files whose embedded sa_id is absent from df_setup.index.

Matches against the four Snakemake rule-output flag families that embed an sa_id (verified against workflow.py rule generation):

• b_prepare_sa-{sa_id}_evt-{event_id}_complete.flag
• c_run_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag
• d_process_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag
• e_consolidate_sa-{sa_id}_complete.flag

The sa_id charset is constrained to ^[A-Za-z0-9_.]+$ per the project stipulation. Returns an empty list if the _status/ directory does not exist.

Source code in src/hhemt/sensitivity_analysis.py

def find_orphan_status_flags(self) -> list[Path]:
    """Return _status/ flag files whose embedded sa_id is absent from df_setup.index.

    Matches against the four Snakemake rule-output flag families that embed
    an sa_id (verified against workflow.py rule generation):

    - ``b_prepare_sa-{sa_id}_evt-{event_id}_complete.flag``
    - ``c_run_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag``
    - ``d_process_{model_type}_sa-{sa_id}_evt-{event_id}_complete.flag``
    - ``e_consolidate_sa-{sa_id}_complete.flag``

    The sa_id charset is constrained to ``^[A-Za-z0-9_.]+$`` per the
    project stipulation. Returns an empty list if the ``_status/``
    directory does not exist.
    """
    import re as _re

    status_dir = self.analysis_paths.analysis_dir / "_status"
    if not status_dir.exists():
        return []
    expected_sa_ids = set(self.df_setup.index.astype(str))
    # Anchored to the four known rule-name prefixes so unrelated 'sa-'
    # substrings (or future non-sensitivity rules that happen to contain
    # 'sa-') cannot trigger a false orphan.
    pat = _re.compile(
        r"^(?:b_prepare|c_run_[A-Za-z0-9]+|d_process_[A-Za-z0-9]+|e_consolidate)_sa-([A-Za-z0-9_.]+?)(?:_evt-[A-Za-z0-9_.]+|_complete|)\.flag$"
    )
    orphans: list[Path] = []
    for entry in status_dir.glob("*.flag"):
        m = pat.match(entry.name)
        if m is None:
            continue
        sa_id = m.group(1)
        if sa_id not in expected_sa_ids:
            orphans.append(entry)
    return sorted(orphans)

find_orphan_subanalysis_dirs()

Return sub-analysis directories on disk whose sa_id is absent from the current CSV.

The authoritative set of expected sub-analysis directory names is derived from self.df_setup.index (the sensitivity CSV's sa_id column) and the self.sub_analyses_prefix constant. This ties orphan detection to the CSV directly, so a partially-constructed self.sub_analyses dict cannot cause legitimate directories to be misclassified as orphans. On-disk sa_* directories whose suffix fails the Snakemake-wildcard-safe charset ^[A-Za-z0-9_.]+$ are skipped — they were not created by this toolkit and must not be deleted by it. If self.subanalysis_dir does not exist, returns [].

Source code in src/hhemt/sensitivity_analysis.py

def find_orphan_subanalysis_dirs(self) -> list[Path]:
    """Return sub-analysis directories on disk whose sa_id is absent from the current CSV.

    The authoritative set of expected sub-analysis directory names is derived
    from ``self.df_setup.index`` (the sensitivity CSV's ``sa_id`` column) and
    the ``self.sub_analyses_prefix`` constant. This ties orphan detection to
    the CSV directly, so a partially-constructed ``self.sub_analyses`` dict
    cannot cause legitimate directories to be misclassified as orphans.
    On-disk ``sa_*`` directories whose suffix fails the Snakemake-wildcard-safe
    charset ``^[A-Za-z0-9_.]+$`` are skipped — they were not created by this
    toolkit and must not be deleted by it. If ``self.subanalysis_dir`` does
    not exist, returns ``[]``.
    """
    import re as _re

    if not self.subanalysis_dir.exists():
        return []
    expected_names = {f"{self.sub_analyses_prefix}{sa_id}" for sa_id in self.df_setup.index.astype(str)}
    charset = _re.compile(r"^[A-Za-z0-9_.]+$")
    orphans: list[Path] = []
    for entry in self.subanalysis_dir.iterdir():
        if not entry.is_dir():
            continue
        if not entry.name.startswith(self.sub_analyses_prefix):
            continue
        suffix = entry.name[len(self.sub_analyses_prefix) :]
        if not charset.match(suffix):
            continue
        if entry.name not in expected_names:
            orphans.append(entry)
    return sorted(orphans)

open_sensitivity_datatree()

Open the consolidated sensitivity DataTree zarr lazily.

Source code in src/hhemt/sensitivity_analysis.py

def open_sensitivity_datatree(self) -> "xr.DataTree":
    """Open the consolidated sensitivity DataTree zarr lazily."""
    path = self.analysis_paths.sensitivity_datatree_zarr
    if path is None or not path.exists():
        raise ValueError("Sensitivity DataTree zarr not found. Run consolidate_sensitivity_datatree() first.")
    return xr.open_datatree(path, engine="zarr", chunks="auto", consolidated=False)

publish(target, *, override_dataset_license=None, software_doi=None)

Deposit the sensitivity MASTER tree to a DOI-minting repo (C6, ADR-11).

Opt-in only — NEVER invoked from run()/submit_workflow(), mirroring render_report()/bundle_report_data(). Deposits the master sensitivity_datatree.zarr + master-rooted ro-crate sidecar; the license is read from the emitted crate. Returns {"target","data_doi","software_doi","record_url"}.

Source code in src/hhemt/sensitivity_analysis.py

def publish(
    self,
    target: "Literal['hydroshare', 'zenodo']",
    *,
    override_dataset_license: "Literal['CC0-1.0', 'CC-BY-NC-4.0'] | None" = None,
    software_doi: "str | None" = None,
) -> dict:
    """Deposit the sensitivity MASTER tree to a DOI-minting repo (C6, ADR-11).

    Opt-in only — NEVER invoked from run()/submit_workflow(), mirroring
    render_report()/bundle_report_data(). Deposits the master
    sensitivity_datatree.zarr + master-rooted ro-crate sidecar; the license is
    read from the emitted crate. Returns {"target","data_doi","software_doi","record_url"}.
    """
    from hhemt.publishing import publish_analysis

    return publish_analysis(
        self.master_analysis,
        target=target,
        override_dataset_license=override_dataset_license,
        software_doi=software_doi,
        consolidated_zarr_relpath="sensitivity_datatree.zarr",
    )

render_report(format='zip', *, reprocess=False)

Render the master report for the sensitivity analysis.

Idempotent: invokes snakemake --report against the master Snakefile without re-executing any rules. Renders only the master-level report; per-sub-analysis reports are not generated (R13).

Parameters:

Name	Type	Description	Default
format	Literal['html', 'zip']	Output format. "html" produces a single self-contained analysis_report.html with all figures inlined as base64, plus React-bundle post-process surgery (title, navbar, sidebar order, click-to-figure shim). "zip" produces analysis_report.zip containing the unbundled report tree (separate HTML + assets); no post-process surgery is applied (the zip layout differs from the single-file HTML).	"zip"
reprocess	bool	When True, render against Snakefile.reprocess (the filtered reprocess DAG) instead of the production Snakefile, so the snakemake --report step only expects the figures the reprocess DAG built. Keyword-only; set by the reprocess render_report rule shell. Default False keeps the production render path byte-identical.	False

Source code in src/hhemt/sensitivity_analysis.py

def render_report(self, format: Literal["html", "zip"] = "zip", *, reprocess: bool = False) -> "Path":
    """Render the master report for the sensitivity analysis.

    Idempotent: invokes ``snakemake --report`` against the master Snakefile
    without re-executing any rules. Renders only the master-level report;
    per-sub-analysis reports are not generated (R13).

    Parameters
    ----------
    format : Literal["html", "zip"], default "zip"
        Output format. ``"html"`` produces a single self-contained
        ``analysis_report.html`` with all figures inlined as base64, plus
        React-bundle post-process surgery (title, navbar, sidebar order,
        click-to-figure shim). ``"zip"`` produces ``analysis_report.zip``
        containing the unbundled report tree (separate HTML + assets);
        no post-process surgery is applied (the zip layout differs from
        the single-file HTML).
    reprocess : bool, default False
        When ``True``, render against ``Snakefile.reprocess`` (the filtered
        reprocess DAG) instead of the production ``Snakefile``, so the
        ``snakemake --report`` step only expects the figures the reprocess
        DAG built. Keyword-only; set by the reprocess ``render_report`` rule
        shell. Default ``False`` keeps the production render path
        byte-identical.
    """
    import subprocess
    import sys

    from .exceptions import WorkflowError
    from .workflow import _assert_snakefile_package_current

    master_dir = self.master_analysis.analysis_paths.analysis_dir
    snakefile_name = "Snakefile.reprocess" if reprocess else "Snakefile"
    snakefile = master_dir / snakefile_name
    _assert_snakefile_package_current(snakefile)
    out = master_dir / f"analysis_report.{format}"
    css_path = master_dir / "report" / "report.css"
    # Brand-theme resolution (ADR-7 layer 2) — symmetric to
    # analysis.render_report. The render_report_runner path builds a FRESH
    # instance without _brand_theme; resolve once via getattr-fallback to
    # serve BOTH the CSS emit and the navbar surgery (D-6; plan-review SE Flag 1).
    from .config.brand_theme import DEFAULT_BRAND_THEME
    from .config.loaders import load_brand_theme
    from .workflow import _brand_theme_css_map

    _theme = getattr(self, "_brand_theme", None)
    if _theme is None:
        _theme = (
            load_brand_theme(self.cfg_analysis.brand_theme)
            if self.cfg_analysis.brand_theme is not None
            else DEFAULT_BRAND_THEME
        )
    # Re-emit report artifacts from package resources so render_report
    # picks up edits made to the source-tree report_templates/.
    _emit_report_artifacts(master_dir, brand_theme=_brand_theme_css_map(_theme))
    cmd = [
        sys.executable,
        "-m",
        "snakemake",
        "--snakefile",
        str(snakefile),
        "--directory",
        str(master_dir),
        "--report",
        str(out),
        "--report-stylesheet",
        str(css_path),
        "--cores",
        "1",
    ]
    result = subprocess.run(cmd, capture_output=True, text=True)
    if result.returncode != 0:
        tail = "\n".join((result.stdout + "\n" + result.stderr).splitlines()[-50:])
        raise WorkflowError(
            phase="render_report",
            return_code=result.returncode,
            stderr=f"snakemake --report exit {result.returncode}; last 50 lines:\n{tail}",
        )
    # Apply React-bundle post-process surgery (title, navbar, sort order,
    # placeholder category, showCategory auto-pop, row-click delegate).
    # Both formats need the surgery:
    #  - HTML: edit the single rendered file in place.
    #  - Zip: extract, edit `analysis_report/report.html` inside, re-zip.
    # Without surgery in zip mode, the eye-icon-hiding CSS in report.css
    # leaves figure tables with no clickable affordance (the JS click
    # delegate that makes rows clickable lives only in the surgery).
    from .report_renderers._react_surgery import (
        apply_post_process_surgery,
        apply_post_process_surgery_to_zip,
    )

    # Navbar upper-left brand text: brand_theme.upper_left_text (ADR-7),
    # defaulting to analysis_id when None (D-6). _theme is resolved above.
    _navbar = _theme.upper_left_text or self.cfg_analysis.analysis_id
    # Resolve the active set's category_order. render_report() is dominantly
    # invoked from render_report_runner.main() on a FRESH analysis that never
    # called run() (see the _brand_theme getattr-fallback above for the
    # identical hazard), so self._active_reporting_set may not exist. getattr-
    # fallback to a config-only resolution (no CSV cross-validation at render
    # time) mirroring the _theme fallback above. Never let the bare attribute
    # AttributeError be swallowed by the surrounding `except Exception: pass`.
    _active_set = getattr(self, "_active_reporting_set", None)
    if _active_set is None:
        # render-without-run() fallback. Fail SOFT (SE F-I-3): the render path
        # bypasses validate_active_reporting_set, so a stale/unknown
        # reporting_set would raise here and surface as an opaque Snakemake
        # rule failure. Degrade to the historical "default" sidebar order + a
        # one-line warning instead of crashing the render rule.
        import logging

        from .config.report import resolve_active_reporting_set_name
        from .report_renderers._reporting_sets import get_reporting_set

        try:
            _cfg_report = getattr(self, "_cfg_report", None)
            if _cfg_report is None:
                _cfg_report = self.cfg_analysis.report
            _set_name = resolve_active_reporting_set_name(
                _cfg_report,
                is_sensitivity=self.cfg_analysis.toggle_sensitivity_analysis,
            )
            _active_set = get_reporting_set(_set_name)
        except Exception as _e:
            logging.getLogger(__name__).warning(
                "render-path reporting_set resolution failed (%s); falling back to 'default' category order",
                _e,
            )
            _active_set = get_reporting_set("default")
    _category_order = list(_active_set.category_order)
    try:
        if format == "html":
            out.write_text(
                apply_post_process_surgery(
                    out.read_text(),
                    navbar_text=_navbar,
                    category_order=_category_order,
                )
            )
        else:
            apply_post_process_surgery_to_zip(out, navbar_text=_navbar, category_order=_category_order)
    except Exception:
        pass
    if format != "html":
        return out
    out_html = out
    # Snap-confined browsers (Ubuntu Firefox snap) cannot read files under
    # ~/.cache/. If the rendered report lands there, surface a one-line
    # workaround so the user does not hit "Access to the file was denied".
    try:
        if "/.cache/" in str(out_html):
            print(
                f"[render_report] {out_html}\n"
                f"[render_report] Note: snap-confined browsers cannot read ~/.cache; "
                f"copy to ~/Downloads to view: cp {out_html} ~/Downloads/",
                flush=True,
            )
    except Exception:
        pass
    return out_html

reprex_bundle(output_path=None)

Emit a reprex-ready Workflow-Run-Crate bundle for the sensitivity master and return its extracted directory root (ADR-10, D3).

Parity peer of bundle_report_data() — the sensitivity master is the PRIMARY reprex surface (the (sa_id, column) problem-pair emission is intrinsically a sensitivity concept). emit_bundle already carries the reprex runnable-template set + WRC crate (Phase 2); this facade extracts the emitted zip to a sibling directory so the round-trip consumes a directory root directly (Bundle.from_directory(...).reprex(...)). Opt-in only.

Returns: Path to the extracted reprex-bundle directory.

Source code in src/hhemt/sensitivity_analysis.py

def reprex_bundle(self, output_path: "Path | None" = None) -> "Path":
    """Emit a reprex-ready Workflow-Run-Crate bundle for the sensitivity master and
    return its extracted directory root (ADR-10, D3).

    Parity peer of ``bundle_report_data()`` — the sensitivity master is the PRIMARY
    reprex surface (the ``(sa_id, column)`` problem-pair emission is intrinsically a
    sensitivity concept). ``emit_bundle`` already carries the reprex runnable-template
    set + WRC crate (Phase 2); this facade extracts the emitted zip to a sibling
    directory so the round-trip consumes a directory root directly
    (``Bundle.from_directory(...).reprex(...)``). Opt-in only.

    Returns:
        Path to the extracted reprex-bundle directory.
    """
    from hhemt.bundle import emit_bundle
    from hhemt.bundle._reprex import extract_reprex_bundle

    return extract_reprex_bundle(emit_bundle(self, output_path))

reprocess(start_with='consolidate', sa_ids=None, execution_mode='auto', which='both', compression_level=5, verbose=True, dry_run=False, report_formats=None, *, regenerate_existing=False, delete_via_slurm=None, override_force_rerun=None)

Master-level reprocess for sensitivity analyses.

Invalidates per-sub-analysis consolidate flags (subset via sa_ids or all sub-analyses by default) plus the master consolidate flag, then emits a scoped master Snakefile via :meth:SensitivityAnalysisWorkflowBuilder.generate_reprocess_master_snakefile_content and submits it via :meth:SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow.

Unlike :meth:TRITONSWMM_analysis.reprocess, this method does NOT invoke the override_clear_raw orphan/abort gate (R12) — sensitivity master reprocess is a downstream-only refresh of consolidation + plotting + rendering against existing per-sa sim outputs and does not need the in-flight reconciliation logic that the analysis-level override_clear_raw flow uses.

Parameters:

Name	Type	Description	Default
start_with	Literal['process', 'consolidate', 'render']	Stage to re-fire from. "consolidate" (default) deletes per-sa e_consolidate_sa-{id}_complete.flag files and the master f_consolidate_master_complete.flag, then re-runs the consolidate + master_consolidation + plot/render rule chain. "render" invalidates only the report artifacts. "process" reconciles stale d_process flags against summary existence and re-emits the per-(sa, event) rebuild rules (Gotcha 34/40); it does NOT collapse onto the "consolidate" Snakefile.	'consolidate'
sa_ids	list[str] | None	Optional subset of sub-analysis IDs (string-cast) to invalidate. When None (default), every sub-analysis's per-sa consolidate flag is invalidated. IDs not in sub_analyses are silently ignored at the unlink call (missing_ok=True).	None
execution_mode	Literal['auto', 'local', 'slurm']	"auto" detects SLURM context; "local" / "slurm" force the mode.	'auto'
which	Literal['TRITON', 'SWMM', 'both']	"both" / "TRITON" / "SWMM" — threaded into the consolidate rule shells' --which flag.	'both'
compression_level	int	Compression level (0-9) for the consolidate rule shells.	5
verbose	bool	If True, print progress messages.	True
dry_run	bool	If True, runs snakemake --dry-run only.	False

Returns:

Type	Description
dict	Status dictionary from :meth:SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow.

Source code in src/hhemt/sensitivity_analysis.py

def reprocess(
    self,
    start_with: Literal["process", "consolidate", "render"] = "consolidate",
    sa_ids: list[str] | None = None,
    execution_mode: Literal["auto", "local", "slurm"] = "auto",
    which: Literal["TRITON", "SWMM", "both"] = "both",
    compression_level: int = 5,
    verbose: bool = True,
    dry_run: bool = False,
    report_formats: list[str] | None = None,
    *,
    regenerate_existing: bool = False,
    delete_via_slurm: bool | None = None,
    override_force_rerun: ForceRerunValue | None = None,
) -> dict:
    """Master-level reprocess for sensitivity analyses.

    Invalidates per-sub-analysis consolidate flags (subset via ``sa_ids``
    or all sub-analyses by default) plus the master consolidate flag,
    then emits a scoped master Snakefile via
    :meth:`SensitivityAnalysisWorkflowBuilder.generate_reprocess_master_snakefile_content`
    and submits it via
    :meth:`SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow`.

    Unlike :meth:`TRITONSWMM_analysis.reprocess`, this method does NOT
    invoke the ``override_clear_raw`` orphan/abort gate (R12) — sensitivity
    master reprocess is a downstream-only refresh of consolidation +
    plotting + rendering against existing per-sa sim outputs and does
    not need the in-flight reconciliation logic that the analysis-level
    ``override_clear_raw`` flow uses.

    Parameters
    ----------
    start_with
        Stage to re-fire from. ``"consolidate"`` (default) deletes per-sa
        ``e_consolidate_sa-{id}_complete.flag`` files and the master
        ``f_consolidate_master_complete.flag``, then re-runs the consolidate
        + master_consolidation + plot/render rule chain. ``"render"``
        invalidates only the report artifacts. ``"process"`` reconciles
        stale ``d_process`` flags against summary existence and re-emits the
        per-(sa, event) rebuild rules (Gotcha 34/40); it does NOT collapse
        onto the ``"consolidate"`` Snakefile.
    sa_ids
        Optional subset of sub-analysis IDs (string-cast) to invalidate.
        When ``None`` (default), every sub-analysis's per-sa consolidate
        flag is invalidated. IDs not in ``sub_analyses`` are silently
        ignored at the unlink call (``missing_ok=True``).
    execution_mode
        ``"auto"`` detects SLURM context; ``"local"`` / ``"slurm"`` force
        the mode.
    which
        ``"both"`` / ``"TRITON"`` / ``"SWMM"`` — threaded into the
        consolidate rule shells' ``--which`` flag.
    compression_level
        Compression level (0-9) for the consolidate rule shells.
    verbose
        If True, print progress messages.
    dry_run
        If True, runs ``snakemake --dry-run`` only.

    Returns
    -------
    dict
        Status dictionary from
        :meth:`SensitivityAnalysisWorkflowBuilder.submit_reprocess_workflow`.
    """
    # Lazy-stamp _version.json at LAYOUT_VERSION (PI-1 pattern). Idempotent.
    from hhemt.version_migration import LAYOUT_VERSION
    from hhemt.version_migration.state import stamp_new_target

    stamp_new_target(self.master_analysis.analysis_paths.analysis_dir, LAYOUT_VERSION)

    # Force-rerun pre-delete (login-node responsibility). Per
    # cleanup-rerun-delete-redesign Phase 4 + R10. Resolves + validates +
    # deletes matched flags before Snakemake plans the reprocess DAG.
    # Skipped on dry_run — it deletes flags and clears per-scenario
    # processing-log records, both filesystem mutations the dry-run
    # no-destructive-mutation contract forbids.
    if not dry_run:
        self.master_analysis._apply_force_rerun(override_force_rerun)

    # Resolve invalidation target set. ``None`` → all sub-analyses; explicit
    # list → subset. String-cast preserves alignment with sub_analyses dict
    # iteration keys regardless of source type (int / str / numpy scalar).
    if sa_ids is None:
        targets = [str(sa_id) for sa_id in self.sub_analyses.keys()]
    else:
        targets = [str(s) for s in sa_ids]

    # Invalidate per-sa consolidate flags + master flag. start_with controls
    # which flags get unlinked; per-sa flag deletion is the entry point for
    # both "consolidate" and "process" (the master generator does not emit
    # process rules, so process invalidation is treated as consolidate
    # invalidation). "render" leaves consolidate flags intact and only
    # invalidates the rendered report artifact.
    from hhemt.du_sentinels import (
        decrement_scope_sentinel,
        restamp_parent_sentinels,
        sum_child_sentinels,
    )
    from hhemt.utils import fast_rmtree as _fast_rmtree

    master_analysis_dir = self.master_analysis.analysis_paths.analysis_dir
    status_dir = master_analysis_dir / "_status"
    if start_with in ("consolidate", "process"):
        for sa_id in targets:
            # EXEMPT-DU: status-flag
            (status_dir / f"e_consolidate_sa-{sa_id}_complete.flag").unlink(missing_ok=True)
        # EXEMPT-DU: status-flag
        (status_dir / "f_consolidate_master_complete.flag").unlink(missing_ok=True)
        # R7 (D2 Option a) — consolidate-stage divergence preflight. Login-node
        # fail-fast that converts the SILENT-partial-master-tree hazard into a
        # clear ConfigurationError. On start_with="consolidate" the generator's
        # _sub_included_for_reprocess (Gotcha 37) SILENTLY EXCLUDES any sub whose
        # summaries are absent, and master consolidation's allow_incomplete=True
        # default (Gotcha 36) then assembles the master tree over the COMPLETED
        # subset and returns success — so a sub whose sim completed (c_run
        # present) but whose summary was deleted is silently dropped from
        # sensitivity_datatree.zarr with only a buried log warning. This is the
        # symmetric analogue of the process-stage
        # _assert_reprocess_rebuild_sources_present preflight. Fires ONLY on the
        # divergence signature (c_run present AND summary absent) so Gotcha 36's
        # tolerance for genuinely-never-ran subs is preserved. Read-only (no
        # flag/mtime touch — zero rerun-trigger surface).
        if start_with == "consolidate" and not dry_run:
            from hhemt.constants import sim_run_flag_per_sa
            from hhemt.scenario import compute_event_id_slug
            from hhemt.workflow import _scenario_summaries_present

            _enabled = self.master_analysis._get_enabled_model_types()
            _diverged: list[str] = []
            for sa_id in targets:
                sub = self.sub_analyses.get(sa_id)
                if sub is None:
                    continue
                for _evt_iloc in sub.df_sims.index:
                    _evt = compute_event_id_slug(sub._retrieve_weather_indexer_using_integer_index(_evt_iloc))
                    _c_run = master_analysis_dir / sim_run_flag_per_sa(_enabled[0], str(sa_id), _evt)
                    if _c_run.exists() and not _scenario_summaries_present(sub, _evt, _enabled):
                        _diverged.append(f"{self.sub_analyses_prefix}{sa_id}@evt-{_evt}")
            if _diverged:
                from hhemt.exceptions import ConfigurationError

                raise ConfigurationError(
                    field="start_with",
                    message=(
                        "reprocess(start_with='consolidate') cannot consolidate "
                        f"{sorted(_diverged)} — the simulation completed (c_run flag "
                        "present) but the per-scenario summary outputs are absent, so "
                        "the master consolidation would silently drop these "
                        "sub-analyses from sensitivity_datatree.zarr. Re-run with "
                        "start_with='process', regenerate_existing=True to rebuild the "
                        "missing summaries first."
                    ),
                )
        # FIX 2 — divergence self-heal runs on the process path on EVERY
        # route REGARDLESS of regenerate_existing (D2). Each sub-analysis
        # reconciles its own d_process flags + per-model processing_log
        # against on-disk summary presence (D3): where a flag survives but
        # the enabled-model summary set is absent (the May-31 divergence:
        # 72 d_process flags vs 0 summary zarrs), unlink the flag + clear
        # the log so the master generator's emit gate (workflow.py:6810)
        # re-emits the per-(sa,evt) process rule and _already_written
        # (Gotcha 28) lets it write. No-op for any sub whose summaries are
        # all present (healthy). Sub-analyses are full Analysis instances
        # and own their scenarios (Gotcha 11); the helper resolves the
        # per-sa flag-token shape from each sub's is_subanalysis context.
        if start_with == "process" and not dry_run:
            for sa_id in targets:
                sub_analysis = self.sub_analyses.get(sa_id)
                if sub_analysis is None:
                    continue
                _reconciled = sub_analysis._reconcile_stale_process_flags_against_summaries(
                    sa_id=sa_id, master_dir=master_analysis_dir
                )
                sub_analysis._assert_reprocess_rebuild_sources_present(_reconciled)
        # FIX 2b — regenerate_existing rebuild parity (D1 Option A: logic in the
        # extracted free function so the fast unit test exercises it without
        # entering reprocess()'s destructive body). Gated on the
        # regenerate_existing arm; no-op otherwise.
        if start_with == "process" and regenerate_existing and not dry_run:
            _unlink_dprocess_flags_for_regenerate(targets, status_dir)
        # Report+plot deletion ALWAYS runs (toggle-independent) — the report
        # regenerates from the preserved zarr on the default path (FQ1 parity).
        _report_html = master_analysis_dir / "analysis_report.html"
        _report_zip = master_analysis_dir / "analysis_report.zip"
        # EXEMPT-DU: du-handled-by-decrement
        _report_html.unlink(missing_ok=True)
        # EXEMPT-DU: du-handled-by-decrement
        _report_zip.unlink(missing_ok=True)
        # FIX 3 — when regenerate_existing (and not dry_run), a LATER
        # deletion restamps the master _du.json anyway (SLURM route: the
        # reprocess-delete workflow's per-sub + master rules; in-process
        # route: compute_and_write_scope_sentinel(master, scope="analysis")
        # below). The early report-restamp here would otherwise force a
        # full-tree GPFS stat() walk on the login node before any SLURM
        # offload — the observed multi-minute stall. The default
        # (regenerate_existing=False) path still restamps (no later deletion).
        if not dry_run and not regenerate_existing:
            restamp_parent_sentinels(_report_html, analysis_dir=master_analysis_dir)  # PATTERN B (FIX 3 gate)
        # Consolidated-zarr deletion + batched DU restamp are the EXPENSIVE
        # GPFS work — gate behind regenerate_existing. Default path preserves
        # the zarrs (consolidate stays inert) and runs NO restamp walk.
        # R8 routing (D-scope Option C) — computed once from the MASTER
        # multi_sim_run_method. None auto-resolves to slurm-offload on HPC
        # modes (D6 refinement 1). When the SLURM path runs, ONE scoped
        # reprocess-delete workflow fans out per-sub (each sub's processed/
        # across events + its analysis_datatree.zarr) + a master rule for
        # sensitivity_datatree.zarr — replacing BOTH the in-process zarr
        # deletions AND the per-sub processed-output delegation below.
        _hpc = self.master_analysis.cfg_analysis.multi_sim_run_method in (
            "batch_job",
            "1_job_many_srun_tasks",
        )
        _resolved_delete_via_slurm = _hpc if delete_via_slurm is None else delete_via_slurm
        route_delete_via_slurm = regenerate_existing and _resolved_delete_via_slurm and not dry_run and _hpc
        if route_delete_via_slurm:
            self._workflow_builder._base_builder.submit_reprocess_delete_workflow(
                start_with=start_with,
                override_in_flight=False,
            )
        elif regenerate_existing and not dry_run:
            # In-process path (local / delete_via_slurm=False) — per-sub +
            # master zarr deletion, plus the Phase-3 per-sub processed-output
            # delegation (FQ2; sub-analyses own their scenarios). The
            # delegated helper internally guards on
            # `start_with == "process"` (analysis.py
            # _delete_processed_outputs_for_reprocess, ~L2979), so a
            # consolidate/render reprocess PRESERVES each sub's processed/
            # (the rebuild source consolidate reads from) — only a
            # process-stage reprocess deletes it. Do NOT inline the
            # processed/ rmtree here without that guard: dropping it makes a
            # consolidate-stage regenerate delete the rebuild source and the
            # consolidate Snakemake step then fails (FIX-1 Phase-1 regression,
            # 2026-05-31). For the process stage the per-model LOG-clear
            # already ran above (FIX 1, hunk 2a) on both routes; the helper's
            # idempotent re-clear here is harmless.
            affected_sub_dirs: set = set()
            for sa_id in targets:
                sub_analysis = self.sub_analyses.get(sa_id)
                if sub_analysis is None:
                    continue
                # FQ2 processed-output deletion (Phase 3) — delegate to the
                # sub-analysis's own helper (sub-analyses own their scenarios;
                # the helper's start_with guard protects consolidate/render).
                sub_analysis._delete_processed_outputs_for_reprocess(
                    start_with, regenerate_existing=regenerate_existing, dry_run=dry_run
                )
                _sub_zarr = sub_analysis.analysis_paths.analysis_datatree_zarr
                if _sub_zarr is not None and _sub_zarr.exists():
                    _fast_rmtree(_sub_zarr, analysis_dir=None)  # batched-restamp
                    affected_sub_dirs.add(sub_analysis.analysis_paths.analysis_dir)
            _master_zarr = self.analysis_paths.sensitivity_datatree_zarr
            if _master_zarr is not None and _master_zarr.exists():
                _fast_rmtree(_master_zarr, analysis_dir=None)  # batched-restamp
            for _sub_dir in affected_sub_dirs:
                sum_child_sentinels(_sub_dir, scope="sub_analysis", child_scope_dirs=["sims"])
            sum_child_sentinels(master_analysis_dir, scope="analysis", child_scope_dirs=["subanalyses", "sims"])
    elif start_with == "render":
        # No _status flag for render — re-fire by deleting the report
        # artifacts so Snakemake's mtime trigger sees the output as absent.
        # The report-artifact unlink is the flag-equivalent trigger, so it
        # runs even on dry_run (see D6); only the DU restamp is gated.
        _report_html = master_analysis_dir / "analysis_report.html"
        _report_zip = master_analysis_dir / "analysis_report.zip"
        # D3 — capture sizes BEFORE unlink so the O(1) decrement has the bytes.
        _html_bytes = _report_html.stat().st_size if _report_html.exists() else 0
        _zip_bytes = _report_zip.stat().st_size if _report_zip.exists() else 0
        # EXEMPT-DU: du-handled-by-decrement
        _report_html.unlink(missing_ok=True)
        # EXEMPT-DU: du-handled-by-decrement
        _report_zip.unlink(missing_ok=True)
        if not dry_run:
            # D3 — O(1) decrement of the two report children (no plots on the
            # sensitivity render arm). Mirrors the non-sensitivity render path;
            # routes through write_du_sentinel (compare-and-write mtime invariant).
            _child_deltas: dict[str, int] = {}
            if _html_bytes:
                _child_deltas["analysis_report.html"] = _html_bytes
            if _zip_bytes:
                _child_deltas["analysis_report.zip"] = _zip_bytes
            if _child_deltas:
                decrement_scope_sentinel(master_analysis_dir, scope="analysis", child_deltas=_child_deltas)
    else:
        raise ValueError(f"start_with must be one of 'process', 'consolidate', 'render'; got {start_with!r}")

    # Delegate to the sensitivity workflow builder.
    return self._workflow_builder.submit_reprocess_workflow(
        start_with=start_with,
        execution_mode=execution_mode,
        which=which,
        compression_level=compression_level,
        dry_run=dry_run,
        verbose=verbose,
        report_formats=report_formats,
    )

submit_workflow(mode='auto', process_system_level_inputs=True, overwrite_system_inputs=False, compile_TRITON_SWMM=True, recompile_if_already_done_successfully=False, prepare_scenarios=True, overwrite_scenario_if_already_set_up=False, rerun_swmm_hydro_if_outputs_exist=False, process_timeseries=True, which='both', override_clear_raw=None, override_force_rerun=None, compression_level=5, pickup_where_leftoff=True, wait_for_completion=False, dry_run=False, verbose=True, override_hpc_total_nodes=None, override_hpc_restart_times_simulate=None, override_hpc_restart_times_other=None, report_formats=None, extra_sbatch_args=None, snakemake_diagnostics=None)

Submit sensitivity analysis workflow using Snakemake.

This orchestrates multiple sub-analysis workflows and a final master consolidation step that combines all sub-analysis outputs.

Parameters:

Name	Type	Description	Default
mode	Literal['local', 'slurm', 'auto']	Execution mode. If "auto", detects based on SLURM environment variables.	'auto'
process_system_level_inputs	bool	If True, process system-level inputs (DEM, Mannings)	True
overwrite_system_inputs	bool	If True, overwrite existing system input files	False
compile_TRITON_SWMM	bool	If True, compile TRITON-SWMM	True
recompile_if_already_done_successfully	bool	If True, recompile even if already compiled successfully	False
prepare_scenarios	bool	If True, prepare scenarios before running	True
overwrite_scenario_if_already_set_up	bool	If True, overwrite existing scenarios	False
rerun_swmm_hydro_if_outputs_exist	bool	If True, rerun SWMM hydrology model even if outputs exist	False
process_timeseries	bool	If True, process timeseries outputs after simulations	True
which	Literal['TRITON', 'SWMM', 'both']	Which outputs to process	'both'
override_clear_raw	ClearRawValue | None	Runtime override for cfg_analysis.clear_raw (None reads YAML).	None
compression_level	int	Compression level for output files (0-9)	5
pickup_where_leftoff	bool	If True, resume simulations from last checkpoint	True
dry_run	bool	If True, only perform a dry run and return that result	False
verbose	bool	If True, print progress messages	True
override_hpc_total_nodes	int | None	If set, overrides hpc_total_nodes in the generated SBATCH script without mutating the config. Only valid for multi_sim_run_method="1_job_many_srun_tasks".	None

Returns:

Type	Description
dict	Status dictionary with keys: - success: bool - mode: str - snakefile_path: Path - message: str

Source code in src/hhemt/sensitivity_analysis.py

def submit_workflow(
    self,
    mode: Literal["local", "slurm", "auto"] = "auto",
    # setup stuff
    process_system_level_inputs: bool = True,
    overwrite_system_inputs: bool = False,
    compile_TRITON_SWMM: bool = True,
    recompile_if_already_done_successfully: bool = False,
    # ensemble run stuff
    prepare_scenarios: bool = True,
    overwrite_scenario_if_already_set_up: bool = False,
    rerun_swmm_hydro_if_outputs_exist: bool = False,
    process_timeseries: bool = True,
    which: Literal["TRITON", "SWMM", "both"] = "both",
    override_clear_raw: ClearRawValue | None = None,
    override_force_rerun: ForceRerunValue | None = None,
    compression_level: int = 5,
    pickup_where_leftoff: bool = True,
    wait_for_completion: bool = False,  # relevant for slurm jobs only
    dry_run: bool = False,
    verbose: bool = True,
    override_hpc_total_nodes: int | None = None,
    override_hpc_restart_times_simulate: int | None = None,
    override_hpc_restart_times_other: int | None = None,
    report_formats: list[str] | None = None,
    extra_sbatch_args: list[str] | None = None,
    snakemake_diagnostics: SnakemakeDiagnostics | None = None,
) -> dict:
    """
    Submit sensitivity analysis workflow using Snakemake.

    This orchestrates multiple sub-analysis workflows and a final master
    consolidation step that combines all sub-analysis outputs.

    Parameters
    ----------
    mode : Literal["local", "slurm", "auto"]
        Execution mode. If "auto", detects based on SLURM environment variables.
    process_system_level_inputs : bool
        If True, process system-level inputs (DEM, Mannings)
    overwrite_system_inputs : bool
        If True, overwrite existing system input files
    compile_TRITON_SWMM : bool
        If True, compile TRITON-SWMM
    recompile_if_already_done_successfully : bool
        If True, recompile even if already compiled successfully
    prepare_scenarios : bool
        If True, prepare scenarios before running
    overwrite_scenario_if_already_set_up : bool
        If True, overwrite existing scenarios
    rerun_swmm_hydro_if_outputs_exist : bool
        If True, rerun SWMM hydrology model even if outputs exist
    process_timeseries : bool
        If True, process timeseries outputs after simulations
    which : Literal["TRITON", "SWMM", "both"]
        Which outputs to process
    override_clear_raw : ClearRawValue | None
        Runtime override for ``cfg_analysis.clear_raw`` (None reads YAML).
    compression_level : int
        Compression level for output files (0-9)
    pickup_where_leftoff : bool
        If True, resume simulations from last checkpoint
    dry_run : bool
        If True, only perform a dry run and return that result
    verbose : bool
        If True, print progress messages
    override_hpc_total_nodes : int | None
        If set, overrides `hpc_total_nodes` in the generated SBATCH script without
        mutating the config. Only valid for `multi_sim_run_method="1_job_many_srun_tasks"`.

    Returns
    -------
    dict
        Status dictionary with keys:
        - success: bool
        - mode: str
        - snakefile_path: Path
        - message: str
    """
    # Force-rerun pre-delete for direct sensitivity.submit_workflow callers.
    # Idempotent when Analysis.submit_workflow already applied it on the
    # dispatch path (matched flags would be absent by now).
    self.master_analysis._apply_force_rerun(override_force_rerun)

    # Driver-start orchestrator-liveness sentinel (Phase 2), keyed on the
    # MASTER analysis_dir. This is the sensitivity-master submit path and
    # always owns its sentinel (the Analysis.submit_workflow guard leaves
    # _driver_id None there and delegates here). Blocking-local drivers
    # remove on return; detached drivers leave a durable sentinel reclaimed
    # by the gate's liveness probes.
    _master_dir = self.master_analysis.analysis_paths.analysis_dir
    _eff_mode = self.master_analysis.cfg_analysis.multi_sim_run_method
    _driver_id = _osent.new_driver_id()
    _osent.write_orchestrator_sentinel(
        _master_dir,
        driver_id=_driver_id,
        workflow_submission_mode=_eff_mode,
    )
    try:
        result = self._workflow_builder.submit_workflow(
            mode=mode,
            process_system_level_inputs=process_system_level_inputs,
            overwrite_system_inputs=overwrite_system_inputs,
            compile_TRITON_SWMM=compile_TRITON_SWMM,
            recompile_if_already_done_successfully=recompile_if_already_done_successfully,
            prepare_scenarios=prepare_scenarios,
            overwrite_scenario_if_already_set_up=overwrite_scenario_if_already_set_up,
            rerun_swmm_hydro_if_outputs_exist=rerun_swmm_hydro_if_outputs_exist,
            process_timeseries=process_timeseries,
            which=which,
            override_clear_raw=override_clear_raw,
            compression_level=compression_level,
            pickup_where_leftoff=pickup_where_leftoff,
            wait_for_completion=wait_for_completion,
            dry_run=dry_run,
            verbose=verbose,
            override_hpc_total_nodes=override_hpc_total_nodes,
            override_hpc_restart_times_simulate=override_hpc_restart_times_simulate,
            override_hpc_restart_times_other=override_hpc_restart_times_other,
            report_formats=report_formats,
            extra_sbatch_args=extra_sbatch_args,
            snakemake_diagnostics=snakemake_diagnostics,
        )
    finally:
        if _eff_mode == "local":
            _osent.remove_orchestrator_sentinel(_master_dir, _driver_id)

    if _eff_mode != "local" and isinstance(result, dict):
        _osent.enrich_orchestrator_sentinel(
            _master_dir,
            driver_id=_driver_id,
            slurm_jobid=result.get("job_id"),
            tmux_session_name=result.get("session_name"),
        )

    return result

Bundle subsystem — render-bundle emission and the consume-side Bundle class.

Public surface

• Bundle — the consume-side entry point for a render bundle. Used by callers that already have a bundle on disk and want to regenerate the analysis report locally without re-running Analysis.run().
• emit_bundle — the emit-side helper. Invoked by Analysis.bundle_report_data() and TRITONSWMM_sensitivity_analysis.bundle_report_data() to produce a portable render bundle from a completed HPC analysis.

Bundle is deliberately NOT a subclass of TRITONSWMM_analysis. Bundle outputs are pre-computed; Analysis.run() is not callable against a bundle. The class boundary is the user's binding constraint per the bundle-portable-report-regeneration plan's Friction 5 design recommendation.

Bundle

A portable render bundle, ready for local report regeneration.

Source code in src/hhemt/bundle/__init__.py

class Bundle:
    """A portable render bundle, ready for local report regeneration."""

    def __init__(
        self,
        root: Path,
        manifest: dict,
        cfg_analysis: analysis_config | None = None,  # noqa: F821 — forward ref
    ) -> None:
        self._root = root.resolve()
        self._manifest = manifest
        self._cfg_analysis = cfg_analysis

    @classmethod
    def from_directory(cls, path: Path | str) -> Bundle:
        # Construct a Bundle from a bundle directory on disk.
        #
        # The directory must contain bundle_manifest.json. All paths
        # resolve via bundle.root at call time — no os.chdir, no
        # persisted absolute paths.
        #
        # Schema version validation: the manifest's
        # bundle_schema_version must be <= the locally-installed
        # toolkit's BUNDLE_SCHEMA_VERSION. A higher version means the
        # bundle was emitted by a newer toolkit; the local
        # installation cannot guarantee correct read.
        #
        # Legacy-bundle backward compatibility: pre-Plan-Phase-3
        # bundles lack the bundle_root_invariants manifest key; this
        # is treated as {} (no enforced invariants — legacy bundles
        # relied on consume-side cwd-based path resolution, which
        # Plan Phases 1 + 3 supersede).
        root = Path(path).resolve()
        manifest_path = root / BUNDLE_MANIFEST_FILENAME
        if not manifest_path.exists():
            raise FileNotFoundError(f"No {BUNDLE_MANIFEST_FILENAME} under {root}. Is this a render bundle?")
        manifest = json.loads(manifest_path.read_text())
        try:
            bundle_version = manifest["bundle_schema_version"]
        except KeyError as exc:
            raise ValueError(
                f"Malformed manifest at {manifest_path}: missing "
                f"required 'bundle_schema_version' field. Legacy "
                f"bundles (pre-Plan-Phase-3) lack bundle_root_invariants "
                f"but do contain bundle_schema_version; absence of this "
                f"field indicates the manifest was not produced by any "
                f"version of the toolkit's bundle emitter."
            ) from exc
        if bundle_version > BUNDLE_SCHEMA_VERSION:
            raise BundleSchemaError(
                f"Bundle {root} has bundle_schema_version="
                f"{bundle_version} which exceeds local "
                f"BUNDLE_SCHEMA_VERSION={BUNDLE_SCHEMA_VERSION}. "
                f"Upgrade the toolkit."
            )
        if bundle_version < BUNDLE_SCHEMA_VERSION:
            raise BundleSchemaError(
                f"Bundle {root} has bundle_schema_version="
                f"{bundle_version} which is below local "
                f"BUNDLE_SCHEMA_VERSION={BUNDLE_SCHEMA_VERSION}. "
                f"Pre-F2 bundles (v1) cannot load under post-F2 toolkit (v2) "
                f"because the bundle layout dropped its legacy peer-file for "
                f"report config and cfg_analysis.yaml gained a required "
                f"`report:` field. Re-emit the bundle from its source "
                f"analysis after running V0005 migration on the source dir."
            )
        invariants = manifest.get("bundle_root_invariants", {})
        if not isinstance(invariants, dict):
            raise ValueError(
                f"bundle_root_invariants in {manifest_path} must be a dict, got {type(invariants).__name__}."
            )
        # Load and Pydantic-validate the bundle's cfg_analysis.yaml at
        # construction time so downstream attribute access
        # (_read_static_backend reading
        # self._cfg_analysis.report.interactive.static_backend) is
        # guaranteed safe by R1's required-field contract.
        from hhemt.config.analysis import analysis_config
        from hhemt.config.loaders import yaml_to_model

        cfg_analysis_path = root / "cfg_analysis.yaml"
        if not cfg_analysis_path.exists():
            raise FileNotFoundError(
                f"Bundle at {root} is missing cfg_analysis.yaml — "
                f"required by R1 (analysis_config.report "
                f"load-time-required)."
            )
        cfg_analysis = yaml_to_model(cfg_analysis_path, analysis_config)
        return cls(root=root, manifest=manifest, cfg_analysis=cfg_analysis)

    @property
    def root(self) -> Path:
        return self._root

    @property
    def manifest(self) -> dict:
        return self._manifest

    def absolute(self, rel: str | Path) -> Path:
        """Resolve a bundle-relative path to an absolute path under
        ``bundle.root``."""
        return (self._root / rel).resolve()

    def eda(self, *, plots_only: bool = True, notebook_filename: str | None = None) -> EdaReportResult:
        """Regenerate the EDA report locally from the bundled data (ADR-10).

        Delegates to the SAME eda/ free functions analysis.eda() uses, passing
        bundle.root as root and the bundled cfg. plots_only=True is the only
        supported mode — a Bundle has no source datatree, so calc cannot run;
        the EDA datasets (eda/<plot_id>.zarr) and rendered plots were carried into
        the bundle by the harvest chain when analysis.eda() ran pre-bundle.
        """
        from hhemt.eda import (
            EdaReportResult,
            render_eda_plots,
        )
        from hhemt.eda._html_export import export_eda_html
        from hhemt.eda._local_surface import emit_eda_local_surface
        from hhemt.eda._notebook import emit_eda_notebook

        if not plots_only:
            raise ValueError(
                "Bundle.eda() supports only plots_only=True — a bundle carries no "
                "source datatree, so the EDA calc stage cannot run."
            )
        eda_cfg = self._cfg_analysis.eda
        emit_eda_local_surface(self._root)
        plot_paths = render_eda_plots(self._root, cfg_analysis=self._cfg_analysis, eda_cfg=eda_cfg)
        notebook_path = emit_eda_notebook(
            self._root,
            cfg_analysis=self._cfg_analysis,
            eda_cfg=eda_cfg,
            is_bundle=True,
            notebook_filename=notebook_filename,
        )
        report_path = export_eda_html(notebook_path, root=self._root)
        return EdaReportResult(
            report_path=report_path,
            notebook_path=notebook_path,
            plot_paths=plot_paths,
            verdicts=[],
        )

    def reprex(self, reprex_config, target_hpc_profile):
        """Validate round-trip runnability against a target HPC profile (ADR-10).

        Verifies the SIF (mandatory sha256 digest match when the crate references
        one, fail-closed; best-effort ``apptainer verify`` PGP), re-aims
        ``validation.py`` preflight at ``target_hpc_profile``, and emits per-``(sa_id,
        column)`` problem pairs plus per-field graduated experiment amendments for
        closest-possible cross-system reproduction. Mirrors ``Bundle.eda()`` —
        delegates to the ``bundle._reprex`` free function with ``bundle.root``.

        Parameters
        ----------
        reprex_config : hhemt.config.reprex_config.reprex_config
            The target user's minimal runnable-field set (account / login node /
            SIF path / scratch + target partition selectors).
        target_hpc_profile : hhemt.config.hpc_system.hpc_system_config
            The reproducer's own HPC profile (partition caps, container spec).

        Returns
        -------
        hhemt.bundle._reprex.ReprexResult
        """
        from hhemt.bundle._reprex import reprex as _reprex

        return _reprex(self._root, reprex_config, target_hpc_profile)

    def _read_static_backend(self) -> Literal["matplotlib", "plotly"]:
        # Resolution (post-F2 rev v2): cfg_analysis.report is required by
        # analysis_config Pydantic schema (Phase 1, R1). The 3-step F1
        # resolution order is deleted along with the legacy report-config
        # peer file (Phase 3). Loading the bundle via Bundle.from_directory(...)
        # already validated cfg_analysis.yaml against analysis_config and
        # would have raised ValidationError if `report:` was absent — so
        # the attribute access below is guaranteed safe at this point.
        # Private (leading underscore) but kept on the public Bundle class
        # so test code can monkey-patch for backend-override coverage.
        return self._cfg_analysis.report.interactive.static_backend

    def regenerate_report(self, *, format: Literal["html", "zip"] = "zip") -> Path:
        """Regenerate the analysis report from bundled data.

        Phase 2 wires (a) the regeneration-scoped Snakefile generator
        and (b) the report-templates staging step. Phase 3 wires the
        subprocess invocation, CLI integration, and the
        ``_read_static_backend()`` cfg-read that derives the static
        backend from ``cfg_analysis.yaml``'s
        ``report.interactive.static_backend`` field (default ``"plotly"``
        per Decision 4 / Plan Phase 2 D3).

        Parameters
        ----------
        format : {"html", "zip"}
            Final report format. Default ``"html"`` is the user-facing
            default at the Python API surface — distinct from an
            internal in-code default. The static backend is NOT a
            caller-facing parameter; it is derived from the bundle's
            cfg files via ``self._read_static_backend()`` so the
            user-visible default is config-controlled (per the
            project's no-in-code-defaults principle).
        """
        from hhemt.bundle.snakefile_generator import (
            write_regeneration_snakefile,
        )
        from hhemt.workflow import _emit_report_artifacts

        static_backend = self._read_static_backend()
        from hhemt.config.brand_theme import DEFAULT_BRAND_THEME
        from hhemt.config.loaders import load_brand_theme
        from hhemt.workflow import _brand_theme_css_map

        _bt = self._cfg_analysis.brand_theme if self._cfg_analysis else None
        _theme = load_brand_theme(self._root / _bt) if _bt else DEFAULT_BRAND_THEME
        _emit_report_artifacts(self._root, brand_theme=_brand_theme_css_map(_theme))
        write_regeneration_snakefile(self._root, static_backend=static_backend)

        # Defense-in-depth stale-lock check (per Decision 3.1A). CLI does
        # silent cleanup before reaching here; this check catches lock state
        # for Python-API callers that bypass the CLI (notebook usage).
        locks_dir = self._root / ".snakemake" / "locks"
        if locks_dir.exists() and any(locks_dir.iterdir()):
            lock_paths = sorted(p.name for p in locks_dir.iterdir())
            raise RuntimeError(
                f"Stale Snakemake locks under {locks_dir}: {lock_paths}. "
                f"Run `python -m snakemake --unlock --snakefile "
                f"{self._root}/Snakefile --directory {self._root}` to clear "
                f"them, or delete the locks/ directory manually. Locks are "
                f"left behind by an interrupted prior render."
            )

        # Determine final report output path under bundle root.
        output_path = self._root / f"analysis_report.{format}"

        # Build snakemake invocation. cwd= is set on the subprocess (not
        # the parent process) per R3 — no process-global os.chdir.
        # run_subprocess_with_tee is imported at module level so test
        # code can monkeypatch the binding site (bundle.__init__) — see
        # VMS-9 tests in tests/test_bundle.py.
        logs_dir = self._root / "_logs"
        logs_dir.mkdir(parents=True, exist_ok=True)
        # Mark all existing plot outputs as up-to-date so --report does
        # not attempt to re-run plot rules against absent source data.
        # The bundle contains rendered plots, NOT the source datatree;
        # without --touch, snakemake's needrun check fires because no
        # .snakemake/metadata/ exists on a fresh local working directory.
        # Per knowledge doc `library/knowledge/snakemake/rerun triggers.md`
        # and the Snakemake `--touch` documented contract.
        touch_cmd = [
            "snakemake",
            "--snakefile",
            str(self._root / "Snakefile"),
            "--directory",
            str(self._root),
            "--cores",
            "1",
            "--touch",
            "--quiet",
        ]
        touch_proc = run_subprocess_with_tee(
            touch_cmd,
            logfile=logs_dir / "regenerate_touch.log",
            cwd=self._root,
            echo_to_stdout=False,
        )
        if touch_proc.returncode != 0:
            raise RuntimeError(
                f"snakemake --touch failed with exit code "
                f"{touch_proc.returncode}. See log: "
                f"{logs_dir / 'regenerate_touch.log'}"
            )
        # Pass the actual output_path to --report regardless of format.
        # Snakemake auto-detects format from extension: ".html" → single-file
        # HTML; ".zip" → multi-file directory tree zip (small index report.html
        # + sibling data/ folder). The native zip shape matches user
        # expectation of "extracts to a folder with a small html + subfolders
        # with the plotting content."
        cmd = [
            "snakemake",
            "--snakefile",
            str(self._root / "Snakefile"),
            "--directory",
            str(self._root),
            "--cores",
            "1",
            "--report",
            str(output_path),
            "--report-stylesheet",
            str(self._root / "report" / "report.css"),
            "--quiet",
        ]
        proc = run_subprocess_with_tee(
            cmd,
            logfile=logs_dir / "regenerate.log",
            cwd=self._root,
            echo_to_stdout=True,
        )
        if proc.returncode != 0:
            raise RuntimeError(
                f"snakemake regeneration failed with exit code "
                f"{proc.returncode}. See log: {logs_dir / 'regenerate.log'}"
            )

        # Apply React-bundle post-process surgery. ``bundle_mode=True`` drops
        # the bundle-irrelevant chrome (Workflow + Statistics menu items, the
        # empty-after-drops "General" ListHeading). Unconditional surgery
        # steps (initial-view to "metadata", navbar text, category order,
        # placeholder category, showCategory auto-pop, click delegate, About
        # drop, title clear) apply to both formats via either branch.
        from ..report_renderers._react_surgery import (
            apply_post_process_surgery,
            apply_post_process_surgery_to_zip,
        )

        # Navbar upper-left brand text from the bundled theme (D-6/D-9), defaulting
        # to the bundle's analysis_id; None falls back to the historical literal.
        _navbar = _theme.upper_left_text or (self._cfg_analysis.analysis_id if self._cfg_analysis else None)
        try:
            if format == "html":
                output_path.write_text(
                    apply_post_process_surgery(
                        output_path.read_text(),
                        bundle_mode=True,
                        navbar_text=_navbar,
                    )
                )
            else:
                apply_post_process_surgery_to_zip(
                    output_path,
                    bundle_mode=True,
                    navbar_text=_navbar,
                )
        except Exception:
            pass

        return output_path

    def _zip_html(self, html_path: Path) -> Path:
        # Bundle the rendered HTML into a single zip at
        # {bundle_root}/analysis_report.zip. Plan Phase 4 may tighten
        # the contents to match its zip-emit determinism contract; this
        # is a minimal implementation for Plan Phase 3.
        import zipfile

        zip_path = self._root / "analysis_report.zip"
        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
            zf.write(html_path, arcname=html_path.name)
        return zip_path

absolute(rel)

Resolve a bundle-relative path to an absolute path under bundle.root.

Source code in src/hhemt/bundle/__init__.py

def absolute(self, rel: str | Path) -> Path:
    """Resolve a bundle-relative path to an absolute path under
    ``bundle.root``."""
    return (self._root / rel).resolve()

eda(*, plots_only=True, notebook_filename=None)

Regenerate the EDA report locally from the bundled data (ADR-10).

Delegates to the SAME eda/ free functions analysis.eda() uses, passing bundle.root as root and the bundled cfg. plots_only=True is the only supported mode — a Bundle has no source datatree, so calc cannot run; the EDA datasets (eda/.zarr) and rendered plots were carried into the bundle by the harvest chain when analysis.eda() ran pre-bundle.

Source code in src/hhemt/bundle/__init__.py

def eda(self, *, plots_only: bool = True, notebook_filename: str | None = None) -> EdaReportResult:
    """Regenerate the EDA report locally from the bundled data (ADR-10).

    Delegates to the SAME eda/ free functions analysis.eda() uses, passing
    bundle.root as root and the bundled cfg. plots_only=True is the only
    supported mode — a Bundle has no source datatree, so calc cannot run;
    the EDA datasets (eda/<plot_id>.zarr) and rendered plots were carried into
    the bundle by the harvest chain when analysis.eda() ran pre-bundle.
    """
    from hhemt.eda import (
        EdaReportResult,
        render_eda_plots,
    )
    from hhemt.eda._html_export import export_eda_html
    from hhemt.eda._local_surface import emit_eda_local_surface
    from hhemt.eda._notebook import emit_eda_notebook

    if not plots_only:
        raise ValueError(
            "Bundle.eda() supports only plots_only=True — a bundle carries no "
            "source datatree, so the EDA calc stage cannot run."
        )
    eda_cfg = self._cfg_analysis.eda
    emit_eda_local_surface(self._root)
    plot_paths = render_eda_plots(self._root, cfg_analysis=self._cfg_analysis, eda_cfg=eda_cfg)
    notebook_path = emit_eda_notebook(
        self._root,
        cfg_analysis=self._cfg_analysis,
        eda_cfg=eda_cfg,
        is_bundle=True,
        notebook_filename=notebook_filename,
    )
    report_path = export_eda_html(notebook_path, root=self._root)
    return EdaReportResult(
        report_path=report_path,
        notebook_path=notebook_path,
        plot_paths=plot_paths,
        verdicts=[],
    )

regenerate_report(*, format='zip')

Regenerate the analysis report from bundled data.

Phase 2 wires (a) the regeneration-scoped Snakefile generator and (b) the report-templates staging step. Phase 3 wires the subprocess invocation, CLI integration, and the _read_static_backend() cfg-read that derives the static backend from cfg_analysis.yaml's report.interactive.static_backend field (default "plotly" per Decision 4 / Plan Phase 2 D3).

Parameters:

Name	Type	Description	Default
format	('html', 'zip')	Final report format. Default "html" is the user-facing default at the Python API surface — distinct from an internal in-code default. The static backend is NOT a caller-facing parameter; it is derived from the bundle's cfg files via self._read_static_backend() so the user-visible default is config-controlled (per the project's no-in-code-defaults principle).	"html"

Source code in src/hhemt/bundle/__init__.py

def regenerate_report(self, *, format: Literal["html", "zip"] = "zip") -> Path:
    """Regenerate the analysis report from bundled data.

    Phase 2 wires (a) the regeneration-scoped Snakefile generator
    and (b) the report-templates staging step. Phase 3 wires the
    subprocess invocation, CLI integration, and the
    ``_read_static_backend()`` cfg-read that derives the static
    backend from ``cfg_analysis.yaml``'s
    ``report.interactive.static_backend`` field (default ``"plotly"``
    per Decision 4 / Plan Phase 2 D3).

    Parameters
    ----------
    format : {"html", "zip"}
        Final report format. Default ``"html"`` is the user-facing
        default at the Python API surface — distinct from an
        internal in-code default. The static backend is NOT a
        caller-facing parameter; it is derived from the bundle's
        cfg files via ``self._read_static_backend()`` so the
        user-visible default is config-controlled (per the
        project's no-in-code-defaults principle).
    """
    from hhemt.bundle.snakefile_generator import (
        write_regeneration_snakefile,
    )
    from hhemt.workflow import _emit_report_artifacts

    static_backend = self._read_static_backend()
    from hhemt.config.brand_theme import DEFAULT_BRAND_THEME
    from hhemt.config.loaders import load_brand_theme
    from hhemt.workflow import _brand_theme_css_map

    _bt = self._cfg_analysis.brand_theme if self._cfg_analysis else None
    _theme = load_brand_theme(self._root / _bt) if _bt else DEFAULT_BRAND_THEME
    _emit_report_artifacts(self._root, brand_theme=_brand_theme_css_map(_theme))
    write_regeneration_snakefile(self._root, static_backend=static_backend)

    # Defense-in-depth stale-lock check (per Decision 3.1A). CLI does
    # silent cleanup before reaching here; this check catches lock state
    # for Python-API callers that bypass the CLI (notebook usage).
    locks_dir = self._root / ".snakemake" / "locks"
    if locks_dir.exists() and any(locks_dir.iterdir()):
        lock_paths = sorted(p.name for p in locks_dir.iterdir())
        raise RuntimeError(
            f"Stale Snakemake locks under {locks_dir}: {lock_paths}. "
            f"Run `python -m snakemake --unlock --snakefile "
            f"{self._root}/Snakefile --directory {self._root}` to clear "
            f"them, or delete the locks/ directory manually. Locks are "
            f"left behind by an interrupted prior render."
        )

    # Determine final report output path under bundle root.
    output_path = self._root / f"analysis_report.{format}"

    # Build snakemake invocation. cwd= is set on the subprocess (not
    # the parent process) per R3 — no process-global os.chdir.
    # run_subprocess_with_tee is imported at module level so test
    # code can monkeypatch the binding site (bundle.__init__) — see
    # VMS-9 tests in tests/test_bundle.py.
    logs_dir = self._root / "_logs"
    logs_dir.mkdir(parents=True, exist_ok=True)
    # Mark all existing plot outputs as up-to-date so --report does
    # not attempt to re-run plot rules against absent source data.
    # The bundle contains rendered plots, NOT the source datatree;
    # without --touch, snakemake's needrun check fires because no
    # .snakemake/metadata/ exists on a fresh local working directory.
    # Per knowledge doc `library/knowledge/snakemake/rerun triggers.md`
    # and the Snakemake `--touch` documented contract.
    touch_cmd = [
        "snakemake",
        "--snakefile",
        str(self._root / "Snakefile"),
        "--directory",
        str(self._root),
        "--cores",
        "1",
        "--touch",
        "--quiet",
    ]
    touch_proc = run_subprocess_with_tee(
        touch_cmd,
        logfile=logs_dir / "regenerate_touch.log",
        cwd=self._root,
        echo_to_stdout=False,
    )
    if touch_proc.returncode != 0:
        raise RuntimeError(
            f"snakemake --touch failed with exit code "
            f"{touch_proc.returncode}. See log: "
            f"{logs_dir / 'regenerate_touch.log'}"
        )
    # Pass the actual output_path to --report regardless of format.
    # Snakemake auto-detects format from extension: ".html" → single-file
    # HTML; ".zip" → multi-file directory tree zip (small index report.html
    # + sibling data/ folder). The native zip shape matches user
    # expectation of "extracts to a folder with a small html + subfolders
    # with the plotting content."
    cmd = [
        "snakemake",
        "--snakefile",
        str(self._root / "Snakefile"),
        "--directory",
        str(self._root),
        "--cores",
        "1",
        "--report",
        str(output_path),
        "--report-stylesheet",
        str(self._root / "report" / "report.css"),
        "--quiet",
    ]
    proc = run_subprocess_with_tee(
        cmd,
        logfile=logs_dir / "regenerate.log",
        cwd=self._root,
        echo_to_stdout=True,
    )
    if proc.returncode != 0:
        raise RuntimeError(
            f"snakemake regeneration failed with exit code "
            f"{proc.returncode}. See log: {logs_dir / 'regenerate.log'}"
        )

    # Apply React-bundle post-process surgery. ``bundle_mode=True`` drops
    # the bundle-irrelevant chrome (Workflow + Statistics menu items, the
    # empty-after-drops "General" ListHeading). Unconditional surgery
    # steps (initial-view to "metadata", navbar text, category order,
    # placeholder category, showCategory auto-pop, click delegate, About
    # drop, title clear) apply to both formats via either branch.
    from ..report_renderers._react_surgery import (
        apply_post_process_surgery,
        apply_post_process_surgery_to_zip,
    )

    # Navbar upper-left brand text from the bundled theme (D-6/D-9), defaulting
    # to the bundle's analysis_id; None falls back to the historical literal.
    _navbar = _theme.upper_left_text or (self._cfg_analysis.analysis_id if self._cfg_analysis else None)
    try:
        if format == "html":
            output_path.write_text(
                apply_post_process_surgery(
                    output_path.read_text(),
                    bundle_mode=True,
                    navbar_text=_navbar,
                )
            )
        else:
            apply_post_process_surgery_to_zip(
                output_path,
                bundle_mode=True,
                navbar_text=_navbar,
            )
    except Exception:
        pass

    return output_path

reprex(reprex_config, target_hpc_profile)

Validate round-trip runnability against a target HPC profile (ADR-10).

Verifies the SIF (mandatory sha256 digest match when the crate references one, fail-closed; best-effort apptainer verify PGP), re-aims validation.py preflight at target_hpc_profile, and emits per-(sa_id, column) problem pairs plus per-field graduated experiment amendments for closest-possible cross-system reproduction. Mirrors Bundle.eda() — delegates to the bundle._reprex free function with bundle.root.

Parameters:

Name	Type	Description	Default
reprex_config	reprex_config	The target user's minimal runnable-field set (account / login node / SIF path / scratch + target partition selectors).	required
target_hpc_profile	hpc_system_config	The reproducer's own HPC profile (partition caps, container spec).	required

Returns:

Type	Description
ReprexResult

Source code in src/hhemt/bundle/__init__.py

def reprex(self, reprex_config, target_hpc_profile):
    """Validate round-trip runnability against a target HPC profile (ADR-10).

    Verifies the SIF (mandatory sha256 digest match when the crate references
    one, fail-closed; best-effort ``apptainer verify`` PGP), re-aims
    ``validation.py`` preflight at ``target_hpc_profile``, and emits per-``(sa_id,
    column)`` problem pairs plus per-field graduated experiment amendments for
    closest-possible cross-system reproduction. Mirrors ``Bundle.eda()`` —
    delegates to the ``bundle._reprex`` free function with ``bundle.root``.

    Parameters
    ----------
    reprex_config : hhemt.config.reprex_config.reprex_config
        The target user's minimal runnable-field set (account / login node /
        SIF path / scratch + target partition selectors).
    target_hpc_profile : hhemt.config.hpc_system.hpc_system_config
        The reproducer's own HPC profile (partition caps, container spec).

    Returns
    -------
    hhemt.bundle._reprex.ReprexResult
    """
    from hhemt.bundle._reprex import reprex as _reprex

    return _reprex(self._root, reprex_config, target_hpc_profile)

BundleSchemaError

Bases: ValueError

A bundle's bundle_schema_version does not match the locally-installed toolkit's BUNDLE_SCHEMA_VERSION. Distinct from generic malformed-manifest errors so callers can branch on schema-version mismatch specifically.

Source code in src/hhemt/bundle/__init__.py

class BundleSchemaError(ValueError):
    """A bundle's bundle_schema_version does not match the locally-installed
    toolkit's BUNDLE_SCHEMA_VERSION. Distinct from generic malformed-manifest
    errors so callers can branch on schema-version mismatch specifically."""

CombinedBundle

Consume-side handle for a standalone combined bundle (mirrors Bundle).

Source code in src/hhemt/bundle/_combine.py

class CombinedBundle:
    """Consume-side handle for a standalone combined bundle (mirrors Bundle)."""

    def __init__(self, root: Path) -> None:
        self._root = root.resolve()

    @classmethod
    def from_directory(cls, path: Path | str) -> CombinedBundle:
        root = Path(path).resolve()
        manifest_path = root / BUNDLE_MANIFEST_FILENAME
        if not manifest_path.exists():
            raise FileNotFoundError(f"No {BUNDLE_MANIFEST_FILENAME} under {root}.")

        from hhemt.bundle import BundleSchemaError

        manifest = json.loads(manifest_path.read_text())
        version = manifest.get("bundle_schema_version")
        if version != BUNDLE_SCHEMA_VERSION:
            raise BundleSchemaError(
                f"Combined bundle {root} has bundle_schema_version={version} "
                f"!= local BUNDLE_SCHEMA_VERSION={BUNDLE_SCHEMA_VERSION}."
            )
        return cls(root=root)

    @property
    def root(self) -> Path:
        return self._root

    def regenerate_report(self, *, format: Literal["html", "zip"] = "zip") -> Path:
        """Regenerate the combined report from the bundled data (mirrors Bundle.regenerate_report).

        Re-invokes the SAME emit-time render path against the bundle root (no
        combined Snakefile -> no ``snakemake --report``). The renderer reads the
        bundled ``combined_compatibility.json`` read-model, so no re-merge is needed.
        """
        _render_combined_report(None, None, self._root)
        report = self._root / f"analysis_report.{format}"
        if not report.exists():
            raise FileNotFoundError(f"Combined report {report} was not produced.")
        return report

    def eda(self, *, plots_only: bool = True):
        """Combined bundles have no aggregate EDA surface (see child bundles)."""
        raise NotImplementedError(
            "Combined bundles have no aggregate EDA surface. The cross-experiment "
            "byte-identity panel is deferred (R6); run .eda() on each child bundle "
            "under child_crates/{experiment_id}/ via Bundle.from_directory(...)."
        )

eda(*, plots_only=True)

Combined bundles have no aggregate EDA surface (see child bundles).

Source code in src/hhemt/bundle/_combine.py

def eda(self, *, plots_only: bool = True):
    """Combined bundles have no aggregate EDA surface (see child bundles)."""
    raise NotImplementedError(
        "Combined bundles have no aggregate EDA surface. The cross-experiment "
        "byte-identity panel is deferred (R6); run .eda() on each child bundle "
        "under child_crates/{experiment_id}/ via Bundle.from_directory(...)."
    )

regenerate_report(*, format='zip')

Regenerate the combined report from the bundled data (mirrors Bundle.regenerate_report).

Re-invokes the SAME emit-time render path against the bundle root (no combined Snakefile -> no snakemake --report). The renderer reads the bundled combined_compatibility.json read-model, so no re-merge is needed.

Source code in src/hhemt/bundle/_combine.py

def regenerate_report(self, *, format: Literal["html", "zip"] = "zip") -> Path:
    """Regenerate the combined report from the bundled data (mirrors Bundle.regenerate_report).

    Re-invokes the SAME emit-time render path against the bundle root (no
    combined Snakefile -> no ``snakemake --report``). The renderer reads the
    bundled ``combined_compatibility.json`` read-model, so no re-merge is needed.
    """
    _render_combined_report(None, None, self._root)
    report = self._root / f"analysis_report.{format}"
    if not report.exists():
        raise FileNotFoundError(f"Combined report {report} was not produced.")
    return report

combine_bundle(bundle_paths, output_path=None)

Combine N completed render bundles into one standalone combined bundle.

Ingests two or more completed, unpacked render bundles from different experiments and emits a single standalone bundle carrying one cross-experiment report. The pipeline is: (1) check metadata compatibility across the bundles and ABORT on any blocking divergence (an EXPERIMENT-identity field mismatch); (2) merge each bundle's root consolidated tree into one DataTree-of-experiments, keyed on a scalar experiment coordinate; (3) render the combined reporting set directly at emit time; (4) write a flat, hasPart-by-reference Provenance-Run-Crate over the N intact child crates.

Both single-analysis bundles (which ship analysis_datatree.zarr at their root) and sensitivity-master bundles (sensitivity_datatree.zarr) are accepted; the root tree is resolved by existence.

Parameters:

Name	Type	Description	Default
bundle_paths	list of Path	Paths to at least two unpacked bundle DIRECTORIES (not .zip archives).	required
output_path	Path	Destination directory for the combined bundle. When omitted, a sibling of the first bundle named combined_{N}bundles_{toolkit_git_sha} is used.	None

Returns:

Type	Description
CombinedBundle	Consume-side handle for the emitted combined bundle. Call regenerate_report() on it to re-render locally.

Raises:

Type	Description
ConfigurationError	If fewer than two bundles are supplied, or if the bundles carry a blocking compatibility divergence (they do not describe the same experiment family).

Examples:

>>> from hhemt.bundle import combine_bundle
>>> combined = combine_bundle([Path("bundle_a"), Path("bundle_b")])
>>> combined.regenerate_report()

Source code in src/hhemt/bundle/_combine.py

def combine_bundle(
    bundle_paths: list[Path],
    output_path: Path | None = None,
) -> CombinedBundle:
    """Combine N completed render bundles into one standalone combined bundle.

    Ingests two or more **completed, unpacked** render bundles from different
    experiments and emits a single standalone bundle carrying one cross-experiment
    report. The pipeline is: (1) check metadata compatibility across the bundles
    and ABORT on any blocking divergence (an EXPERIMENT-identity field mismatch);
    (2) merge each bundle's root consolidated tree into one DataTree-of-experiments,
    keyed on a scalar ``experiment`` coordinate; (3) render the ``combined``
    reporting set directly at emit time; (4) write a flat, ``hasPart``-by-reference
    Provenance-Run-Crate over the N intact child crates.

    Both single-analysis bundles (which ship ``analysis_datatree.zarr`` at their
    root) and sensitivity-master bundles (``sensitivity_datatree.zarr``) are
    accepted; the root tree is resolved by existence.

    Parameters
    ----------
    bundle_paths : list of Path
        Paths to at least two unpacked bundle DIRECTORIES (not ``.zip`` archives).
    output_path : Path, optional
        Destination directory for the combined bundle. When omitted, a sibling of
        the first bundle named ``combined_{N}bundles_{toolkit_git_sha}`` is used.

    Returns
    -------
    CombinedBundle
        Consume-side handle for the emitted combined bundle. Call
        ``regenerate_report()`` on it to re-render locally.

    Raises
    ------
    ConfigurationError
        If fewer than two bundles are supplied, or if the bundles carry a blocking
        compatibility divergence (they do not describe the same experiment family).

    Examples
    --------
    >>> from hhemt.bundle import combine_bundle
    >>> combined = combine_bundle([Path("bundle_a"), Path("bundle_b")])  # doctest: +SKIP
    >>> combined.regenerate_report()  # doctest: +SKIP
    """
    roots = sorted(Path(p).resolve() for p in bundle_paths)
    if len(roots) < 2:
        raise ConfigurationError(
            field="bundle_paths",
            message=f"combine_bundle needs >=2 bundles, got {len(roots)}.",
            config_path=None,
        )
    report = check_bundle_compatibility(roots)
    if not report.is_compatible:
        blocking = "; ".join(
            f"{d.field_name} ({d.bucket}): {d.bundle_a}={d.value_a!r} vs {d.bundle_b}={d.value_b!r}"
            for d in report.blocking
        )
        raise ConfigurationError(
            field="bundle_paths",
            message=f"Bundles are not combine-compatible (blocking divergences): {blocking}",
            config_path=None,
        )
    merged = merge_experiment_trees(roots)  # consumed by the emit-time render step
    if output_path is None:
        sha = _get_toolkit_git_sha(strict=False)
        output_path = roots[0].parent / f"combined_{len(roots)}bundles_{sha}"
    output_path = Path(output_path)
    _emit_combined_bundle(roots, merged, report, output_path)
    return CombinedBundle.from_directory(output_path)

emit_bundle(analysis, output_path=None)

Emit a portable render bundle from a completed HPC analysis.

The bundle file set is the union of source paths declared via prov.artist().add_channel(...) calls during the most recent render_report() execution, harvested from *.manifest.json sidecars under {analysis_dir}/plots/. Configs are rewritten to relative paths. The HPC-baseline analysis_report.{html,zip} are preserved under bundle_baseline/.

Source code in src/hhemt/bundle/_emit.py

def emit_bundle(
    analysis: TRITONSWMM_analysis,
    output_path: Path | None = None,
) -> Path:
    """Emit a portable render bundle from a completed HPC analysis.

    The bundle file set is the union of source paths declared via
    prov.artist().add_channel(...) calls during the most recent
    render_report() execution, harvested from *.manifest.json sidecars
    under {analysis_dir}/plots/. Configs are rewritten to relative paths.
    The HPC-baseline analysis_report.{html,zip} are preserved under
    bundle_baseline/.
    """
    analysis_dir = analysis.analysis_paths.analysis_dir
    plots_dir = analysis_dir / BUNDLE_PLOTS_SUBDIR
    if not plots_dir.exists() or not list(plots_dir.rglob("*.manifest.json")):
        raise FileNotFoundError(
            f"No *.manifest.json sidecars found under {plots_dir}. "
            f"Bundle emission requires a completed render_report(). "
            f"Run analysis.render_report() on HPC first."
        )

    sources_by_renderer = harvest_source_paths(plots_dir, analysis_dir)
    git_sha = _get_toolkit_git_sha()
    analysis_id = analysis.cfg_analysis.analysis_id

    if output_path is None:
        output_path = (
            analysis_dir / BUNDLE_OUTPUT_SUBDIR
            / f"{analysis_id}_{git_sha}_v{BUNDLE_SCHEMA_VERSION}.zip"
        )
    output_path.parent.mkdir(parents=True, exist_ok=True)

    with _staging_dir(output_path.parent) as staging:
        _harvest_and_copy_sources(sources_by_renderer, analysis_dir, staging)
        _copy_bundle_baseline(analysis_dir, staging)
        aggregated_invariants = _copy_configs_with_relative_paths(analysis, staging)
        _emit_resolved_brand_theme(analysis, staging)
        _copy_supporting_files(analysis, staging)
        _emit_runnable_template_set(staging)
        _upgrade_crate_to_workflow_run_crate(staging)
        _write_bundle_manifest(
            staging,
            sources_by_renderer=sources_by_renderer,
            analysis_id=analysis_id,
            git_sha=git_sha,
            bundle_root_invariants=aggregated_invariants,
        )
        _emit_bundle_zip(staging, output_path)

    return output_path

eda/ — governed EDA data-prep subpackage (ADR-9).

EDA-supporting data-prep functions live here, one module per EDA family. Each returns an EdaResult (a CheckResult verdict + a manifest-sidecar'd derived artifact under {analysis_dir}/eda/). The first member is the cross-sim byte-for-byte identity verification (peak flood depth + conduit flow/ratios across sims sharing an event iloc on a sensitivity master).

These functions are NOT methods on processing_analysis.TRITONSWMM_analysis_post_processing (no coupling of EDA findings to the consolidation DAG) and are NOT pure analysis_validation.check_* functions (EDA prepares plottable data, not side-effect-free verdicts). The calc is invoked in-process by the downstream analysis.eda() facade — there is no Snakemake rule for the EDA loop.

EdaContext dataclass

Re-derived experiment context bound as named variables in the EDA notebook.

Source code in src/hhemt/eda/_context.py

@dataclass(frozen=True)
class EdaContext:
    """Re-derived experiment context bound as named variables in the EDA notebook."""

    datatree: xr.DataTree | None
    sensitivity_datatree: xr.DataTree | None
    cfg_analysis: analysis_config
    cfg_system: system_config
    scenario_status: pd.DataFrame | None
    swmm_features: dict[str, gpd.GeoDataFrame] | None
    triton_dem: xr.DataArray | None
    performance: xr.DataTree | None
    is_bundle: bool

EdaReportResult dataclass

Return value of analysis.eda() / Bundle.eda() (the facade layer).

Carries the three artifact classes the facade produces: the assembled doc, the rendered EDA plots, and the calc-stage verdicts (empty on Bundle.eda, which skips calc).

Source code in src/hhemt/eda/_result.py

@dataclass(frozen=True)
class EdaReportResult:
    """Return value of analysis.eda() / Bundle.eda() (the facade layer).

    Carries the three artifact classes the facade produces: the assembled doc,
    the rendered EDA plots, and the calc-stage verdicts (empty on Bundle.eda,
    which skips calc).
    """

    report_path: Path | None
    notebook_path: Path | None = None
    plot_paths: list[Path] = field(default_factory=list)
    verdicts: list[CheckResult] = field(default_factory=list)

EdaResult dataclass

Return value of an eda/ data-prep function.

Source code in src/hhemt/eda/_result.py

@dataclass
class EdaResult:
    """Return value of an eda/ data-prep function."""

    verdict: CheckResult | None = None
    artifact_path: Path | None = None
    plot_id: str | None = None
    skipped: bool = False

check_cross_sim_identity(analysis, *, within_family=True)

ADR-4: verify cross-sim reproducibility and EMIT a characterized-divergence verdict.

Returns a skipped EdaResult on a non-sensitivity analysis. On a sensitivity master, compares each enabled (event_iloc, mode, variable) across sub-analyses against the lexicographically-first present sa_id reference, writes {analysis_dir}/eda/<plot_id>.zarr (max-abs-diff + identical maps) and <plot_id>.verdict.json, and returns an EdaResult carrying the verdict + artifact path.

within_family=True (default — same signed SIF / same hardware family): assert bit-identity (np.array_equal(equal_nan=True)); a divergence is a CheckResult passed=False. This is today's behavior, unchanged.

within_family=False (across hardware families, e.g. Frontier-ROCm vs UVA-CUDA): do NOT assert equality — ADR-4 concedes cross-family bit-identity is not achievable. Instead compute the BOUNDED divergence (max abs diff and max relative diff per tracked variable) and emit it as a passed=True characterized-divergence verdict. The boundary disclosure IS the contribution (disclosed -> verifiable), not an equality claim. The persisted <plot_id>.verdict.json shape is unchanged (still dataclasses.asdict(CheckResult)); only the verdict's passed/summary/ details semantics branch on within_family.

Source code in src/hhemt/eda/cross_sim_identity.py

def check_cross_sim_identity(analysis: TRITONSWMM_analysis, *, within_family: bool = True) -> EdaResult:
    """ADR-4: verify cross-sim reproducibility and EMIT a characterized-divergence verdict.

    Returns a skipped ``EdaResult`` on a non-sensitivity analysis. On a sensitivity
    master, compares each enabled ``(event_iloc, mode, variable)`` across
    sub-analyses against the lexicographically-first present ``sa_id`` reference,
    writes ``{analysis_dir}/eda/<plot_id>.zarr`` (max-abs-diff + identical maps) and
    ``<plot_id>.verdict.json``, and returns an ``EdaResult`` carrying the verdict +
    artifact path.

    ``within_family=True`` (default — same signed SIF / same hardware family): assert
    bit-identity (``np.array_equal(equal_nan=True)``); a divergence is a
    ``CheckResult`` ``passed=False``. This is today's behavior, unchanged.

    ``within_family=False`` (across hardware families, e.g. Frontier-ROCm vs
    UVA-CUDA): do NOT assert equality — ADR-4 concedes cross-family bit-identity is
    not achievable. Instead compute the BOUNDED divergence (max abs diff and max
    relative diff per tracked variable) and emit it as a ``passed=True``
    characterized-divergence verdict. The boundary disclosure IS the contribution
    (disclosed -> verifiable), not an equality claim. The persisted
    ``<plot_id>.verdict.json`` shape is unchanged (still
    ``dataclasses.asdict(CheckResult)``); only the verdict's ``passed``/``summary``/
    ``details`` semantics branch on ``within_family``.
    """
    name = "Cross-sim byte-identity"
    sub_items = list(_iter_subanalyses_or_self(analysis))
    # Non-sensitivity: _iter_subanalyses_or_self yields a single (None, analysis).
    if len(sub_items) == 1 and sub_items[0][0] is None:
        return EdaResult(
            skipped=True,
            verdict=CheckResult(
                name=name,
                level="aggregate",
                passed=True,
                summary="N/A — single sim per event iloc",
            ),
        )

    # Reference = lexicographically-first sa_id whose summaries are present.
    subs = dict(sorted(((str(sa), sub) for sa, sub in sub_items), key=lambda kv: kv[0]))
    ref_id = next((sa for sa, sub in subs.items() if _enabled_modes(sub)), None)
    if ref_id is None:
        return EdaResult(
            skipped=True,
            verdict=CheckResult(
                name=name,
                level="aggregate",
                passed=True,
                summary="N/A — no sub-analysis has present summaries",
            ),
        )
    ref_sub = subs[ref_id]
    ref_modes = _enabled_modes(ref_sub)

    details: list[dict] = []
    diff_arrays: dict[str, list[xr.DataArray]] = {}
    identical_arrays: dict[str, list[xr.DataArray]] = {}
    all_identical = True
    # ADR-4 across-family accumulator: per-variable running max (abs, rel) divergence.
    # Populated only when within_family is False; ignored on the strict path.
    divergence: dict[str, dict[str, float]] = {}

    for sa_id, sub in subs.items():
        if sa_id == ref_id:
            continue
        if not _enabled_modes(sub):
            details.append({"sa_id": sa_id, "detail": "summaries absent — skipped"})
            continue
        for mode in ref_modes:
            try:
                ds_ref = ref_sub.process._retrieve_combined_output(mode)
                ds_cmp = sub.process._retrieve_combined_output(mode)
            except (FileNotFoundError, ValueError):
                continue
            for var in TRACKED_VARS:
                if var not in ds_ref.data_vars or var not in ds_cmp.data_vars:
                    continue
                for e in ds_ref["event_iloc"].values:
                    da_ref_sel = ds_ref[var].sel(event_iloc=e)
                    res = compare_variable_exact(da_ref_sel, ds_cmp[var].sel(event_iloc=e))
                    if within_family:
                        # Strict path (within-family / same signed SIF): a divergence
                        # is a verdict failure (today's behavior, unchanged).
                        if not res["identical"]:
                            all_identical = False
                            details.append(
                                {
                                    "sa_id": sa_id,
                                    "event_iloc": int(e),
                                    "variable": var,
                                    "detail": (
                                        f"max_abs_diff={res['max_abs_diff']:.6g}, "
                                        f"dtype_match={res['dtype_match']}, coord_match={res['coord_match']}"
                                    ),
                                }
                            )
                    else:
                        # ADR-4 across-family: characterize, do NOT fail on divergence.
                        # A NaN max_abs_diff means the cell sets are not comparable
                        # (coord mismatch / different mesh); record it as disclosed
                        # incomparability rather than folding it into the bounds.
                        max_abs = res["max_abs_diff"]
                        if not np.isfinite(max_abs):
                            details.append(
                                {
                                    "sa_id": sa_id,
                                    "event_iloc": int(e),
                                    "variable": var,
                                    "detail": "not comparable (coord/dtype mismatch)",
                                }
                            )
                        else:
                            ref_vals = da_ref_sel.values.astype("float64")
                            with np.errstate(invalid="ignore"):
                                denom = float(np.nanmax(np.abs(ref_vals))) if np.isfinite(ref_vals).any() else 0.0
                            denom = denom or 1.0
                            acc = divergence.setdefault(var, {"max_abs": 0.0, "max_rel": 0.0})
                            acc["max_abs"] = max(acc["max_abs"], max_abs)
                            acc["max_rel"] = max(acc["max_rel"], max_abs / denom)
                    # Collect diff/identical scalars for the plottable artifact.
                    diff_arrays.setdefault(var, []).append(
                        xr.DataArray(res["max_abs_diff"]).expand_dims({"sa_id": [sa_id], "event_iloc": [int(e)]})
                    )
                    identical_arrays.setdefault(var, []).append(
                        xr.DataArray(res["identical"]).expand_dims({"sa_id": [sa_id], "event_iloc": [int(e)]})
                    )

    # Assemble the plottable artifact (one max_abs_diff + identical var per tracked
    # variable, keyed by (sa_id, event_iloc)). The per-cell diff_map is retained in
    # the verdict details only; the scalar max-abs-diff is the plottable summary the
    # downstream eda-plotting plan keys on. (Per-cell map persistence is a downstream
    # enrichment — see Follow-up Ideas.)
    ds_vars: dict[str, xr.DataArray] = {}
    for var, arrs in diff_arrays.items():
        ds_vars[f"max_abs_diff__{var}"] = _combine_cells(arrs)
    for var, arrs in identical_arrays.items():
        ds_vars[f"identical__{var}"] = _combine_cells(arrs)
    artifact_ds = xr.Dataset(ds_vars)
    artifact_ds.attrs["reference_sa_id"] = ref_id

    if within_family:
        summary = (
            f"All tracked variables bit-identical across {len(subs) - 1} "
            f"non-reference sub-analyses (ref sa_id={ref_id})."
            if all_identical
            else f"{len([d for d in details if 'variable' in d])} (sa, event, variable) "
            f"tuple(s) diverged from reference sa_id={ref_id}."
        )
        passed = all_identical
    else:
        # ADR-4 across-family: the disclosed bounds ARE the verdict; passed=True
        # regardless of divergence magnitude (the boundary is verifiable, not a
        # claim of equality). Append the per-variable bounds to details so the
        # persisted verdict.json carries them.
        for var, acc in sorted(divergence.items()):
            details.append(
                {
                    "variable": var,
                    "max_abs_diff": acc["max_abs"],
                    "max_rel_diff": acc["max_rel"],
                }
            )
        if divergence:
            bounds = ", ".join(f"{var}={acc['max_abs']:.6g}" for var, acc in sorted(divergence.items()))
            summary = (
                f"Characterized divergence (across-family, disclosed; ref sa_id={ref_id}): "
                f"max_abs_diff per variable: {bounds}."
            )
        else:
            summary = (
                f"Characterized divergence (across-family): no comparable variables "
                f"across {len(subs) - 1} non-reference sub-analyses (ref sa_id={ref_id})."
            )
        passed = True
    verdict = CheckResult(
        name=name,
        level="aggregate",
        passed=passed,
        summary=summary,
        details=details,
    )

    # Persist artifact + verdict under {analysis_dir}/eda/. plot_id == stem (ADR-2).
    eda_dir = Path(analysis.analysis_paths.analysis_dir) / "eda"
    eda_dir.mkdir(parents=True, exist_ok=True)
    plot_id = canonical_plot_id("eda_cross_sim_identity")
    artifact_path = eda_dir / f"{plot_id}.zarr"
    artifact_ds.to_zarr(artifact_path, mode="w", consolidated=False)

    # Source paths = every per-sub summary file the comparison consumed. Declared so
    # the artifact is a first-class harvest_source_paths provenance source (ADR-6).
    # _validate_source_path (in emit_data_artifact_with_sources) REJECTS a bare
    # non-zarr directory with ValueError. Declare each contributing sub's
    # consolidated zarr store (a real .zarr dir that passes the gate) as the
    # provenance source — one per present sub.
    source_paths = [
        Path(sub.analysis_paths.analysis_dir) / "analysis_datatree.zarr"
        for sa_id, sub in subs.items()
        if _enabled_modes(sub)
    ]
    emit_data_artifact_with_sources(
        artifact_path=artifact_path,
        source_paths=source_paths,
        analysis_dir=Path(analysis.analysis_paths.analysis_dir),
        plot_id=plot_id,
    )

    verdict_path = eda_dir / f"{plot_id}.verdict.json"
    verdict_path.write_text(json.dumps(dataclasses.asdict(verdict), indent=2, default=str))

    return EdaResult(verdict=verdict, artifact_path=artifact_path, plot_id=plot_id)

config_diff_maps_figure_from_root(root)

Re-build the config-diff-maps figure from the carried sensitivity_datatree.zarr.

The scrollable doc is RE-RENDERED from the consolidated tree (NOT assembled from the saved standalone plots/eda/*.html fragments, which are full_html=True documents that cannot be concatenated). Delegates to the same builder the per-figure renderer uses so the doc figure matches the standalone artifact.

Source code in src/hhemt/eda/_report.py

def config_diff_maps_figure_from_root(root: Path) -> go.Figure:
    """Re-build the config-diff-maps figure from the carried sensitivity_datatree.zarr.

    The scrollable doc is RE-RENDERED from the consolidated tree (NOT assembled from
    the saved standalone plots/eda/*.html fragments, which are full_html=True documents
    that cannot be concatenated). Delegates to the same builder the per-figure renderer
    uses so the doc figure matches the standalone artifact.
    """
    from hhemt.eda._config_diff import build_config_diff_figure

    return build_config_diff_figure(root)

load_eda_context(root)

Re-derive the 9 EdaContext fields from artifacts resident at root.

Raises FileNotFoundError only for the two config files (a root without a config is not a valid EDA root). Every data field is None when its artifact is legitimately absent (non-sensitivity → no sensitivity_datatree; a bundle or partial tree → no datatree/scenario_status/swmm_features/triton_dem).

Source code in src/hhemt/eda/_context.py

def load_eda_context(root: Path | str) -> EdaContext:
    """Re-derive the 9 EdaContext fields from artifacts resident at ``root``.

    Raises ``FileNotFoundError`` only for the two config files (a root without a
    config is not a valid EDA root). Every data field is ``None`` when its artifact
    is legitimately absent (non-sensitivity → no ``sensitivity_datatree``; a bundle
    or partial tree → no ``datatree``/``scenario_status``/``swmm_features``/``triton_dem``).
    """
    import geopandas as gpd
    import pandas as pd
    import rioxarray as rxr
    import xarray as xr

    from hhemt.config.analysis import analysis_config
    from hhemt.config.loaders import yaml_to_model
    from hhemt.config.system import system_config

    root = Path(root)
    is_bundle = (root / "bundle_manifest.json").exists()

    cfg_analysis_path = root / "cfg_analysis.yaml"
    cfg_system_path = root / "cfg_system.yaml"
    if not cfg_analysis_path.exists():
        raise FileNotFoundError(
            f"{root} is not a valid EDA root: cfg_analysis.yaml absent. "
            f"load_eda_context expects an analysis_dir or a render-bundle root."
        )
    if not cfg_system_path.exists():
        raise FileNotFoundError(f"{root} is not a valid EDA root: cfg_system.yaml absent.")
    cfg_analysis = yaml_to_model(cfg_analysis_path, analysis_config)
    cfg_system = yaml_to_model(cfg_system_path, system_config)

    def _open_tree(p: Path) -> xr.DataTree | None:
        return xr.open_datatree(str(p), engine="zarr", consolidated=False) if p.exists() else None

    datatree = _open_tree(root / "analysis_datatree.zarr")
    sensitivity_datatree = (
        _open_tree(root / "sensitivity_datatree.zarr") if cfg_analysis.toggle_sensitivity_analysis else None
    )

    status_csv = root / "scenario_status.csv"
    scenario_status = pd.read_csv(status_csv) if status_csv.exists() else None

    # SWMM GIS layers are exported under system_dir/gis (system_overview.py:138),
    # NOT analysis_dir/gis. On a bundle root the harvested copy is at {root}/gis;
    # prefer the bundle copy, fall back to the system-dir source for a live analysis.
    bundle_gis = root / "gis"
    system_gis = Path(cfg_system.system_directory) / "gis"
    gis_dir = bundle_gis if (bundle_gis.is_dir() and any(bundle_gis.glob("*.gpkg"))) else system_gis
    swmm_features = (
        {p.stem: gpd.read_file(p) for p in sorted(gis_dir.glob("*.gpkg"))}
        if gis_dir.is_dir() and any(gis_dir.glob("*.gpkg"))
        else None
    )

    # The processed DEM is a SysPaths-derived artifact, NOT a cfg_system field:
    # system.py builds it as system_dir / f"elevation_{target_dem_resolution:.2f}m.dem".
    # On a bundle root the harvested copy is root-relative; prefer it when present.
    dem_name = f"elevation_{cfg_system.target_dem_resolution:.2f}m.dem"
    bundle_dem = root / dem_name
    system_dem = Path(cfg_system.system_directory) / dem_name
    dem_path = bundle_dem if bundle_dem.exists() else system_dem
    triton_dem = rxr.open_rasterio(dem_path) if dem_path.exists() else None

    # performance is a derived VIEW of the consolidated datatree (the */performance
    # nodes), never a separate artifact — None exactly when datatree is None or
    # carries no performance node.
    performance: xr.DataTree | None = None
    if datatree is not None:
        perf_nodes = [n for n in datatree.subtree if str(n.name) == "performance"]
        performance = datatree if perf_nodes else None

    return EdaContext(
        datatree=datatree,
        sensitivity_datatree=sensitivity_datatree,
        cfg_analysis=cfg_analysis,
        cfg_system=cfg_system,
        scenario_status=scenario_status,
        swmm_features=swmm_features,
        triton_dem=triton_dem,
        performance=performance,
        is_bundle=is_bundle,
    )

promote_eda_plot_to_static_config(plot_id, *, output_path=None, caption=None)

Emit a base StaticPlotBaseConfig YAML for an EDA plot (D-2/D-3 option a).

Populates ONLY backend-neutral base fields; matplotlib-specific colorbar/colormap fields stay at schema defaults (inert-but-harmless for a Plotly figure). NEVER a per-function subclass. output_format stays at the base default "pdf" (portable across matplotlib AND Plotly-Kaleido). Writes a REAL file (config/analysis.py::_check_static_plot_configs_exist raises on a missing path). output_path=None defaults to a cwd-relative promoted_static_configs/{plot_id}.yaml -- NEVER under analysis_dir/.

Source code in src/hhemt/eda/_promote.py

def promote_eda_plot_to_static_config(
    plot_id: str,
    *,
    output_path: Path | None = None,
    caption: str | None = None,
) -> Path:
    """Emit a base ``StaticPlotBaseConfig`` YAML for an EDA plot (D-2/D-3 option a).

    Populates ONLY backend-neutral base fields; matplotlib-specific colorbar/colormap
    fields stay at schema defaults (inert-but-harmless for a Plotly figure). NEVER a
    per-function subclass. ``output_format`` stays at the base default ``"pdf"``
    (portable across matplotlib AND Plotly-Kaleido). Writes a REAL file
    (``config/analysis.py::_check_static_plot_configs_exist`` raises on a missing
    path). ``output_path=None`` defaults to a cwd-relative
    ``promoted_static_configs/{plot_id}.yaml`` -- NEVER under ``analysis_dir/``.
    """
    if not _PLOT_ID_CHARSET.match(plot_id):
        raise ValueError(f"plot_id {plot_id!r} is not charset-safe; must match ^[A-Za-z0-9_.]+$ (ADR-2).")
    # renderer_kind is the ADR-2 plot-ID's leading '__'-segment (the renderer
    # module name), the same convention static_snakefile_generator uses. For a
    # promoted EDA plot it is not (yet) a key in STATIC_PLOT_CONFIG_REGISTRY —
    # that is expected per the header comment (EDA-plot-ID render dispatch is
    # deferred); the field is required so the emitted YAML validates round-trip.
    renderer_kind = plot_id.split("__", 1)[0]
    cfg = StaticPlotBaseConfig(plot_id=plot_id, renderer_kind=renderer_kind, caption=caption)
    if cfg.output_format not in _PLOTLY_PORTABLE_FORMATS:
        raise ValueError(
            f"output_format {cfg.output_format!r} is not producible by Plotly-Kaleido; "
            f"a promoted Plotly-sourced EDA plot must use one of {sorted(_PLOTLY_PORTABLE_FORMATS)}."
        )
    if output_path is None:
        output_path = Path.cwd() / "promoted_static_configs" / f"{plot_id}.yaml"
    output_path = Path(output_path)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    body = yaml.safe_dump(cfg.model_dump(mode="json"), sort_keys=False)
    output_path.write_text(_PROMOTED_HEADER + body)
    # lean (c): surface the paste-ready cfg_analysis line so the user can wire the
    # emitted static config into their analysis config without hunting for the field.
    print(
        f"[eda-promote] Static config for {plot_id!r} written to {output_path}.\n"
        f"[eda-promote] Add it to your analysis config to render it as a publication static plot:\n"
        f"[eda-promote]   static_plot_configs:\n"
        f"[eda-promote]     - {output_path}",
        flush=True,
    )
    return output_path

register_eda_plot_in_reporting_set(plot_id, set_name)

Record an EDA plot-ID's promotion intent against an ADR-5 ReportingSet (D-1 option c).

Register-ONLY: validates plot_id (ADR-2 charset) and set_name (must be a key in REPORTING_SETS), and returns a registration record the future report()-routing adapter (reporting-system_eda-skill) consumes. It does NOT mutate a live renderer_selection tuple (a live append KeyErrors the dispatcher pre-adapter) and does NOT author a report_renderers/ adapter, a workflow.py builder, or a dispatcher-map edit.

Source code in src/hhemt/eda/_promote.py

def register_eda_plot_in_reporting_set(plot_id: str, set_name: str) -> EdaReportingSetRegistration:
    """Record an EDA plot-ID's promotion intent against an ADR-5 ``ReportingSet`` (D-1 option c).

    Register-ONLY: validates ``plot_id`` (ADR-2 charset) and ``set_name`` (must be a
    key in ``REPORTING_SETS``), and returns a registration record the future
    ``report()``-routing adapter (``reporting-system_eda-skill``) consumes. It does
    NOT mutate a live ``renderer_selection`` tuple (a live append KeyErrors the
    dispatcher pre-adapter) and does NOT author a ``report_renderers/`` adapter, a
    ``workflow.py`` builder, or a dispatcher-map edit.
    """
    if not _PLOT_ID_CHARSET.match(plot_id):
        raise ValueError(f"plot_id {plot_id!r} is not charset-safe; must match ^[A-Za-z0-9_.]+$ (ADR-2).")
    if set_name not in REPORTING_SETS:
        raise ValueError(
            f"set_name {set_name!r} is not a registered ReportingSet; valid sets: {sorted(REPORTING_SETS)}."
        )
    # lean (d): diagnose whether the target set already renders this plot. R11
    # wired the compute-sensitivity set's eda_compute_sensitivity adapter, so a
    # registration against a set whose selection already carries that renderer IS
    # routed (config-selectable via report_config.reporting_set); other sets are
    # register-only (report()-routing deferred to reporting-system_eda-skill).
    _wired = any(sel.builder_key == "eda_compute_sensitivity" for sel in REPORTING_SETS[set_name].renderer_selection)
    if _wired:
        print(
            f"[eda-promote] {plot_id!r} registered against reporting set {set_name!r} — "
            f"this set already renders the EDA adapter; select it with "
            f"report_config.reporting_set={set_name!r}.",
            flush=True,
        )
    else:
        print(
            f"[eda-promote] {plot_id!r} registered against reporting set {set_name!r} — "
            f"NOT yet routed (this set has no EDA render adapter; report()-routing is "
            f"deferred to reporting-system_eda-skill).",
            flush=True,
        )
    return EdaReportingSetRegistration(
        plot_id=plot_id,
        set_name=set_name,
        routing="deferred:reporting-system_eda-skill",
    )

render_eda_plots(root, *, cfg_analysis, eda_cfg)

Render every plot in eda_cfg.enabled_plots to {root}/plots/eda/.

Returns the list of emitted HTML paths. Unknown renderer-kind keys raise ValueError (fail-fast at the facade boundary). root is the analysis_dir on the Analysis side and bundle.root on the Bundle side.

Source code in src/hhemt/eda/_plotting.py

def render_eda_plots(
    root: Path,
    *,
    cfg_analysis: analysis_config,
    eda_cfg: eda_config,
) -> list[Path]:
    """Render every plot in ``eda_cfg.enabled_plots`` to ``{root}/plots/eda/``.

    Returns the list of emitted HTML paths. Unknown renderer-kind keys raise
    ValueError (fail-fast at the facade boundary). ``root`` is the analysis_dir
    on the Analysis side and bundle.root on the Bundle side.
    """
    out: list[Path] = []
    for kind in eda_cfg.enabled_plots:
        renderer = _EDA_RENDERERS.get(kind)
        if renderer is None:
            raise ValueError(f"unknown EDA plot kind {kind!r}; known: {sorted(_EDA_RENDERERS)}")
        out.append(renderer(root, cfg_analysis=cfg_analysis, eda_cfg=eda_cfg))
    return out

Synthetic compute-config sensitivity experiment framework (entry point).

SINGLE SOURCE of the experiment matrix (F-B-1): this module owns the compute-config sweep enumeration — the fixed GPU/serial/openmp/hybrid rows plus the mpi-rank rows GENERATED from synthetic_experiment_config.rank_sweep (the default (2, 4, 8) reproduces the historical 28-row baseline byte-for-byte). scripts/experiments/_matrix_builder.py is retired; its sole importer (synth_compute_config.py) re-points at the write_clean_matrix_csv / write_resume_matrix_csv writers here. A src -> scripts import would break pip install -e ., so the enumeration lives in src/.

Public surface: experiment_matrix_rows(cfg) -> list[dict] (the shared row enumeration; the config _validate_caps guard also consumes it) build_experiment_matrix(cfg) -> pandas.DataFrame write_clean_matrix_csv(path, , rank_sweep=...) write_resume_matrix_csv(path, , runtime_min_by_sa=None, rank_sweep=..., ...) generate_synthetic_experiment(cfg, dest_dir) -> Path (build the synth case) size_resume_walltimes(clean_analysis) -> dict[str, int] (FQ3 two-pass helper)

build_experiment_matrix(cfg)

The partition-as-axis sensitivity matrix as a DataFrame (canonical columns).

Source code in src/hhemt/synthetic_experiment.py

def build_experiment_matrix(cfg: synthetic_experiment_config) -> pd.DataFrame:
    """The partition-as-axis sensitivity matrix as a DataFrame (canonical columns)."""
    return pd.DataFrame(experiment_matrix_rows(cfg), columns=_COLS)

experiment_matrix_rows(cfg)

The shared experiment-matrix row enumeration for cfg (clean walltime).

Consumed by build_experiment_matrix AND by the config's _validate_caps guard — a single enumeration so the validated matrix and the emitted matrix cannot drift.

Source code in src/hhemt/synthetic_experiment.py

def experiment_matrix_rows(cfg: synthetic_experiment_config) -> list[dict]:
    """The shared experiment-matrix row enumeration for ``cfg`` (clean walltime).

    Consumed by ``build_experiment_matrix`` AND by the config's ``_validate_caps``
    guard — a single enumeration so the validated matrix and the emitted matrix
    cannot drift.
    """
    return _rows(_configs(tuple(cfg.rank_sweep)), walltime_min=_CLEAN_WALLTIME_MIN)

generate_synthetic_experiment(cfg, dest_dir)

Build the synthetic TRITON-SWMM case for cfg under dest_dir and return the case directory.

Threads the four config model knobs (grid + forcing) into SyntheticModelParams and delegates to the lifted hhemt.synthetic_model.build_synthetic_case. The remaining SyntheticModelParams fields (event shaping — sim duration, compound surge, reporting cadence) retain their generic defaults in this Phase-1 first cut; promoting the full compound-event shaping to the config model is a separate follow-up (the production experiment's shaping currently lives in scripts/experiments/synth_compute_config.py::_EXPERIMENT_PARAMS).

Source code in src/hhemt/synthetic_experiment.py

def generate_synthetic_experiment(cfg: synthetic_experiment_config, dest_dir: Path) -> Path:
    """Build the synthetic TRITON-SWMM case for ``cfg`` under ``dest_dir`` and
    return the case directory.

    Threads the four config model knobs (grid + forcing) into
    ``SyntheticModelParams`` and delegates to the lifted
    ``hhemt.synthetic_model.build_synthetic_case``. The remaining
    ``SyntheticModelParams`` fields (event shaping — sim duration, compound
    surge, reporting cadence) retain their generic defaults in this Phase-1 first
    cut; promoting the full compound-event shaping to the config model is a
    separate follow-up (the production experiment's shaping currently lives in
    ``scripts/experiments/synth_compute_config.py::_EXPERIMENT_PARAMS``).
    """
    from hhemt.synthetic_model import SyntheticModelParams, build_synthetic_case

    params = SyntheticModelParams(
        n_cols=cfg.n_cols,
        n_rows=cfg.n_rows,
        cell_size_m=cfg.cell_size_m,
        rainfall_peak_mm_per_hr=cfg.rainfall_peak_mm_per_hr,
    )
    dest_dir = Path(dest_dir)
    build_synthetic_case(params, dest_dir)
    return dest_dir

size_resume_walltimes(clean_analysis)

Two-pass (FQ3): read each clean-sweep sa_id's full-completion wallclock (minutes) from the completed clean analysis, for feeding write_resume_matrix_csv(runtime_min_by_sa=...).

Source: df_status['perf_Total'] (cumulative wallclock in seconds -> /60), max over each sa's rows. On the CLEAN run this equals SLURM Elapsed because clean is never resumed (perf_Total only exceeds Elapsed when resumes occurred). Run AFTER the clean sweep has completed.

Source code in src/hhemt/synthetic_experiment.py

def size_resume_walltimes(clean_analysis) -> dict[str, int]:
    """Two-pass (FQ3): read each clean-sweep sa_id's full-completion wallclock
    (minutes) from the completed clean analysis, for feeding
    ``write_resume_matrix_csv(runtime_min_by_sa=...)``.

    Source: ``df_status['perf_Total']`` (cumulative wallclock in seconds -> /60),
    max over each sa's rows. On the CLEAN run this equals SLURM ``Elapsed`` because
    clean is never resumed (perf_Total only exceeds Elapsed when resumes occurred).
    Run AFTER the clean sweep has completed.
    """
    df = clean_analysis.df_status
    return (
        df.dropna(subset=["perf_Total"])
        .assign(_min=lambda d: d["perf_Total"] / 60.0)
        .groupby("sa_id")["_min"]
        .max()
        .round()
        .astype(int)
        .to_dict()
    )

write_clean_matrix_csv(path, *, rank_sweep=_DEFAULT_RANK_SWEEP)

Clean experiment CSV: generous walltime guaranteeing single-allocation completion. rank_sweep default reproduces the historical baseline.

Source code in src/hhemt/synthetic_experiment.py

def write_clean_matrix_csv(path: Path, *, rank_sweep: tuple[int, ...] = _DEFAULT_RANK_SWEEP) -> None:
    """Clean experiment CSV: generous walltime guaranteeing single-allocation
    completion. ``rank_sweep`` default reproduces the historical baseline."""
    df = pd.DataFrame(_rows(_configs(tuple(rank_sweep)), walltime_min=_CLEAN_WALLTIME_MIN), columns=_COLS)
    df.to_csv(path, index=False)

write_resume_matrix_csv(path, *, runtime_min_by_sa=None, rank_sweep=_DEFAULT_RANK_SWEEP, kill_divisor=3, min_walltime_min=1)

Resume sweep CSV: per-row walltime sized to force a mid-sim kill AND complete within restart-times from ONE analysis.run().

For each row, hpc_time_min_per_sim = max(min_walltime_min, round(T_sa / kill_divisor)) where T_sa is that sa's measured full-completion wallclock (minutes) from the CLEAN sweep, keyed by sa_id in runtime_min_by_sa. When runtime_min_by_sa is None (off-cluster dry-run only), fall back to a conservative GPU=4 min / CPU=18 min estimate by row type — REPLACE with real clean-sweep numbers (via size_resume_walltimes) before the production run.

Source code in src/hhemt/synthetic_experiment.py

def write_resume_matrix_csv(
    path: Path,
    *,
    runtime_min_by_sa: dict[str, float] | None = None,
    rank_sweep: tuple[int, ...] = _DEFAULT_RANK_SWEEP,
    kill_divisor: int = 3,
    min_walltime_min: int = 1,
) -> None:
    """Resume sweep CSV: per-row walltime sized to force a mid-sim kill AND
    complete within ``restart-times`` from ONE ``analysis.run()``.

    For each row, ``hpc_time_min_per_sim = max(min_walltime_min, round(T_sa /
    kill_divisor))`` where ``T_sa`` is that sa's measured full-completion wallclock
    (minutes) from the CLEAN sweep, keyed by ``sa_id`` in ``runtime_min_by_sa``.
    When ``runtime_min_by_sa`` is None (off-cluster dry-run only), fall back to a
    conservative GPU=4 min / CPU=18 min estimate by row type — REPLACE with real
    clean-sweep numbers (via ``size_resume_walltimes``) before the production run.
    """
    runtimes = runtime_min_by_sa or {}
    rows = _rows(_configs(tuple(rank_sweep)), walltime_min=None)
    for r in rows:
        t_full = runtimes.get(r["sa_id"], 4.0 if r["n_gpus"] else 18.0)
        r["hpc_time_min_per_sim"] = max(min_walltime_min, round(t_full / kill_divisor))
    pd.DataFrame(rows, columns=_COLS).to_csv(path, index=False)

hhemt version-migration system.

Forward-only migrations of persistent on-disk state (analysis trees and system directories), authored as numbered Python modules under versions/ and applied via the MigrationContext primitive DSL.

Public surface: LAYOUT_VERSION - current canonical layout version MINIMUM_SUPPORTED_VERSION - floor below which migration is refused run_migration, status, baseline, verify - high-level functions (also CLI) MigrationError - exception base

BaselineRequiredError

Bases: MigrationError

Raised when the detection ladder cannot infer a layout version and the user must explicitly baseline.

Source code in src/hhemt/version_migration/exceptions.py

class BaselineRequiredError(MigrationError):
    """Raised when the detection ladder cannot infer a layout version and
    the user must explicitly baseline."""

    def __init__(self, target_dir: Path) -> None:
        self.target_dir = target_dir
        super().__init__(
            f"cannot infer layout version for {target_dir}; "
            f"use `python -m hhemt.version_migration "
            f"baseline {target_dir} {{N}}`"
        )

LayoutVersionError

Bases: MigrationError

Raised when current/target version state is invalid.

Covers downgrade attempts, target below the floor, and applying a migration against an unexpected on-disk version.

Source code in src/hhemt/version_migration/exceptions.py

class LayoutVersionError(MigrationError):
    """Raised when current/target version state is invalid.

    Covers downgrade attempts, target below the floor, and applying a
    migration against an unexpected on-disk version.
    """

    def __init__(self, current: int, target: int, reason: str) -> None:
        self.current = current
        self.target = target
        self.reason = reason
        super().__init__(f"layout version error: current={current}, target={target}: {reason}")

MigrationConflictError

Bases: MigrationError

Raised mid-apply when a primitive cannot complete.

Covers filesystem conflicts, permission errors, and other runtime failures surfaced by a primitive.

Source code in src/hhemt/version_migration/exceptions.py

class MigrationConflictError(MigrationError):
    """Raised mid-apply when a primitive cannot complete.

    Covers filesystem conflicts, permission errors, and other runtime
    failures surfaced by a primitive.
    """

    def __init__(self, version: int, op_index: int, reason: str) -> None:
        self.version = version
        self.op_index = op_index
        self.reason = reason
        super().__init__(
            f"migration V{version:04d} failed at operation {op_index}; "
            f"partial state at layout_version={version - 1}: {reason}; "
            f"re-run --apply after resolving"
        )

MigrationError

Bases: TRITONSWMMError

Base for all version-migration errors.

Source code in src/hhemt/version_migration/exceptions.py

class MigrationError(TRITONSWMMError):
    """Base for all version-migration errors."""

RegistryError

Bases: MigrationError

Raised when migration discovery finds gaps, duplicates, or invalid modules.

Source code in src/hhemt/version_migration/exceptions.py

class RegistryError(MigrationError):
    """Raised when migration discovery finds gaps, duplicates, or invalid
    modules."""

    def __init__(self, reason: str, paths: list[Path] | None = None) -> None:
        self.reason = reason
        self.paths = paths or []
        super().__init__(f"registry error: {reason}; paths={self.paths}")

baseline(target_dir, version, *, force=False)

Stamp _version.json at version without running migrations.

Source code in src/hhemt/version_migration/runner.py

def baseline(target_dir: Path, version: int, *, force: bool = False) -> RunResult:
    """Stamp _version.json at ``version`` without running migrations."""
    if version < MINIMUM_SUPPORTED_VERSION or version > LAYOUT_VERSION:
        raise LayoutVersionError(
            current=-1,
            target=version,
            reason=(f"baseline {version} outside supported range [{MINIMUM_SUPPORTED_VERSION}, {LAYOUT_VERSION}]"),
        )
    existing = state.read_version_file(target_dir)
    if existing is not None and existing.layout_version != version and not force:
        raise LayoutVersionError(
            current=existing.layout_version,
            target=version,
            reason="refusing to baseline over existing _version.json without --force",
        )
    if existing is None:
        state.stamp_new_target(target_dir, version)
    else:
        if version < existing.layout_version:
            # --force path: dropping below current resets history (semantic
            # mismatch between layout_version and history would otherwise
            # be a logical inconsistency).
            new_state = state.VersionState(
                layout_version=version,
                toolkit_version=existing.toolkit_version,
                created_at=existing.created_at,
                migration_history=[],
            )
        else:
            new_state = _replace_layout(existing, version)
        state.write_version_file(target_dir, new_state)
    return RunResult(
        target_dir=target_dir,
        current_version=version,
        target_version=version,
        migrations_planned=[],
        migrations_applied=[],
        applied=True,
    )

run_migration(target_dir, *, target=None, apply=False, cfg_paths=None)

Plan and (optionally) apply the migration sequence to target_dir.

Renamed from migrate to disambiguate from the package name and CLI verb.

cfg_paths (optional) maps {"system": Path, "analysis": Path} and is threaded into every MigrationContext via _construct_context. Required by migrations whose upgrade() calls ctx.build_expected_slugs_for_current_version() (currently V0001). Ignored by migrations that do not.

Source code in src/hhemt/version_migration/runner.py

def run_migration(
    target_dir: Path,
    *,
    target: int | None = None,
    apply: bool = False,
    cfg_paths: dict[str, Path] | None = None,
) -> RunResult:
    """Plan and (optionally) apply the migration sequence to ``target_dir``.

    Renamed from ``migrate`` to disambiguate from the package name and CLI
    verb.

    ``cfg_paths`` (optional) maps {"system": Path, "analysis": Path} and is
    threaded into every MigrationContext via ``_construct_context``.
    Required by migrations whose ``upgrade()`` calls
    ``ctx.build_expected_slugs_for_current_version()`` (currently V0001).
    Ignored by migrations that do not.
    """
    target = LAYOUT_VERSION if target is None else target
    current = _resolve_current(target_dir)
    if current < MINIMUM_SUPPORTED_VERSION:
        raise LayoutVersionError(
            current=current,
            target=target,
            reason=(
                f"current layout_version={current} below "
                f"MINIMUM_SUPPORTED_VERSION={MINIMUM_SUPPORTED_VERSION}; "
                f"this analysis is too old to migrate"
            ),
        )
    plan_modules = registry.plan(current, target)
    if not plan_modules:
        logger.info(
            "[%s] current=%d, target=%d, no migrations needed",
            target_dir,
            current,
            target,
        )
        return RunResult(
            target_dir=target_dir,
            current_version=current,
            target_version=target,
            migrations_planned=[],
            migrations_applied=[],
            applied=apply,
        )

    # If _version.json is absent but detection inferred a version from
    # layout artifacts, stamp it before the first record_migration bumps it.
    # Dry-run path does not mutate disk.
    if apply and state.read_version_file(target_dir) is None:
        state.stamp_new_target(target_dir, current)

    applied: list[str] = []
    for m in plan_modules:
        ctx = _construct_context(target_dir, m, dry_run=not apply, cfg_paths=cfg_paths)
        try:
            m.upgrade(ctx)
        except (OSError, ValueError, KeyError, TypeError) as exc:
            # Narrow wrapping: OSError covers filesystem conflicts;
            # ValueError / KeyError cover primitive argument validation;
            # TypeError catches Pydantic model-class resolution failures.
            # Programming errors (AttributeError, NameError) propagate
            # as-is so they are not mistaken for retryable migration
            # conflicts.
            raise MigrationConflictError(
                version=m.version_to,
                op_index=len(ctx.plan),
                reason=str(exc),
            ) from exc
        for op in ctx.plan:
            print(f"[V{m.version_to:04d}] {op}", flush=True)
        if apply:
            ctx.execute()
            state.record_migration(target_dir, m.version_from, m.version_to, m.module_name)
            applied.append(m.module_name)
    return RunResult(
        target_dir=target_dir,
        current_version=current,
        target_version=target,
        migrations_planned=[m.module_name for m in plan_modules],
        migrations_applied=applied,
        applied=apply,
    )

status(target_dir)

Report current vs LAYOUT_VERSION; do not mutate.

Source code in src/hhemt/version_migration/runner.py

def status(target_dir: Path) -> RunResult:
    """Report current vs LAYOUT_VERSION; do not mutate."""
    current = _resolve_current(target_dir)
    plan_modules = registry.plan(current, LAYOUT_VERSION) if current >= 0 else []
    return RunResult(
        target_dir=target_dir,
        current_version=current,
        target_version=LAYOUT_VERSION,
        migrations_planned=[m.module_name for m in plan_modules],
        migrations_applied=[],
        applied=False,
    )

verify(target_dir)

Compare target_dir to schema/layout_schema_v{LAYOUT_VERSION}.json.

Phase 1 returns True if _version.json shows current LAYOUT_VERSION, False otherwise. Phase 5 expands this to full schema-walk comparison.

Source code in src/hhemt/version_migration/runner.py

def verify(target_dir: Path) -> bool:
    """Compare target_dir to schema/layout_schema_v{LAYOUT_VERSION}.json.

    Phase 1 returns True if _version.json shows current LAYOUT_VERSION,
    False otherwise. Phase 5 expands this to full schema-walk comparison.
    """
    st = state.read_version_file(target_dir)
    if st is None:
        return False
    return st.layout_version == LAYOUT_VERSION