fix: robust + multi-fidelity play well with each other

Author: Elwardi

src/foambo/api_server.py

@@ -1889,6 +1889,35 @@ def get_status():
             "context_samples": rc.get("context_samples", 10),
         }
 
+    # Multi-fidelity info
+    mf_info = None
+    if raw_cfg:
+        params = raw_cfg.get("experiment", {}).get("parameters", [])
+        fid_params = [p for p in params if (p.get("is_fidelity") if hasattr(p, "get") else getattr(p, "is_fidelity", False))]
+        if fid_params:
+            fp = fid_params[0]
+            fid_name = fp.get("name", "") if hasattr(fp, "get") else getattr(fp, "name", "")
+            fid_target = fp.get("target_value") if hasattr(fp, "get") else getattr(fp, "target_value", None)
+            fid_values = fp.get("values") if hasattr(fp, "get") else getattr(fp, "values", None)
+            cost_metrics = [m.get("name", "") if hasattr(m, "get") else getattr(m, "name", "")
+                           for m in raw_cfg.get("optimization", {}).get("metrics", [])
+                           if (m.get("is_cost") if hasattr(m, "get") else getattr(m, "is_cost", False))]
+            # Count trials per fidelity level
+            fid_counts = {}
+            if client is not None:
+                for t in client._experiment.trials.values():
+                    if t.arm:
+                        fv = t.arm.parameters.get(fid_name)
+                        if fv is not None:
+                            fid_counts[str(fv)] = fid_counts.get(str(fv), 0) + 1
+            mf_info = {
+                "fidelity_param": fid_name,
+                "target_value": fid_target,
+                "levels": fid_values or [],
+                "cost_metric": cost_metrics[0] if cost_metrics else None,
+                "trials_per_fidelity": fid_counts,
+            }
+
     return SafeJSONResponse(content={
         "running": True,
         "uptime_s": round(time.time() - _state.start_time, 1),
@@ -1899,6 +1928,7 @@ def get_status():
         "model_fitted": has_model,
         "timing": getattr(_state, '_timing', None),
         "robust": robust_info,
+        "multi_fidelity": mf_info,
     })
 
 

src/foambo/common.py

@@ -321,9 +321,9 @@ def prepare_case(
     if "files" in cfg['templating'].keys() and cfg['templating']['files']:
         for entry in cfg['templating']['files']:
             param_name = entry['parameter']
-            template_path = entry['file_path']
+            template_path = entry['file_path'].lstrip('/')
             if param_name in parameters:
-                shutil.copyfile(
+                shutil.copy2(
                     os.path.join(case.path, template_path + "." + parameters[param_name]),
                     os.path.join(case.path, template_path)
                 )

src/foambo/default_config.py

@@ -852,6 +852,114 @@ def __init__(self, **kwargs):
             bootstrap loader clears that flag automatically before mutating.
         """).strip(),
     }
+    harvested["experiment.parameters[].is_fidelity"] = {
+        "category": "Config",
+        "content": textwrap.dedent("""
+            Mark a parameter as the fidelity dimension for multi-fidelity BO.
+            The parameter represents evaluation quality — e.g. 0 = cheap
+            meanline, 1 = expensive CFD. Exactly one parameter should have
+            this flag. ``target_value`` is the high-fidelity level the
+            optimizer ultimately cares about.
+
+            ```yaml
+            experiment:
+              parameters:
+                - name: fidelity
+                  parameter_type: float
+                  bounds: [0.0, 1.0]
+                  is_fidelity: true
+                  target_value: 1.0
+            ```
+
+            When detected, foamBO auto-selects ``SingleTaskMultiFidelityGP``
+            as the surrogate and extracts ``target_fidelities`` for the
+            acquisition function. The runner should branch on the fidelity
+            value to dispatch cheap vs expensive evaluations.
+        """).strip(),
+    }
+
+    harvested["optimization.metrics[].is_cost"] = {
+        "category": "Config",
+        "content": textwrap.dedent("""
+            Mark a metric as the cost signal for multi-fidelity acquisition.
+            The metric value should represent actual execution cost (e.g.
+            wall-clock seconds) and must be emitted by the runner at every
+            fidelity level.
+
+            ```yaml
+            optimization:
+              metrics:
+                - name: executionTime
+                  command: ["scripts/metric.sh", "executionTime"]
+                  is_cost: true
+            ```
+
+            Exactly one metric should have ``is_cost: true``. foamBO learns
+            per-fidelity mean cost from observed values and updates the
+            acquisition function's cost model each callback cycle. Without
+            an ``is_cost`` metric, MF acquisition uses uniform cost
+            (no cost-aware fidelity selection).
+        """).strip(),
+    }
+
+    harvested["trial_generation.generation_nodes[].generator_specs[].model_kwargs.botorch_acqf_class"] = {
+        "category": "Config",
+        "content": textwrap.dedent("""
+            Override the default BoTorch acquisition function class. Accepts
+            a string class name that foamBO resolves to the actual Python
+            class. Supported MF acquisition functions:
+
+            ``qMultiFidelityHypervolumeKnowledgeGradient`` (recommended):
+            - Multi-objective, cost-aware, one-step lookahead.
+            - Maximizes expected hypervolume improvement at target fidelity.
+            - Cost model auto-wired from ``is_cost`` metric via
+              ``cost_intercept`` + ``fidelity_weights``.
+
+            ``MOMF`` (alternative):
+            - Multi-objective multi-fidelity via fidelity pseudo-objective.
+            - Faster candidate generation, less sample efficient.
+            - Requires manual ``cost_call`` in ``botorch_acqf_options``.
+
+            ```yaml
+            trial_generation:
+              method: custom
+              generation_nodes:
+                - node_name: MF
+                  generator_specs:
+                    - generator_enum: BOTORCH_MODULAR
+                      model_kwargs:
+                        botorch_acqf_class: "qMultiFidelityHypervolumeKnowledgeGradient"
+            ```
+
+            With ``method: fast``, setting ``botorch_acqf_class`` is not
+            needed — foamBO auto-selects ``qMultiFidelityHypervolumeKnowledgeGradient``
+            when an ``is_fidelity`` parameter is detected. Custom generation
+            nodes are only needed to override the default (e.g. pick MOMF).
+
+            **Runner dispatch (recommended):** use ``file_substitution`` with
+            a string ``ChoiceParameter`` fidelity. Place ``Allrun.meanline``
+            + ``Allrun.CFD`` in the template case:
+            ```yaml
+            experiment:
+              parameters:
+                - name: fidelity
+                  values: ["meanline", "CFD"]
+                  parameter_type: str
+                  is_fidelity: true
+                  target_value: "CFD"
+
+            optimization:
+              case_runner:
+                file_substitution:
+                  - parameter: fidelity
+                    file_path: /Allrun
+            ```
+
+            Other resolvable names: ``qMultiFidelityKnowledgeGradient``
+            (single-objective MF), ``qMultiFidelityMaxValueEntropy``.
+        """).strip(),
+    }
+
     harvested["dashboard"] = {
         "category": "Dashboard",
         "content": textwrap.dedent("""

src/foambo/docs_concepts.py

@@ -624,4 +624,157 @@
 it does not (or only changes via ``specialize``).""",
     },
 
+    "concept.multi_fidelity": {
+        "category": "Concept",
+        "content": """\
+Multi-fidelity optimization — cheap approximations guide expensive evaluations.
+
+When a fast surrogate (eg. coarse mesh) correlates with the
+expensive truth (fine-mesh CFD), multi-fidelity BO spends most of its budget
+on cheap queries and reserves expensive evaluations for the most promising
+candidates. The GP learns the bias between fidelity levels and the
+acquisition function trades information gain against evaluation cost.
+
+**Setup in foamBO:**
+
+1. Add a fidelity parameter with ``is_fidelity: true`` and ``target_value``:
+```yaml
+experiment:
+  parameters:
+    - name: fidelity
+      parameter_type: str
+      values: ["coarse", "fine"]
+      is_fidelity: true
+      target_value: "fine"
+```
+
+2. Mark one metric as the cost signal with ``is_cost: true``:
+```yaml
+optimization:
+  metrics:
+    - name: executionTime
+      command: ["scripts/metric.sh", "executionTime"]
+      is_cost: true
+```
+
+3. Use ``method: fast`` (or any generation strategy — MF is auto-wired):
+```yaml
+trial_generation:
+  method: fast
+```
+
+That's it. No custom generation nodes required for MF.
+
+**Auto-wiring:** when ``is_fidelity`` is detected, foamBO automatically:
+- Selects ``qMultiFidelityHypervolumeKnowledgeGradient`` as the acqf.
+- Sets ``SingleTaskMultiFidelityGP`` as the surrogate.
+- Extracts ``target_fidelities`` from the search space (Ax built-in).
+- Learns ``cost_intercept`` and ``fidelity_weights`` from observed
+  ``is_cost`` metric data each callback cycle.
+
+To override the acqf (e.g. use MOMF instead), use a custom generation node:
+```yaml
+trial_generation:
+  method: custom
+  generation_nodes:
+    - node_name: MF
+      generator_specs:
+        - generator_enum: BOTORCH_MODULAR
+          model_kwargs:
+            botorch_acqf_class: "MOMF"
+```
+
+**Two acquisition functions are supported:**
+
+``qMultiFidelityHypervolumeKnowledgeGradient`` (qMF-HVKG):
+- Multi-objective, cost-aware, one-step lookahead.
+- Maximizes expected hypervolume improvement at target fidelity.
+- Better sample efficiency; slower candidate generation.
+- Takes ``cost_intercept`` + ``fidelity_weights`` via input constructor;
+  foamBO auto-derives these from the ``is_cost`` metric.
+- **Recommended for most use cases** (trial cost dominates gen time).
+
+``MOMF`` (Multi-Objective Multi-Fidelity):
+- Adds fidelity as a pseudo-objective (trust reward for higher fidelity).
+- Faster candidate generation; less sample efficient.
+- Takes ``cost_call`` (a callable) directly.
+- foamBO does NOT auto-wire cost for MOMF; pass ``cost_call`` manually
+  via ``botorch_acqf_options`` if needed.
+
+**Runner dispatch** via ``file_substitution`` (recommended):
+Use a string ``ChoiceParameter`` for fidelity and foamBO's built-in file
+substitution to swap the runner script (or a portion of it) per fidelity level. Place
+``Allrun.coarse`` and ``Allrun.fine`` in the template case:
+```yaml
+optimization:
+  case_runner:
+    file_substitution:
+      - parameter: fidelity
+        file_path: /Allrun
+```
+When ``fidelity=coarse``, the runner copies ``Allrun.coarse`` →
+``Allrun`` before execution. When ``fidelity=fine``, copies
+``Allrun.fine`` → ``Allrun``. No if/else branching in scripts needed.
+
+**Alternative** (continuous fidelity via env var):
+```bash
+if [ "$FIDELITY" = "0" ] || [ "$FIDELITY" = "0.0" ]; then
+    ./Allrun.coarse
+else
+    ./Allrun.fine
+fi
+```
+
+**Cost model evolution:** before any trials complete, uniform cost is
+assumed (cost ratio = 1). As ``is_cost`` data arrives, foamBO recomputes
+per-fidelity mean cost and updates ``cost_intercept`` / ``fidelity_weights``
+each callback — no restart needed.
+
+**Cost scaling warning:** the ``is_cost`` metric should emit *scaled*
+costs, not raw wall-clock seconds. With extreme cost ratios (e.g. coarse
+1s vs fine 3600s = 1:3600), the acquisition function may defer expensive
+evaluations indefinitely — the info-gain-per-cost ratio always favors
+cheap queries when the denominator is 3600× larger.
+
+**Recommendation:** cap the effective ratio to 1:50–1:100 by emitting
+a normalized cost. Use the baseline trial's execution time as the
+reference scale:
+
+- Coarse fidelity metric script: ``echo 1``
+- Fine fidelity metric script: ``echo 50``  (not the raw wall time)
+
+This tells the optimizer "fine is 50× more expensive" — enough to prefer
+coarse for exploration, but not so extreme that fine is never selected.
+Tune the ratio based on how many fine-fidelity evaluations you can afford
+in your budget. A ratio of 1:N means roughly 1 fine trial per N coarse
+trials.
+
+Also consider seeding 3–5 initial trials at target fidelity (via SOBOL
+init phase with ``fixed_features``) so the GP has fine-fidelity signal
+from the start to estimate the coarse→fine bias.
+
+**Composition with robust mode:** fully automatic. When both
+``is_fidelity`` and ``robust_optimization`` are present, foamBO composes
+them into a single acquisition loop:
+
+- **Surrogate**: ``SingleTaskMultiFidelityGP`` (handles fidelity kernel)
+  with ``SubstituteContextFeatures`` (handles context fan-out) as chained
+  input transforms on the same model.
+- **Acquisition**: ``qMultiFidelityHypervolumeKnowledgeGradient`` with a
+  ``RobustMCObjective`` (MARS or CVaR) passed as the ``objective``.
+- **Per-candidate evaluation**: qMFHVKG proposes ``(design, fidelity)``,
+  the GP posterior fans to K context points via SubstituteContextFeatures,
+  the risk-measure objective reduces K contexts to risk-adjusted values,
+  and qMFHVKG computes cost-aware HV improvement at target fidelity.
+
+No extra YAML is needed — setting ``is_fidelity`` on a parameter alongside
+``robust_optimization`` triggers composition automatically. The cost model
+(``is_cost`` metric) works identically in the composed path.
+
+**Staged fallback** is still available for simpler workflows: MF BO at
+nominal context first, then robust verification via ``bootstrap``.
+
+See also: ``concept.robust_optimization``, ``concept.bootstrap_and_specialize``.""",
+    },
+
 }

src/foambo/metrics.py

@@ -315,6 +315,14 @@ class LocalJobMetric(FoamBOBaseModel):
         "Use this when optimization parameters affect simulation speed (e.g. AMR/mesh refinement), "
         "so trials are compared at the same physical time rather than the same poll count."
     ))
+    is_cost: bool = Field(default=False, description=(
+        "Mark this metric as the cost signal for multi-fidelity acquisition (MOMF). "
+        "Exactly one metric must have is_cost=True when using a cost-aware MF acqf. "
+        "The metric value should represent actual execution cost (e.g. wall-clock "
+        "seconds) and must be emitted at every fidelity level. foamBO builds a "
+        "per-fidelity mean-cost lookup from observed values and passes it as "
+        "cost_call to the acquisition function."
+    ))
 
     def to_metric(self):
         cfg = DictConfig({
@@ -419,17 +427,18 @@ def streaming_metric(client: Client, opt_cfg: Dict):
                 metric["progress"]
                     .replace("$FOAMBO_STEP", str(trial_progression_step[trial_idx][metric["name"]]))
                     .replace("$STEP", str(trial_progression_step[trial_idx][metric["name"]]))
-                if "progress" in metric and isinstance(metric["progress"], str)
+                if "progress" in metric and isinstance(metric.get("progress"), str)
                 else [
                     p.replace("$FOAMBO_STEP", str(trial_progression_step[trial_idx][metric["name"]]))
                      .replace("$STEP", str(trial_progression_step[trial_idx][metric["name"]]))
                     if isinstance(p, str) else p
-                    for p in metric["progress"]
+                    for p in metric.get("progress") or []
                 ]
             )
         }
         for metric in metrics_cfg
         if metric["name"] not in objective_names
+        and metric.get("progress") and metric.get("progress") != [] and metric.get("progress") != ""
     ]
     # If we have no eligible progress items (command or callable), skip this
     has_progress_cmd = any(cfg.get("progress") and cfg["progress"] != "" and cfg["progress"] != "none"