feat: json file format support for parameter input

Author: Elwardi

src/foambo/archive.py

@@ -143,8 +143,12 @@ def _resolve_trial_indices(include_trials: str | None, cfg, client) -> list[int]
     if include_trials == "all":
         trial_dest = cfg.get("optimization", {}).get("case_runner", {}).get("trial_destination", "trials")
         if os.path.isdir(trial_dest):
-            return sorted(int(d.split("_")[-1]) for d in os.listdir(trial_dest)
-                          if os.path.isdir(os.path.join(trial_dest, d)))
+            entries = os.listdir(trial_dest)
+            has_dirs = any(os.path.isdir(os.path.join(trial_dest, d)) for d in entries)
+            has_files = any(f.endswith(".json") and os.path.isfile(os.path.join(trial_dest, f)) for f in entries)
+            if has_dirs or has_files:
+                # Mix: prefer Ax trial index mapping to avoid fragile name parsing
+                return sorted(exp.trials.keys())
         return sorted(exp.trials.keys())
 
     if include_trials == "best":
@@ -318,28 +322,48 @@ def pack(config_path: str, include_trials: str | None = None,
                 if trial is None:
                     log.warning(f"Trial {tidx} not found in experiment")
                     continue
-                # Find trial folder
+                # Find trial folder OR caseless json file
                 trial_dir = None
+                trial_file = None
                 run_meta = trial.run_metadata or {}
                 case_path = run_meta.get("case_path") or run_meta.get("job", {}).get("case_path")
                 if case_path and os.path.isdir(case_path):
                     trial_dir = case_path
+                elif case_path and os.path.isfile(case_path):
+                    trial_file = case_path
                 else:
-                    # Try common naming patterns
+                    # Try common naming patterns (directory, then flat-file caseless)
+                    import glob
                     for pattern in [f"{exp_name}_trial_*_{tidx}", f"trial_{tidx}", f"*_trial_{tidx:04d}"]:
-                        import glob
                         matches = glob.glob(os.path.join(trial_destination, pattern))
                         if matches:
                             trial_dir = matches[0]
                             break
+                    if not trial_dir:
+                        for pattern in [f"{exp_name}_trial_*.json", f"trial_{tidx}.json"]:
+                            matches = glob.glob(os.path.join(trial_destination, pattern))
+                            if matches:
+                                trial_file = matches[0]
+                                break
                 if trial_dir and os.path.isdir(trial_dir):
                     tgrp = trials_grp.create_group(str(tidx))
                     tgrp.attrs["status"] = trial.status.name
                     tgrp.attrs["parameters"] = json.dumps(trial.arm.parameters if trial.arm else {})
                     n, sz = _store_directory(tgrp, trial_dir, all_skip)
                     print(f"  + trials/{tidx}/ ({n} files, {sz / 1024 / 1024:.1f} MB)")
+                elif trial_file and os.path.isfile(trial_file):
+                    tgrp = trials_grp.create_group(str(tidx))
+                    tgrp.attrs["status"] = trial.status.name
+                    tgrp.attrs["parameters"] = json.dumps(trial.arm.parameters if trial.arm else {})
+                    tgrp.attrs["caseless"] = True
+                    tgrp.attrs["filename"] = os.path.basename(trial_file)
+                    with open(trial_file, "rb") as tf:
+                        data = tf.read()
+                    ds = tgrp.create_dataset("__file__", data=data, **_compression())
+                    ds.attrs["mode"] = os.stat(trial_file).st_mode & 0o777
+                    print(f"  + trials/{tidx} (caseless file, {len(data)} bytes)")
                 else:
-                    log.warning(f"Trial {tidx} folder not found")
+                    log.warning(f"Trial {tidx} folder/file not found")
 
     archive_size = os.path.getsize(archive_path)
     print(f"\nArchive created: {archive_path} ({archive_size / 1024 / 1024:.1f} MB)")
@@ -413,10 +437,23 @@ def unpack(archive_path: str, output_dir: str | None = None) -> str:
         # Trials
         if "trials" in f:
             trials_dir = os.path.join(target, "trials")
+            os.makedirs(trials_dir, exist_ok=True)
             for tidx_str in f["trials"]:
-                tdir = os.path.join(trials_dir, f"{exp_name}_trial_{tidx_str}")
-                n = _extract_directory(f["trials"][tidx_str], tdir)
-                print(f"  + trials/{tidx_str}/ ({n} files)")
+                tgrp = f["trials"][tidx_str]
+                if tgrp.attrs.get("caseless", False):
+                    fname = tgrp.attrs.get("filename", f"{exp_name}_trial_{tidx_str}.json")
+                    if isinstance(fname, bytes):
+                        fname = fname.decode()
+                    out_path = os.path.join(trials_dir, fname)
+                    with open(out_path, "wb") as out:
+                        out.write(bytes(tgrp["__file__"][()]))
+                    mode = tgrp["__file__"].attrs.get("mode", 0o644)
+                    os.chmod(out_path, mode)
+                    print(f"  + trials/{fname} (caseless file)")
+                else:
+                    tdir = os.path.join(trials_dir, f"{exp_name}_trial_{tidx_str}")
+                    n = _extract_directory(tgrp, tdir)
+                    print(f"  + trials/{tidx_str}/ ({n} files)")
 
         # Rewrite config paths
         cfg = OmegaConf.load_from_string(config_bytes.decode())

src/foambo/common.py

@@ -2,8 +2,9 @@
 Common constants and shared definitions for the foambo package.
 """
 
-import hashlib, os, shutil, time
-from typing import List, Literal, Dict
+import hashlib, json, os, shutil, time
+from contextlib import contextmanager
+from typing import List, Literal, Dict, Optional
 from omegaconf import DictConfig, OmegaConf
 from foamlib import FoamCase, FoamFile
 from pydantic import BaseModel, ConfigDict
@@ -140,6 +141,56 @@ def assign_foam_path(foamfile, dotted_path, new_value):
     return foamfile[last]
 
 
+def _detect_format(path: str, explicit: Optional[str] = None) -> str:
+    """Pick file-format handler. Explicit wins. Else extension. Else openfoam."""
+    if explicit:
+        return explicit
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".json":
+        return "json"
+    if ext in (".yaml", ".yml"):
+        return "yaml"
+    return "openfoam"
+
+
+@contextmanager
+def _open_param_file(path: str, fmt: Optional[str] = None):
+    """Open a parameter file for dotted-path mutation across formats.
+
+    Yields a mapping-like object that supports __getitem__/__setitem__.
+    Writes back on exit for json/yaml (foamlib handles persistence itself).
+    """
+    resolved = _detect_format(path, fmt)
+    if resolved == "openfoam":
+        with FoamFile(path) as ff:
+            yield ff
+        return
+    if resolved == "json":
+        if os.path.exists(path):
+            with open(path, "r") as f:
+                data = json.load(f)
+        else:
+            data = {}
+        yield data
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+        return
+    if resolved == "yaml":
+        from ruamel.yaml import YAML
+        yaml = YAML()
+        yaml.preserve_quotes = True
+        if os.path.exists(path):
+            with open(path, "r") as f:
+                data = yaml.load(f) or {}
+        else:
+            data = {}
+        yield data
+        with open(path, "w") as f:
+            yaml.dump(data, f)
+        return
+    raise ValueError(f"Unknown parameter file format: {resolved}")
+
+
 class CasePreprocessor:
     """Protocol for case setup before trial execution.
 
@@ -171,18 +222,45 @@ def __str__(self):
 
 
 class NoCasePreprocessor(CasePreprocessor):
-    """No-op preprocessor for caseless (pure-Python) optimization.
+    """Caseless preprocessor: writes a single JSON file per trial.
 
-    Uses a virtual path as the trial "case path" — no directories are
-    created on disk since there are no files to store.
+    Output: `{trial_destination}/{EXPERIMENT_NAME}_trial_{hash}.json`
+    containing the full parameter dict. No case directory is created.
+    Any `variable_substitution` entries with empty/`.` file target this JSON.
     """
-    _counter: int = 0
-
     def setup(self, parameters: dict, cfg) -> dict:
-        NoCasePreprocessor._counter += 1
-        virtual_path = f"<caseless_trial_{NoCasePreprocessor._counter}>"
-        fake = _FakeCasePath(virtual_path)
-        return {"case": fake, "casename": virtual_path}
+        hash = hashlib.md5()
+        encoded = repr(OmegaConf.to_yaml(parameters)).encode()
+        hash.update(encoded + f"{time.time()}".encode())
+        hash = hash.hexdigest()[:DEFAULT_HASH_LENGTH]
+
+        trial_dest = cfg['template_case'].get('trial_destination', 'trials') \
+            if isinstance(cfg, (dict, DictConfig)) and 'template_case' in cfg \
+            else 'trials'
+        os.makedirs(trial_dest, exist_ok=True)
+        trial_name = f"{EXPERIMENT_NAME}_trial_{hash}"
+        trial_path = os.path.join(trial_dest, f"{trial_name}.json")
+
+        payload = dict(parameters)
+        with open(trial_path, "w") as f:
+            json.dump(payload, f, indent=2)
+
+        # Apply variable_substitution entries targeting the trial JSON
+        if isinstance(cfg, (dict, DictConfig)) and 'templating' in cfg:
+            for elmt in cfg['templating'].get('variables', []) or []:
+                tgt = (elmt.get('file') or '').strip()
+                if tgt in ('', '.', '/'):
+                    # Entry targets the trial JSON itself — already written as flat dict.
+                    # Dotted paths in parameter_scopes allow nested rewrites.
+                    elmv = elmt['parameter_scopes']
+                    with _open_param_file(trial_path, 'json') as pf:
+                        for p in elmv:
+                            if p in parameters:
+                                assign_foam_path(pf, elmv[p], parameters[p])
+
+        fake = _FakeCasePath(trial_path)
+        fake.name = trial_name
+        return {"case": fake, "casename": trial_path}
 
 
 # Module-level default preprocessor; can be swapped by users
@@ -255,7 +333,7 @@ def prepare_case(
             elm = elmt['file']
             elmv = elmt['parameter_scopes']
             param_file_path = os.path.join(case.path, elm.lstrip('/'))
-            with FoamFile(param_file_path) as paramFile:
+            with _open_param_file(param_file_path, elmt.get('format')) as paramFile:
                 for param in elmv:
                     try:
                         if param in parameters.keys():

src/foambo/orchestrate.py

@@ -9,7 +9,7 @@
 from .common import *
 from .common import FoamBOBaseModel
 from pydantic import Field, field_validator, model_validator
-from typing import Any, Iterable, List, Literal, Dict
+from typing import Any, Iterable, List, Literal, Dict, Optional
 from functools import reduce
 from ax.global_stopping.strategies import BaseGlobalStoppingStrategy, ImprovementGlobalStoppingStrategy
 from ax.api.configs import StorageConfig, RangeParameterConfig, ChoiceParameterConfig
@@ -835,11 +835,15 @@ def set_generation_strategy(self, client):
 
 
 class VariableSubstOptions(FoamBOBaseModel):
-    """Maps parameters to OpenFOAM file fields for value substitution via foamlib."""
-    file: str = Field(description="Relative path to the OpenFOAM file (e.g. `/0orig/field`)",
+    """Maps parameters to file fields (OpenFOAM dict, JSON, or YAML) for value substitution."""
+    file: str = Field(description="Relative path to the target file (e.g. `/0orig/field`, `/params.json`). Empty/`.` in caseless mode targets the trial's auto-generated JSON.",
                        examples=["/0orig/field"])
     parameter_scopes: Dict[str, str] = Field(description="Mapping of parameter name to dotted path in the file (e.g. `x: someDict.x`)",
                                               examples=[{"x": "someDict.x"}])
+    format: Optional[Literal["openfoam", "json", "yaml"]] = Field(
+        default=None,
+        description="File format. `None` (default) = openfoam (backward-compat). Set to `json` or `yaml` for non-OpenFOAM files.",
+        examples=["openfoam"])
 
 
 class FileSubstOptions(FoamBOBaseModel):