feat(resolver,python): lazy package discovery via load_family callback (closes #86)

CHANGELOG.md

@@ -12,6 +12,37 @@ page.
 
 ## [Unreleased]
 
+### Added
+
+- **`load_family` callback** on `pyrer.solve()` — opt-in lazy package
+  discovery: pass `load_family: Callable[[str], list[PackageData]]` and the
+  solver calls it on demand the first time it needs a family it hasn't seen.
+  Each family is loaded at most once per solve; returning `[]` means "no
+  such family". Aimed at cold-cache / network-filesystem integrations
+  (Windows + CIFS in particular) where the up-front BFS of every reachable
+  family dominates the wall-clock cost of `rez env`. See the
+  [lazy-discovery section of the rez integration page](https://doubleailes.github.io/rer/docs/getting-started/rez-integration/#lazy-package-discovery-on-cold-caches).
+  Closes #86.
+- **`resolved_ephemerals`** on `pyrer.SolveResult` — list of rez-style
+  ephemeral requirement strings (e.g. `[".feature-1.5", ".mode-debug"]`)
+  surfaced from the solver, matching `rez.solver.Solver.resolved_ephemerals`.
+  Closes #84.
+- **Borrowing-iterator forms** on the Rust API: `Solver::resolved_packages_iter`
+  / `resolved_ephemerals_iter` and `ResolvePhase::iter_solved_variants` /
+  `iter_solved_ephemerals`. Avoid the intermediate `Vec` (and, for
+  ephemerals, the per-element `Requirement::clone`) when callers just want
+  to iterate.
+
+### Changed
+
+- **`PackageRepo` is now a struct**, not a `HashMap` type alias. Carries a
+  cache (`RefCell<HashMap<…>>`) and an optional `FamilyLoader` closure.
+  Construct with `PackageRepo::from_map(map)` for the eager case, or
+  `PackageRepo::with_loader(loader)` for lazy. `From<HashMap<…>>` is
+  implemented for back-compat with the old type-alias shape. The eager
+  path's perf is unchanged in measurement (within run-to-run noise of the
+  README baseline).
+
 ## [1.0.0] — TBD
 
 The first stable release. Public API is now under semver — see the

Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*"]
 resolver = "2"
 
 [workspace.package]
-version = "0.1.0-rc.7"
+version = "0.1.0-rc.8"
 authors = [
     "Lorenzo Montant <lo.montant.pro@gmail.com>",
     "Maxim Doucet <maxim.doucet@gmail.com>",
@@ -23,8 +23,8 @@ lazy_static = "1.5.0"
 rand = "0.8.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-rer-version = { path = "crates/rer-version", version = "0.1.0-rc.7" }
-rer-resolver = { path = "crates/rer-resolver", version = "0.1.0-rc.7" }
+rer-version = { path = "crates/rer-version", version = "0.1.0-rc.8" }
+rer-resolver = { path = "crates/rer-resolver", version = "0.1.0-rc.8" }
 pyo3 = { version = "0.23.5", features = ["extension-module"] }
 # `mimalloc` is wired into the bench binary as a `#[global_allocator]`.
 # Callgrind shows ~33 % of cycles in libc malloc/free; mimalloc has measurably

crates/examples/rez_benchmark_dataset.rs

@@ -8,9 +8,8 @@
 //! cargo run --release -p examples --example rez_benchmark_dataset
 //! ```
 
-use rer_resolver::rez_solver::{
-    make_shared_cache, PackageRepo, Requirement, Solver, SolverStatus,
-};
+use rer_resolver::rez_solver::{make_shared_cache, PackageRepo, Requirement, Solver, SolverStatus};
+use std::collections::HashMap;
 
 // Callgrind on this binary shows ~33 % of cycles in libc malloc/free —
 // `SmallVec` extends inside `Ranges`, per-call `FxHashMap`s in `reduce_by`,
@@ -57,13 +56,16 @@ fn percentile(sorted: &[f64], p: f64) -> f64 {
 }
 
 fn main() {
-    let repo: Rc<PackageRepo> = Rc::new(load_json("benchmark_packages.json"));
+    type RepoMap = HashMap<String, HashMap<String, rer_resolver::PackageData>>;
+    let repo_map: RepoMap = load_json("benchmark_packages.json");
+    let family_count = repo_map.len();
+    let repo: Rc<PackageRepo> = Rc::new(PackageRepo::from_map(repo_map));
     let cases: Vec<BenchmarkCase> = load_json("benchmark_expected.json");
 
     println!(
         "running {} requests against {} package families\n",
         cases.len(),
-        repo.len()
+        family_count
     );
 
     let mut times_ms: Vec<f64> = Vec::with_capacity(cases.len());

crates/rer-python/src/lib.rs

@@ -14,9 +14,10 @@ use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
 use pyo3::types::PyType;
 use rer_resolver::rez_solver::{
-    make_shared_cache, PackageRepo, Requirement, ScopeError, Solver, SolverStatus,
-    VariantSelectMode,
+    make_shared_cache, FamilyLoader, FamilyMap, PackageRepo, Requirement, ScopeError, Solver,
+    SolverStatus, VariantSelectMode,
 };
+use std::cell::RefCell;
 use std::collections::HashMap;
 use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::rc::Rc;
@@ -235,13 +236,13 @@ impl SolveResult {
 // Repository conversion
 // ---------------------------------------------------------------------------
 
-/// Fold a flat `list[PackageData]` into the `family → version → data` repo
+/// Fold a flat `list[PackageData]` into the `family → version → data` map
 /// shape the solver works on. Duplicates (same family + version) raise an
 /// `error` result rather than silently shadowing.
-fn packages_to_repo(packages: Vec<PackageData>) -> Result<PackageRepo, String> {
-    let mut repo: PackageRepo = HashMap::new();
+fn packages_to_map(packages: Vec<PackageData>) -> Result<HashMap<String, FamilyMap>, String> {
+    let mut map: HashMap<String, FamilyMap> = HashMap::new();
     for p in packages {
-        let entry = repo.entry(p.name.clone()).or_default();
+        let entry = map.entry(p.name.clone()).or_default();
         if entry
             .insert(
                 p.version.clone(),
@@ -255,7 +256,68 @@ fn packages_to_repo(packages: Vec<PackageData>) -> Result<PackageRepo, String> {
             return Err(format!("duplicate package: {}-{}", p.name, p.version));
         }
     }
-    Ok(repo)
+    Ok(map)
+}
+
+/// Build a [`FamilyLoader`] that calls the given Python callable for each
+/// family the solver hasn't yet seen, mirroring issue #86's lazy-discovery
+/// shape.
+///
+/// `load_err` is shared with the caller — if the Python callback raises,
+/// the loader stores the error there and returns an empty `Vec`, which the
+/// repo memoises as "no such family". The outer `solve()` checks the
+/// `RefCell` after the solver finishes and surfaces the captured error as
+/// a `"error"`-status `SolveResult`.
+///
+/// Entries whose `name` doesn't match the requested family are dropped
+/// defensively — a misbehaving loader can't poison the repo for unrelated
+/// families.
+fn make_loader(callback: Py<PyAny>, load_err: Rc<RefCell<Option<String>>>) -> FamilyLoader {
+    Box::new(
+        move |name: &str| -> Vec<(String, rer_resolver::PackageData)> {
+            // Already errored on a previous call — short-circuit so we don't
+            // pile up errors and don't keep calling a broken callback.
+            if load_err.borrow().is_some() {
+                return Vec::new();
+            }
+            let result: PyResult<Vec<(String, rer_resolver::PackageData)>> =
+                Python::with_gil(|py| -> PyResult<_> {
+                    let ret = callback.bind(py).call1((name,))?;
+                    let pkgs: Vec<PackageData> = ret.extract()?;
+                    let mut out: Vec<(String, rer_resolver::PackageData)> =
+                        Vec::with_capacity(pkgs.len());
+                    let mut seen_versions: std::collections::HashSet<String> =
+                        std::collections::HashSet::new();
+                    for p in pkgs {
+                        if p.name != name {
+                            continue;
+                        }
+                        if !seen_versions.insert(p.version.clone()) {
+                            return Err(PyValueError::new_err(format!(
+                                "load_family({:?}) returned duplicate version {:?}",
+                                name, p.version
+                            )));
+                        }
+                        out.push((
+                            p.version,
+                            rer_resolver::PackageData {
+                                requires: p.requires,
+                                variants: p.variants,
+                            },
+                        ));
+                    }
+                    Ok(out)
+                });
+            match result {
+                Ok(pairs) => pairs,
+                Err(err) => {
+                    let msg = Python::with_gil(|py| err.value(py).to_string());
+                    *load_err.borrow_mut() = Some(format!("load_family({name:?}) raised: {msg}"));
+                    Vec::new()
+                }
+            }
+        },
+    )
 }
 
 // ---------------------------------------------------------------------------
@@ -284,7 +346,12 @@ fn parse_variant_select_mode(s: &str) -> PyResult<VariantSelectMode> {
 /// * `packages` — a `list[PackageData]`, mirroring rez's already-loaded
 ///   packages. Construct each entry from a `rez.Package` (via
 ///   `rez.packages.iter_package_families` etc.) — `pyrer` does not read
-///   the filesystem itself.
+///   the filesystem itself. Optional if `load_family` is supplied.
+/// * `load_family` — Optional `Callable[[str], list[PackageData]]` invoked
+///   on demand the first time the solver needs a family that isn't already
+///   in `packages`. The result is cached for the lifetime of the solve,
+///   so each family is loaded at most once. An empty list means "no such
+///   family" and is treated the same as an absent family. See issue #86.
 /// * `variant_select_mode` — either `"version_priority"` (default, rez's
 ///   default config) or `"intersection_priority"`. Mirrors rez's
 ///   `config.variant_select_mode`.
@@ -299,14 +366,17 @@ fn parse_variant_select_mode(s: &str) -> PyResult<VariantSelectMode> {
 #[pyfunction]
 #[pyo3(
     signature = (
-        package_requests, packages, /,
+        package_requests, packages=None, /,
+        *,
+        load_family=None,
         variant_select_mode="version_priority",
         filters=None, max_iterations=None,
     )
 )]
 fn solve(
     package_requests: Vec<String>,
-    packages: Vec<PackageData>,
+    packages: Option<Vec<PackageData>>,
+    load_family: Option<Py<PyAny>>,
     variant_select_mode: &str,
     filters: Option<Vec<(String, String)>>,
     max_iterations: Option<u32>,
@@ -316,11 +386,27 @@ fn solve(
 
     let mode = parse_variant_select_mode(variant_select_mode)?;
 
-    let repo: PackageRepo = match packages_to_repo(packages) {
-        Ok(repo) => repo,
+    let initial_map = match packages_to_map(packages.unwrap_or_default()) {
+        Ok(map) => map,
         Err(msg) => return Ok(SolveResult::error(msg, start)),
     };
 
+    // Shared error slot for the loader. Populated if the Python callback
+    // raises; checked after the solver finishes to surface the failure.
+    let load_err: Rc<RefCell<Option<String>>> = Rc::new(RefCell::new(None));
+
+    let repo = if let Some(callback) = load_family {
+        let lazy = PackageRepo::with_loader(make_loader(callback, Rc::clone(&load_err)));
+        // Seed the eager set so the loader is never called for families
+        // the caller already supplied.
+        for (name, fam) in initial_map {
+            lazy.insert_family(name, fam);
+        }
+        lazy
+    } else {
+        PackageRepo::from_map(initial_map)
+    };
+
     // `Requirement::parse` panics on a syntactically invalid version range;
     // catch that at the FFI boundary and report it as `"error"` rather than
     // letting it surface as a Python `PanicException`.
@@ -329,12 +415,17 @@ fn solve(
             .iter()
             .map(|s| Requirement::parse(s))
             .collect();
-        let mut solver =
-            Solver::new_with_options(reqs, Rc::new(repo), make_shared_cache(), mode)?;
+        let mut solver = Solver::new_with_options(reqs, Rc::new(repo), make_shared_cache(), mode)?;
         solver.solve();
         Ok::<Solver, ScopeError>(solver)
     }));
 
+    // If the loader raised, that's the user-facing error — surface it
+    // before whatever fallback status the solver may have produced.
+    if let Some(msg) = load_err.borrow_mut().take() {
+        return Ok(SolveResult::error(msg, start));
+    }
+
     let solver = match outcome {
         Ok(Ok(solver)) => solver,
         // A missing top-level package family/version. rez reports this as a

crates/rer-resolver/benches/solver_micro.rs

@@ -141,7 +141,7 @@ fn fam(
 /// through the same cache code path the solver uses. Used by the slice-level
 /// benches below.
 fn slice_for(family: &str, range: &VersionRange) -> PackageVariantSlice {
-    let repo = Rc::new(build_repo());
+    let repo = Rc::new(rer_resolver::rez_solver::PackageRepo::from_map(build_repo()));
     let ctx = Rc::new(SolverContext::new(repo, RequirementList::new(vec![])));
     ctx.get_variant_slice(family, range)
         .expect("slice for the requested family/range")
@@ -300,7 +300,7 @@ fn bench_slice(c: &mut Criterion) {
 
 fn bench_solve(c: &mut Criterion) {
     let mut group = c.benchmark_group("Solver");
-    let repo = Rc::new(build_repo());
+    let repo = Rc::new(rer_resolver::rez_solver::PackageRepo::from_map(build_repo()));
     let cache = make_shared_cache();
 
     let cases: &[(&str, &[&str])] = &[