fix(native): backfill silently-dropped files via WASM for engine parity (#970)

* fix(native): backfill silently-dropped files via WASM for engine parity

The native engine's `parse_files_parallel` (Rust) uses `filter_map` with the
`?` operator, which silently drops any file whose read/parse/extract step
fails — including files in languages where the installed native addon lacks
an extractor (e.g. HCL/Scala/Swift on v3.9.2). WASM handles these languages
correctly, so benchmarks showed 668 native vs 728 WASM file nodes on the
same tree (#967).

Fix the parity gap in two places:

- `parseFilesAuto`: track which files the native parser actually returned;
  re-parse the rest with WASM so the JS pipeline gets complete coverage.
- `tryNativeOrchestrator`: after the native Rust orchestrator writes to the
  DB, diff `collectFiles()` against the `kind='file'` rows and backfill any
  missing files via a WASM pass + `batchInsertNodes`.

Both paths warn explicitly when a drop happens so the underlying Rust
regression stays visible. This complements the Rust-side fix (adding
HCL/Scala/Swift extractors in newer binaries) by protecting users who
still have older native addons installed.

* fix: address review feedback on native backfill (#970)

- Scope backfill/warn to installed WASM grammars. Extract
  getInstalledWasmExtensions() helper so the parity check in
  parseFilesAuto and the post-orchestrator backfill both ignore
  languages neither engine can parse (keeps warn count meaningful).
- Populate qualified_name and scope on backfilled symbol nodes so
  cross-file reference resolution and go-to-definition queries can
  find them (matches insertDefinitionsAndExports).
- Only run the post-orchestrator filesystem scan on full builds.
  Incremental builds parse through parseFilesAuto, which already has
  its own per-file backfill, so the extra scan was wasted work.
- Fix import ordering flagged by biome.

Impact: 4 functions changed, 14 affected

* fix(native): backfill exported symbols in WASM-dropped-files path (#970)

The engine-parity backfill in `backfillNativeDroppedFiles` was only
inserting `symbols.definitions` and skipping `symbols.exports`. Queries
filtering on `exported = 1` (e.g. dead-export detection, reverse-dep
walks, "exports of file") would miss symbols from backfilled files —
HCL/Scala/Swift on stale native addons.

Mirror the reference flow in
`src/domain/graph/builder/stages/insert-nodes.ts:insertDefinitionsAndExports`:

- Iterate `symbols.exports ?? []`, push an `INSERT OR IGNORE` row (no-op
  if a definition row for the same name/kind/line already exists), and
  collect the (name, kind, file, line) key for a second pass.
- After `batchInsertNodes`, run chunked `UPDATE nodes SET exported = 1
  WHERE …` statements (chunk size 500 with a prepared-statement cache)
  so backfilled exports are discoverable.

Also fix a smaller parity gap in the file row: `qualified_name` was
`relPath`; the reference flow uses `NULL`. Matches the canonical insert
so `kind='file'` rows are shape-identical across both paths.

Addresses Greptile P1 (comment 3107870224) on PR #970.

Impact: 1 functions changed, 4 affected

Author: carlos-alm

src/domain/graph/builder/pipeline.ts

@@ -21,6 +21,7 @@ import { detectWorkspaces, loadConfig } from '../../../infrastructure/config.js'
 import { debug, info, warn } from '../../../infrastructure/logger.js';
 import { loadNative } from '../../../infrastructure/native.js';
 import { semverCompare } from '../../../infrastructure/update-check.js';
+import { normalizePath } from '../../../shared/constants.js';
 import { toErrorMessage } from '../../../shared/errors.js';
 import { CODEGRAPH_VERSION } from '../../../shared/version.js';
 import type {
@@ -29,11 +30,12 @@ import type {
   BuildResult,
   Definition,
   ExtractorOutput,
+  SqliteStatement,
 } from '../../../types.js';
-import { getActiveEngine } from '../../parser.js';
+import { getActiveEngine, getInstalledWasmExtensions, parseFilesAuto } from '../../parser.js';
 import { setWorkspaces } from '../resolve.js';
 import { PipelineContext } from './context.js';
-import { loadPathAliases } from './helpers.js';
+import { batchInsertNodes, collectFiles as collectFilesUtil, loadPathAliases } from './helpers.js';
 import { NativeDbProxy } from './native-db-proxy.js';
 import { buildEdges } from './stages/build-edges.js';
 import { buildStructure } from './stages/build-structure.js';
@@ -696,10 +698,123 @@ async function tryNativeOrchestrator(
     }
   }
 
+  // Engine parity: the native orchestrator silently drops files whose
+  // Rust extractor/grammar is missing or fails (e.g. HCL, Scala, Swift on
+  // stale native binaries). WASM handles those — backfill via WASM so both
+  // engines process the same file set (#967).
+  //
+  // Only runs on full builds: incremental builds only touch changed files,
+  // which are parsed through parseFilesAuto (which has its own per-file
+  // backfill), so a full filesystem scan here would be wasted work.
+  if (result.isFullBuild) {
+    await backfillNativeDroppedFiles(ctx);
+  }
+
   closeDbPair({ db: ctx.db, nativeDb: ctx.nativeDb });
   return formatNativeTimingResult(p, structurePatchMs, analysisTiming);
 }
 
+/**
+ * Backfill files that the native orchestrator silently dropped during parse.
+ * Falls back to WASM + inserts file/symbol nodes so engine counts match (#967).
+ */
+async function backfillNativeDroppedFiles(ctx: PipelineContext): Promise<void> {
+  // Needs a real better-sqlite3 connection for INSERT.
+  if (ctx.nativeFirstProxy) {
+    closeNativeDb(ctx, 'pre-parity-backfill');
+    ctx.db = openDb(ctx.dbPath);
+    ctx.nativeFirstProxy = false;
+  }
+
+  const collected = collectFilesUtil(ctx.rootDir, [], ctx.config, new Set<string>());
+  const expected = new Set(
+    collected.files.map((f) => normalizePath(path.relative(ctx.rootDir, f))),
+  );
+
+  const existingRows = ctx.db
+    .prepare("SELECT DISTINCT file FROM nodes WHERE kind = 'file'")
+    .all() as Array<{ file: string }>;
+  const existing = new Set(existingRows.map((r) => r.file));
+
+  // Restrict backfill to files with an installed WASM grammar. Extensions in
+  // LANGUAGE_REGISTRY without a shipped grammar file (e.g. groovy, erlang on
+  // minimal installs) can't be parsed by either engine, so they're not a
+  // native regression — excluding them keeps the warn count meaningful.
+  const installedExts = getInstalledWasmExtensions();
+  const missingAbs: string[] = [];
+  for (const rel of expected) {
+    if (existing.has(rel)) continue;
+    const ext = path.extname(rel).toLowerCase();
+    if (!installedExts.has(ext)) continue;
+    missingAbs.push(path.join(ctx.rootDir, rel));
+  }
+  if (missingAbs.length === 0) return;
+
+  warn(
+    `Native orchestrator dropped ${missingAbs.length} file(s); backfilling via WASM for engine parity`,
+  );
+  const wasmResults = await parseFilesAuto(missingAbs, ctx.rootDir, { engine: 'wasm' });
+
+  const rows: unknown[][] = [];
+  const exportKeys: unknown[][] = [];
+  for (const [relPath, symbols] of wasmResults) {
+    // File row — mirrors insertDefinitionsAndExports: qualified_name is null.
+    rows.push([relPath, 'file', relPath, 0, null, null, null, null, null]);
+    for (const def of symbols.definitions ?? []) {
+      // Populate qualified_name/scope the same way the JS fallback does so
+      // downstream queries (cross-file references, "go to definition") find
+      // these symbols.
+      const dotIdx = def.name.lastIndexOf('.');
+      const scope = dotIdx !== -1 ? def.name.slice(0, dotIdx) : null;
+      rows.push([
+        def.name,
+        def.kind,
+        relPath,
+        def.line,
+        def.endLine ?? null,
+        null,
+        def.name,
+        scope,
+        def.visibility ?? null,
+      ]);
+    }
+    // Exports: insert the row (INSERT OR IGNORE — a matching definition row
+    // is a no-op) and queue a key for the second-pass exported=1 update, so
+    // queries filtering on exported=1 find backfilled symbols (#970).
+    for (const exp of symbols.exports ?? []) {
+      rows.push([exp.name, exp.kind, relPath, exp.line, null, null, exp.name, null, null]);
+      exportKeys.push([exp.name, exp.kind, relPath, exp.line]);
+    }
+  }
+  const db = ctx.db as unknown as BetterSqlite3Database;
+  batchInsertNodes(db, rows);
+
+  // Mark exported symbols in batches — mirrors insertDefinitionsAndExports.
+  if (exportKeys.length > 0) {
+    const EXPORT_CHUNK = 500;
+    const exportStmtCache = new Map<number, SqliteStatement>();
+    for (let i = 0; i < exportKeys.length; i += EXPORT_CHUNK) {
+      const end = Math.min(i + EXPORT_CHUNK, exportKeys.length);
+      const chunkSize = end - i;
+      let updateStmt = exportStmtCache.get(chunkSize);
+      if (!updateStmt) {
+        const conditions = Array.from(
+          { length: chunkSize },
+          () => '(name = ? AND kind = ? AND file = ? AND line = ?)',
+        ).join(' OR ');
+        updateStmt = db.prepare(`UPDATE nodes SET exported = 1 WHERE ${conditions}`);
+        exportStmtCache.set(chunkSize, updateStmt);
+      }
+      const vals: unknown[] = [];
+      for (let j = i; j < end; j++) {
+        const k = exportKeys[j] as unknown[];
+        vals.push(k[0], k[1], k[2], k[3]);
+      }
+      updateStmt.run(...vals);
+    }
+  }
+}
+
 // ── Pipeline stages execution ───────────────────────────────────────────
 
 async function runPipelineStages(ctx: PipelineContext): Promise<void> {

src/domain/parser.ts

@@ -338,6 +338,27 @@ export function isWasmAvailable(): boolean {
   );
 }
 
+/**
+ * Return the set of lowercase file extensions whose WASM grammar is actually
+ * installed on disk. Used to scope engine-parity backfill to files that WASM
+ * can recover — languages without an installed grammar are skipped by both
+ * engines, so they don't represent a native-engine drop.
+ *
+ * Cached on first call; the grammars directory is shipped immutable.
+ */
+let _installedWasmExts: Set<string> | null = null;
+export function getInstalledWasmExtensions(): Set<string> {
+  if (_installedWasmExts) return _installedWasmExts;
+  const exts = new Set<string>();
+  for (const entry of LANGUAGE_REGISTRY) {
+    if (fs.existsSync(grammarPath(entry.grammarFile))) {
+      for (const ext of entry.extensions) exts.add(ext.toLowerCase());
+    }
+  }
+  _installedWasmExts = exts;
+  return exts;
+}
+
 // ── Unified API ──────────────────────────────────────────────────────────────
 
 function resolveEngine(opts: ParseEngineOpts = {}): ResolvedEngine {
@@ -884,8 +905,10 @@ export async function parseFilesAuto(
     ? native.parseFilesFull(filePaths, rootDir)
     : native.parseFiles(filePaths, rootDir, true, true);
   const needsTypeMap: { filePath: string; relPath: string }[] = [];
+  const nativeParsed = new Set<string>();
   for (const r of nativeResults) {
     if (!r) continue;
+    nativeParsed.add(r.file);
     const patched = patchNativeResult(r);
     const relPath = path.relative(rootDir, r.file).split(path.sep).join('/');
     result.set(relPath, patched);
@@ -901,6 +924,24 @@ export async function parseFilesAuto(
   if (needsTypeMap.length > 0) {
     await backfillTypeMapBatch(needsTypeMap, result);
   }
+
+  // Engine parity: native may silently drop files whose extensions are in
+  // SUPPORTED_EXTENSIONS (because a WASM grammar exists) but whose Rust
+  // extractor/grammar is missing or fails. WASM handles these — fall back so
+  // both engines process the same file set (#967). Restrict to installed WASM
+  // grammars so we don't warn about files that neither engine can parse.
+  const installedExts = getInstalledWasmExtensions();
+  const dropped = filePaths.filter(
+    (f) => !nativeParsed.has(f) && installedExts.has(path.extname(f).toLowerCase()),
+  );
+  if (dropped.length > 0) {
+    warn(`Native engine dropped ${dropped.length} file(s); falling back to WASM for parity`);
+    const wasmResults = await parseFilesWasm(dropped, rootDir);
+    for (const [relPath, symbols] of wasmResults) {
+      result.set(relPath, symbols);
+    }
+  }
+
   return result;
 }