more porting stuff

Author: KSemenenko

Directory.Packages.props

@@ -2,8 +2,10 @@
   <ItemGroup>
     <PackageVersion Include="Aspire.Hosting" Version="8.0.0-preview.5.24201.12" />
     <PackageVersion Include="Aspire.Hosting.Testing" Version="8.0.0-preview.5.24201.12" />
+    <PackageVersion Include="coverlet.collector" Version="6.0.2" />
     <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.54.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.10" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="1.0.2" />
     <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.2" />

GraphRag.slnx

@@ -11,6 +11,7 @@
     <Project Path="src/ManagedCode.GraphRag/ManagedCode.GraphRag.csproj" />
   </Folder>
   <Folder Name="/tests/">
+    <Project Path="tests/ManagedCode.GraphRag.Tests/ManagedCode.GraphRag.Tests.csproj" />
     <Project Path="tests\ManagedCode.GraphRag.Tests.Integration\ManagedCode.GraphRag.Tests.Integration.csproj" />
   </Folder>
 </Solution>

docs/dotnet-port-plan.md

@@ -5,12 +5,10 @@ This working note documents the mapping between the Python implementation that l
 ## High-Level Architecture
 
 - **Configuration** – `GraphRagConfig` and companion models will be introduced under `GraphRag.Config`.  They mirror the Pydantic models (`graphrag.config.models`) and keep JSON/YAML compatibility with the original schema.
-- **Indexing Pipeline** – `GraphRag.Indexing` provides:
-  - `PipelineBuilder`, `PipelineRunContext`, `PipelineRunResult`, `WorkflowDelegate`.
-  - Workflow implementations translated from `graphrag.index.workflows.*`.
-  - Operation helpers from `graphrag.index.operations.*` rewritten against .NET primitives (`List<T>`, `ImmutableArray<T>`, `DataFrame` where necessary).
+- **Indexing Pipeline** – `GraphRag.Indexing` now ships `PipelineExecutor`, `IndexingPipelineRunner`, and translated workflows (`load_input_documents`, `create_base_text_units`, `create_final_documents`) with tokenization powered by `Microsoft.ML.Tokenizers`. Text ingestion supports text/CSV/JSON sources with the same grouping semantics as Python.
 - **Query Pipeline** – `GraphRag.Query` mirrors `graphrag.query.*` with orchestrators for question generation, context assembly, and answer synthesis.
 - **Storage** – `GraphRag.Storage` offers a provider model equivalent to `PipelineStorage` (file, memory, Blob, Cosmos).  A JSON-backed table serializer is in place while the Parquet implementation is ported.
+- **Graph Stores** – Postgres adapter issues parameterised Cypher queries (AGE) to avoid injection; unit tests assert payload sanitisation.
 - **Language Models & Tokenizers** – `GraphRag.LanguageModel` wraps Azure OpenAI/LiteLLM equivalents.  Configuration, retry, and rate limiting concepts are ported.
 - **Vector Stores** – `GraphRag.VectorStores` brings adapters for local FAISS-like embeddings, Azure Cognitive Search, and Postgres pgvector matching the Python `vector_stores`.
 - **Callbacks & Telemetry** – `GraphRag.Callbacks` contains workflow lifecycle hooks, tracing, and instrumentation mirroring `WorkflowCallbacks`.
@@ -36,12 +34,12 @@ This working note documents the mapping between the Python implementation that l
 
 ## Immediate TODOs
 
-1. Implement configuration model layer (`GraphRag.Config`).
-2. Port pipeline runtime (`GraphRag.Indexing.Runtime`) including callback chain, run loop, benchmarking.
-3. Recreate storage adapters (File, Memory) and Parquet serializer.
-4. Start translating workflows beginning with ingestion (`load_input_documents`, `create_base_text_units`, `create_final_documents`).
+1. Implement configuration model layer (`GraphRag.Config`). ✅
+2. Port pipeline runtime (`GraphRag.Indexing.Runtime`) including callback chain, run loop, benchmarking. ✅
+3. Recreate storage adapters (File, Memory) and Parquet serializer (JSON stub ready, Parquet pending).
+4. Translate remaining workflows beyond ingestion (graph extraction, summarisation, embeddings).
 5. Migrate vector store + embedding interfaces and integrate into indexing pipeline.
 6. Recreate query orchestrator and evaluation pipelines.
-7. Port tests iteratively, ensuring coverage parity with Python.
+7. Grow the .NET test suite (unit + integration) to ~85 % coverage and parity with Python scenarios.
 
 > This file is intentionally temporary; it guides the phased port while the codebase is in flux.

src/ManagedCode.GraphRag.Postgres/PostgresGraphStore.cs

@@ -1,15 +1,16 @@
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Runtime.CompilerServices;
-using System.Text;
 using System.Text.Json;
 using GraphRag.Graphs;
 using Microsoft.Extensions.Logging;
 using Npgsql;
+using NpgsqlTypes;
 
 namespace GraphRag.Storage.Postgres;
 
-public sealed class PostgresGraphStore : IGraphStore
+public class PostgresGraphStore : IGraphStore, IAsyncDisposable
 {
     private readonly string _connectionString;
     private readonly string _graphName;
@@ -50,8 +51,14 @@ public async Task UpsertNodeAsync(string id, string label, IReadOnlyDictionary<s
         ArgumentException.ThrowIfNullOrWhiteSpace(label);
         ArgumentNullException.ThrowIfNull(properties);
 
-        var cypher = BuildNodeUpsertCypher(id, label, properties);
-        await ExecuteCypherAsync(cypher, cancellationToken).ConfigureAwait(false);
+        var query = $"MERGE (n:{EscapeLabel(label)} {{ id: $node_id }}) SET n += $props RETURN n";
+        var parameters = new Dictionary<string, object?>
+        {
+            ["node_id"] = id,
+            ["props"] = ConvertProperties(properties)
+        };
+
+        await ExecuteCypherAsync(query, parameters, cancellationToken).ConfigureAwait(false);
         _logger.LogDebug("Upserted node {Id} ({Label}) into graph {GraphName}.", id, label, _graphName);
     }
 
@@ -62,8 +69,15 @@ public async Task UpsertRelationshipAsync(string sourceId, string targetId, stri
         ArgumentException.ThrowIfNullOrWhiteSpace(type);
         ArgumentNullException.ThrowIfNull(properties);
 
-        var cypher = BuildRelationshipUpsertCypher(sourceId, targetId, type, properties);
-        await ExecuteCypherAsync(cypher, cancellationToken).ConfigureAwait(false);
+        var query = $"MATCH (source {{ id: $source_id }}), (target {{ id: $target_id }}) MERGE (source)-[rel:{EscapeLabel(type)}]->(target) SET rel += $props RETURN rel";
+        var parameters = new Dictionary<string, object?>
+        {
+            ["source_id"] = sourceId,
+            ["target_id"] = targetId,
+            ["props"] = ConvertProperties(properties)
+        };
+
+        await ExecuteCypherAsync(query, parameters, cancellationToken).ConfigureAwait(false);
         _logger.LogDebug("Upserted relationship {Source}-[{Type}]->{Target} in graph {GraphName}.", sourceId, type, targetId, _graphName);
     }
 
@@ -76,17 +90,25 @@ async IAsyncEnumerable<GraphRelationship> FetchAsync(string nodeId, [EnumeratorC
         {
             await using var connection = await OpenConnectionAsync(token).ConfigureAwait(false);
             await using var command = connection.CreateCommand();
-            command.CommandText = $@"
+            command.CommandText = @"
 SELECT 
     source_id::text,
     target_id::text,
     edge_type::text,
     edge_props::text
-FROM cypher('{_graphName}', $$
-    MATCH (source {{ id: '{EscapeString(nodeId)}' }})-[rel]->(target)
+FROM cypher(@graph_name, $$
+    MATCH (source { id: $node_id })-[rel]->(target)
     RETURN source.id AS source_id, target.id AS target_id, type(rel) AS edge_type, properties(rel) AS edge_props
-$$) AS (source_id agtype, target_id agtype, edge_type agtype, edge_props agtype);
+$$, @params) AS (source_id agtype, target_id agtype, edge_type agtype, edge_props agtype);
 ";
+            command.Parameters.AddWithValue("graph_name", _graphName);
+            command.Parameters.Add(new NpgsqlParameter("params", NpgsqlDbType.Jsonb)
+            {
+                Value = JsonSerializer.Serialize(new Dictionary<string, object?>
+                {
+                    ["node_id"] = nodeId
+                })
+            });
 
             await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false);
             while (await reader.ReadAsync(token).ConfigureAwait(false))
@@ -102,11 +124,20 @@ FROM cypher('{_graphName}', $$
         }
     }
 
-    private async Task ExecuteCypherAsync(string statement, CancellationToken cancellationToken)
+    protected virtual async Task ExecuteCypherAsync(string query, IReadOnlyDictionary<string, object?> parameters, CancellationToken cancellationToken)
     {
         await using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
         await using var command = connection.CreateCommand();
-        command.CommandText = statement;
+        command.CommandText = @"
+SELECT *
+FROM cypher(@graph_name, @query, @params) AS (result agtype);
+";
+        command.Parameters.AddWithValue("graph_name", _graphName);
+        command.Parameters.AddWithValue("query", query);
+        command.Parameters.Add(new NpgsqlParameter("params", NpgsqlDbType.Jsonb)
+        {
+            Value = JsonSerializer.Serialize(parameters)
+        });
         await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
     }
 
@@ -131,74 +162,6 @@ private static async Task ExecuteNonQueryAsync(NpgsqlConnection connection, stri
         await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
     }
 
-    private string BuildNodeUpsertCypher(string id, string label, IReadOnlyDictionary<string, object?> properties)
-    {
-        var builder = new StringBuilder();
-        builder.AppendLine($"SELECT * FROM cypher('{_graphName}', $$");
-        builder.Append("    MERGE (n:");
-        builder.Append(EscapeLabel(label));
-        builder.Append(" { id: '");
-        builder.Append(EscapeString(id));
-        builder.Append("' })");
-
-        var setClause = BuildSetClause("n", properties, excludeId: true);
-        if (!string.IsNullOrEmpty(setClause))
-        {
-            builder.AppendLine();
-            builder.Append("    ");
-            builder.Append(setClause);
-        }
-
-        builder.AppendLine();
-        builder.AppendLine("RETURN n");
-        builder.Append("$$) AS (n agtype);");
-        return builder.ToString();
-    }
-
-    private string BuildRelationshipUpsertCypher(string sourceId, string targetId, string type, IReadOnlyDictionary<string, object?> properties)
-    {
-        var builder = new StringBuilder();
-        builder.AppendLine($"SELECT * FROM cypher('{_graphName}', $$");
-        builder.AppendLine($"    MATCH (source {{ id: '{EscapeString(sourceId)}' }}), (target {{ id: '{EscapeString(targetId)}' }})");
-        builder.Append("    MERGE (source)-[rel:");
-        builder.Append(EscapeLabel(type));
-        builder.Append("]->(target)");
-
-        var setClause = BuildSetClause("rel", properties, excludeId: false);
-        if (!string.IsNullOrEmpty(setClause))
-        {
-            builder.AppendLine();
-            builder.Append("    ");
-            builder.Append(setClause);
-        }
-
-        builder.AppendLine();
-        builder.AppendLine("RETURN rel");
-        builder.Append("$$) AS (rel agtype);");
-        return builder.ToString();
-    }
-
-    private static string BuildSetClause(string alias, IReadOnlyDictionary<string, object?> properties, bool excludeId)
-    {
-        var assignments = new List<string>();
-        foreach (var (key, value) in properties)
-        {
-            if (value is null)
-            {
-                continue;
-            }
-
-            if (excludeId && string.Equals(key, "id", StringComparison.OrdinalIgnoreCase))
-            {
-                continue;
-            }
-
-            assignments.Add($"{EscapePropertyKey(key)}: {FormatValue(value)}");
-        }
-
-        return assignments.Count == 0 ? string.Empty : $"SET {alias} += {{{string.Join(", ", assignments)}}}";
-    }
-
     private static string EscapeLabel(string label)
     {
         if (string.IsNullOrWhiteSpace(label))
@@ -217,88 +180,60 @@ private static string EscapeLabel(string label)
         return label;
     }
 
-    private static string EscapePropertyKey(string key)
+    private static IDictionary<string, object?> ConvertProperties(IReadOnlyDictionary<string, object?> properties)
     {
-        if (string.IsNullOrWhiteSpace(key))
-        {
-            throw new ArgumentException("Property key cannot be null or whitespace.", nameof(key));
-        }
+        var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
 
-        foreach (var ch in key)
+        foreach (var (key, value) in properties)
         {
-            if (!char.IsLetterOrDigit(ch) && ch != '_')
+            if (value is null || string.Equals(key, "id", StringComparison.OrdinalIgnoreCase))
             {
-                throw new ArgumentException($"Invalid character '{ch}' in property key '{key}'.", nameof(key));
+                continue;
             }
+
+            result[key] = value switch
+            {
+                string s => s,
+                int or long or short or byte => Convert.ToInt64(value, CultureInfo.InvariantCulture),
+                float or double or decimal => Convert.ToDouble(value, CultureInfo.InvariantCulture),
+                bool b => b,
+                DateTime dt => dt.ToUniversalTime(),
+                IEnumerable<string> list => list.ToArray(),
+                _ => value.ToString()
+            };
         }
 
-        return key;
+        return result;
     }
 
-    private static string EscapeString(string value) => value.Replace("'", "''", StringComparison.Ordinal);
+    private static string NormalizeAgTypeText(string value) => value.Trim('"');
 
-    private static string FormatValue(object value) => value switch
+    private static IReadOnlyDictionary<string, object?> ParseProperties(string json)
     {
-        null => "null",
-        string s => $"'{EscapeString(s)}'",
-        bool b => b ? "true" : "false",
-        int or long or short or byte => Convert.ToString(value, CultureInfo.InvariantCulture)!,
-        float f => f.ToString(CultureInfo.InvariantCulture),
-        double d => d.ToString(CultureInfo.InvariantCulture),
-        decimal dec => dec.ToString(CultureInfo.InvariantCulture),
-        Guid guid => $"'{guid:D}'",
-        DateTime dt => $"'{dt.ToUniversalTime():O}'",
-        DateTimeOffset dto => $"'{dto.ToUniversalTime():O}'",
-        _ => $"'{EscapeString(value.ToString() ?? string.Empty)}'"
-    };
-
-    private static Dictionary<string, object?> ParseProperties(string json)
-    {
-        if (string.IsNullOrWhiteSpace(json))
+        try
         {
-            return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
-        }
+            using var document = JsonDocument.Parse(json);
+            var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
+            foreach (var property in document.RootElement.EnumerateObject())
+            {
+                result[property.Name] = property.Value.ValueKind switch
+                {
+                    JsonValueKind.String => property.Value.GetString(),
+                    JsonValueKind.Number => property.Value.TryGetInt64(out var i64) ? i64 : property.Value.GetDouble(),
+                    JsonValueKind.True => true,
+                    JsonValueKind.False => false,
+                    JsonValueKind.Null => null,
+                    _ => property.Value.GetRawText()
+                };
+            }
 
-        using var document = JsonDocument.Parse(json);
-        if (document.RootElement.ValueKind != JsonValueKind.Object)
-        {
-            return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
+            return result;
         }
-
-        var result = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
-        foreach (var property in document.RootElement.EnumerateObject())
+        catch
         {
-            result[property.Name] = ConvertJsonElement(property.Value);
+            return new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
         }
-
-        return result;
     }
 
-    private static object? ConvertJsonElement(JsonElement element) => element.ValueKind switch
-    {
-        JsonValueKind.Null => null,
-        JsonValueKind.String => element.GetString(),
-        JsonValueKind.Number when element.TryGetInt64(out var longValue) => longValue,
-        JsonValueKind.Number => element.GetDouble(),
-        JsonValueKind.True => true,
-        JsonValueKind.False => false,
-        JsonValueKind.Array => element.EnumerateArray().Select(ConvertJsonElement).ToArray(),
-        JsonValueKind.Object => element.EnumerateObject().ToDictionary(prop => prop.Name, prop => ConvertJsonElement(prop.Value), StringComparer.OrdinalIgnoreCase),
-        _ => null
-    };
-
-    private static string NormalizeAgTypeText(string value)
-    {
-        if (string.IsNullOrEmpty(value))
-        {
-            return value;
-        }
-
-        if (value.Length >= 2 && ((value[0] == '"' && value[^1] == '"') || (value[0] == '\'' && value[^1] == '\'')))
-        {
-            return value[1..^1];
-        }
-
-        return value;
-    }
+    public ValueTask DisposeAsync() => ValueTask.CompletedTask;
 }

src/ManagedCode.GraphRag/Chunking/ChunkSlice.cs

@@ -0,0 +1,3 @@
+namespace GraphRag.Chunking;
+
+public sealed record ChunkSlice(string DocumentId, string Text);

src/ManagedCode.GraphRag/Chunking/ChunkerResolver.cs

@@ -0,0 +1,16 @@
+using GraphRag.Config;
+
+namespace GraphRag.Chunking;
+
+internal sealed class ChunkerResolver(TokenTextChunker tokenChunker, MarkdownTextChunker markdownChunker) : IChunkerResolver
+{
+    public ITextChunker Resolve(ChunkStrategyType strategy)
+    {
+        return strategy switch
+        {
+            ChunkStrategyType.Tokens => tokenChunker,
+            ChunkStrategyType.Sentence => markdownChunker,
+            _ => tokenChunker
+        };
+    }
+}

src/ManagedCode.GraphRag/Chunking/IChunkerResolver.cs

@@ -0,0 +1,8 @@
+using GraphRag.Config;
+
+namespace GraphRag.Chunking;
+
+public interface IChunkerResolver
+{
+    ITextChunker Resolve(ChunkStrategyType strategy);
+}