From 82346b4b5e2032012caddb83636da180b94fedf4 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Wed, 11 Mar 2026 14:54:44 -0700 Subject: [PATCH 01/23] 1000 custom search params test --- .../Rest/Reindex/ReindexTests.cs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 44735d6a4b..fbee350dfc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -10,6 +10,7 @@ using System.Net; using System.Security.Cryptography; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Extensions.Xunit; @@ -38,6 +39,56 @@ public ReindexTests(HttpIntegrationTestFixture fixture) _isSql = _fixture.DataStore == DataStore.SqlServer; } + [Fact] + public async Task Given1000SearchParams_WhenReindexCompletes_ThenSearchParamsAreEnabled() + { + await CancelAnyRunningReindexJobsAsync(); + + const int numberOfSearchParams = 1000; + + var searchParams = new List(); + try + { + // this takes 8 minutes locally + await Parallel.ForAsync(0, numberOfSearchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (i, cancel) => + { + var randomSuffix = Guid.NewGuid().ToString("N").Substring(0, 8); + var searchParam = await CreateCustomSearchParameterAsync($"custom-person-name-{randomSuffix}", ["Person"], "Person.name.given", SearchParamType.String); + Assert.NotNull(searchParam); + lock (searchParam) + { + searchParams.Add(searchParam); + } + }); + + var parameters = new Parameters + { + Parameter = + [ + new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerQuery", Value = new Integer(1) }, + new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerWrite", Value = new Integer(1) }, + ], + }; + + var value = ((FhirResponse response, Uri jobUri))await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + Assert.Equal(HttpStatusCode.Created, value.response.Response.StatusCode); + + await WaitForJobCompletionAsync(value.jobUri, TimeSpan.FromSeconds(300)); + + await Task.Delay(TimeSpan.FromSeconds(10)); + + // this takes minutes locally + await Parallel.ForEachAsync(searchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (param, cancel) => + { + await VerifySearchParameterIsEnabledAsync($"Person?{param.Code}=Test", param.Code); + }); + } + finally + { + await CleanupSearchParametersAsync(searchParams.ToArray()); + } + } + [Fact] public async Task GivenReindexJobWithConcurrentUpdates_ThenReportedCountsAreLessThanOriginal() { @@ -921,6 +972,15 @@ e.Resource is OperationOutcome oo && oo.Issue?.Any(i => i.Code?.ToString() == "NotSupported") == true) ?? false; } + private async Task VerifySearchParameterIsEnabledAsync(string searchQuery, string searchParameterCode) + { + var response = await _fixture.TestFhirClient.SearchAsync(searchQuery); + Assert.NotNull(response); + var error = HasNotSupportedError(response.Resource); + Assert.False(error, $"Search param {searchParameterCode} is NOT supported after reindex."); + return; + } + /// /// Verifies that a search parameter is properly indexed and working by executing a search query with retry logic. /// Retries handle timing issues with search parameter cache refresh after reindex operations. From bbc66cd3c7b47c178073ca01e2d7fde7d18ee526 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Wed, 11 Mar 2026 15:26:21 -0700 Subject: [PATCH 02/23] parallel delete --- .../Rest/Reindex/ReindexTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index fbee350dfc..2fc33aa3dc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -77,7 +77,6 @@ public async Task Given1000SearchParams_WhenReindexCompletes_ThenSearchParamsAre await Task.Delay(TimeSpan.FromSeconds(10)); - // this takes minutes locally await Parallel.ForEachAsync(searchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (param, cancel) => { await VerifySearchParameterIsEnabledAsync($"Person?{param.Code}=Test", param.Code); @@ -85,7 +84,10 @@ public async Task Given1000SearchParams_WhenReindexCompletes_ThenSearchParamsAre } finally { - await CleanupSearchParametersAsync(searchParams.ToArray()); + await Parallel.ForEachAsync(searchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (param, cancel) => + { + await _fixture.TestFhirClient.DeleteAsync($"SearchParameter/{param.Id}"); + }); } } From cd9286000d33d62e376eb72b66e73e472e1f58c1 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 07:14:26 -0700 Subject: [PATCH 03/23] 1000 search params test --- .../Parameters/SearchParameterOperations.cs | 16 +- .../FhirClient.cs | 2 + .../Parameters/SearchParameterValidator.cs | 2 +- .../Rest/Reindex/ReindexTests.cs | 140 +++++++++++------- 4 files changed, 101 insertions(+), 59 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index 289d016299..5d3f892a0b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -89,7 +89,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to add // a search parameter. This is to avoid creating a duplicate search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + ////await GetAndApplySearchParameterUpdates(cancellationToken); // verify the parameter is supported before continuing var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -166,7 +166,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to delete // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + ////await GetAndApplySearchParameterUpdates(cancellationToken); // First we delete the status metadata from the data store as this function depends on // the in memory definition manager. Once complete we remove the SearchParameter from @@ -217,7 +217,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to update // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + ////await GetAndApplySearchParameterUpdates(cancellationToken); var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -292,6 +292,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( /// A task. public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken = default, bool forceFullRefresh = false) { + var st = DateTime.UtcNow; var results = await _searchParameterStatusManager.GetSearchParameterStatusUpdates(cancellationToken, forceFullRefresh ? null : _searchParamLastUpdated); var statuses = results.Statuses; @@ -327,6 +328,7 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati cancellationToken); var paramsToAdd = new List(); + var hasAllResources = true; foreach (var searchParam in statusesToFetch) { if (!searchParamResources.TryGetValue(searchParam.Uri.OriginalString, out var searchParamResource)) @@ -334,6 +336,7 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati _logger.LogInformation( "Updated SearchParameter status found for SearchParameter: {Url}, but did not find any SearchParameter resources when querying for this url.", searchParam.Uri); + hasAllResources = false; continue; } @@ -365,10 +368,13 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati var inCache = ParametersAreInCache(statusesToFetch, cancellationToken); - if (results.LastUpdated.HasValue && inCache) // this should be the ony place in the code to assign last updated + if (results.LastUpdated.HasValue && hasAllResources && inCache) // this should be the ony place in the code to assign last updated { _searchParamLastUpdated = results.LastUpdated.Value; } + + ////using var search = _searchServiceFactory.Invoke(); + ////await search.Value.TryLogEvent("GetAndApplySearchParameterUpdates", "Warn", $"HasResources={hasResources} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}", st, cancellationToken); } // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, @@ -380,7 +386,7 @@ private bool ParametersAreInCache(IReadOnlyCollection(); + const int numberOfSearchParams = 500; + const string urlPrefix = "http://example.org/fhir/SearchParameter/"; + var codes = new List(); + var urls = new List(); try { - // this takes 8 minutes locally - await Parallel.ForAsync(0, numberOfSearchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (i, cancel) => + for (var i = 0; i < numberOfSearchParams; i++) { - var randomSuffix = Guid.NewGuid().ToString("N").Substring(0, 8); - var searchParam = await CreateCustomSearchParameterAsync($"custom-person-name-{randomSuffix}", ["Person"], "Person.name.given", SearchParamType.String); - Assert.NotNull(searchParam); - lock (searchParam) - { - searchParams.Add(searchParam); - } - }); + var code = $"c-p-n-{i}"; + codes.Add(code); + urls.Add($"{urlPrefix}{code}"); + } + + var bundle = await CreatePersonSearchParamsAsync(); + Assert.Equal(numberOfSearchParams, bundle.Entry.Count); + foreach (var entry in bundle.Entry) + { + Assert.True(entry.Resource as SearchParameter != null, $"actual={JsonConvert.SerializeObject(entry)}"); + } + + // check by urls + var response = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", urls)}"); + Assert.True(response.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={response.Resource.Total}"); var parameters = new Parameters { @@ -70,24 +79,79 @@ public async Task Given1000SearchParams_WhenReindexCompletes_ThenSearchParamsAre ], }; - var value = ((FhirResponse response, Uri jobUri))await _fixture.TestFhirClient.PostReindexJobAsync(parameters); - Assert.Equal(HttpStatusCode.Created, value.response.Response.StatusCode); - - await WaitForJobCompletionAsync(value.jobUri, TimeSpan.FromSeconds(300)); + var value = ((FhirResponse Response, Uri JobUri))await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); - await Task.Delay(TimeSpan.FromSeconds(10)); + await WaitForJobCompletionAsync(value.JobUri, TimeSpan.FromSeconds(300)); - await Parallel.ForEachAsync(searchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (param, cancel) => + await Parallel.ForEachAsync(codes, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (code, cancel) => { - await VerifySearchParameterIsEnabledAsync($"Person?{param.Code}=Test", param.Code); + await VerifySearchParameterIsEnabledAsync($"Person?{code}=Test", code); }); } finally { - await Parallel.ForEachAsync(searchParams, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (param, cancel) => + await DeletePersonSearchParamsAsync(); + } + + async Task CreatePersonSearchParamsAsync() + { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; + + #if R5 + var resourceTypes = new List(); + resourceTypes.Add(Enum.Parse("Person")); + #else + var resourceTypes = new List(); + resourceTypes.Add(Enum.Parse("Person")); + #endif + + foreach (var code in codes) { - await _fixture.TestFhirClient.DeleteAsync($"SearchParameter/{param.Id}"); - }); + var searchParam = new SearchParameter + { + Id = code, + Url = $"{urlPrefix}{code}", + Name = code, + Code = code, + Status = PublicationStatus.Active, + Type = SearchParamType.String, + Expression = "Person.name.given", + Description = "any", + Base = resourceTypes, + }; + + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{code}" }, Resource = searchParam }); + } + + var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); + return result; + } + + async Task VerifySearchParameterIsEnabledAsync(string searchQuery, string searchParameterCode) + { + var response = await _fixture.TestFhirClient.SearchAsync(searchQuery); + Assert.NotNull(response); + var error = HasNotSupportedError(response.Resource); + Assert.False(error, $"Search param {searchParameterCode} is NOT supported after reindex."); + return; + } + + async Task DeletePersonSearchParamsAsync() + { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; + + foreach (var code in codes) + { + bundle.Entry.Add( + new EntryComponent + { + Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{code}" }, + }); + } + + var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); + return result; } } @@ -698,27 +762,6 @@ private Person CreatePersonResource(string id, string name) } } - /// - /// Helper method to create and post a resource in a single operation - /// - private async Task CreateAndPostResourceAsync(string id, string name, Func> createResourceFunc) - where T : Resource - { - var resource = await createResourceFunc(id, name); - - try - { - // Post the resource using the client's CreateAsync method - var response = await _fixture.TestFhirClient.CreateAsync(resource); - return response; - } - catch (Exception ex) - { - System.Diagnostics.Debug.WriteLine($"Failed to create resource {id}: {ex.Message}"); - return resource; // Return the original resource even on failure so ID can be tracked - } - } - private async Task CreateCustomSearchParameterAsync(string code, string[] baseResourceTypes, string expression, SearchParamType searchParamType = SearchParamType.String) { #if R5 @@ -974,15 +1017,6 @@ e.Resource is OperationOutcome oo && oo.Issue?.Any(i => i.Code?.ToString() == "NotSupported") == true) ?? false; } - private async Task VerifySearchParameterIsEnabledAsync(string searchQuery, string searchParameterCode) - { - var response = await _fixture.TestFhirClient.SearchAsync(searchQuery); - Assert.NotNull(response); - var error = HasNotSupportedError(response.Resource); - Assert.False(error, $"Search param {searchParameterCode} is NOT supported after reindex."); - return; - } - /// /// Verifies that a search parameter is properly indexed and working by executing a search query with retry logic. /// Retries handle timing issues with search parameter cache refresh after reindex operations. From 6cfd67c3217dcf7aeafa8f7c7eb2423e3b0b8ffc Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 15:38:14 -0700 Subject: [PATCH 04/23] GetAndApplySearchParameterUpdates calls back --- .../Features/Search/Parameters/SearchParameterOperations.cs | 6 +++--- .../Features/Search/Parameters/SearchParameterValidator.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index e8379b8def..f94feb0075 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -89,7 +89,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to add // a search parameter. This is to avoid creating a duplicate search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - ////await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken); // verify the parameter is supported before continuing var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -166,7 +166,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to delete // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - ////await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken); // First we delete the status metadata from the data store as this function depends on // the in memory definition manager. Once complete we remove the SearchParameter from @@ -217,7 +217,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to update // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - ////await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken); var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index 43a3838130..7257a568e2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -117,7 +117,7 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri else { // Refresh the search parameter cache in the search parameter definition manager before starting the validation. - ////await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // If a search parameter with the same uri exists already if (_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParameterInfo)) From c5fe482a47f48987aed7729828ad349640e7d009 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 09:07:07 -0700 Subject: [PATCH 05/23] Replace sql connection factory by sql retry in search param data store --- ...FilebasedSearchParameterStatusDataStore.cs | 5 + .../ISearchParameterStatusDataStore.cs | 2 + .../CosmosDbSearchParameterStatusDataStore.cs | 5 + .../Features/Schema/SchemaVersionConstants.cs | 4 +- ...SqlServerSearchParameterStatusDataStore.cs | 193 ++++++++---------- .../SqlServerFhirStorageTestsFixture.cs | 9 +- 6 files changed, 101 insertions(+), 117 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs index 8b9b4de74d..8f2c1b3dba 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs @@ -37,6 +37,11 @@ public FilebasedSearchParameterStatusDataStore( public delegate ISearchParameterStatusDataStore Resolver(); + public Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken) + { + throw new NotFiniteNumberException(); + } + public async Task> GetSearchParameterStatuses(CancellationToken cancellationToken, DateTimeOffset? startLastUpdated = null) { if (_statusResults == null) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs index f4b0927661..7e1e111c0a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs @@ -17,5 +17,7 @@ public interface ISearchParameterStatusDataStore Task UpsertStatuses(IReadOnlyCollection statuses, CancellationToken cancellationToken); void SyncStatuses(IReadOnlyCollection statuses); + + Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs index 6c4fd06a81..5b5a89e963 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs @@ -37,6 +37,11 @@ public CosmosDbSearchParameterStatusDataStore( _queryFactory = queryFactory; } + public Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken) + { + throw new NotFiniteNumberException(); + } + public async Task> GetSearchParameterStatuses(CancellationToken cancellationToken, DateTimeOffset? startLastUpdated = null) { using IScoped clientScope = _containerScopeFactory.Invoke(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 17e1c40c2f..c5d7393e67 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -7,9 +7,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema { public static class SchemaVersionConstants { - public const int Min = (int)SchemaVersion.V100; + public const int Min = (int)SchemaVersion.V103; public const int Max = (int)SchemaVersion.V106; - public const int MinForUpgrade = (int)SchemaVersion.V100; // this is used for upgrade tests only + public const int MinForUpgrade = (int)SchemaVersion.V103; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; public const int SearchParameterHashSchemaVersion = (int)SchemaVersion.V8; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs index d8c73db43a..5d2f11d921 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs @@ -6,10 +6,13 @@ using System; using System.Collections.Generic; using System.Data; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Transactions; using EnsureThat; +using Hl7.Fhir.Model; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Microsoft.Health.Extensions.DependencyInjection; @@ -31,117 +34,100 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry { internal class SqlServerSearchParameterStatusDataStore : ISearchParameterStatusDataStore { - private readonly IScopeProvider _scopedSqlConnectionWrapperFactory; + private readonly ISqlRetryService _sqlRetryService; private readonly SchemaInformation _schemaInformation; - private readonly SqlServerSortingValidator _sortingValidator; private readonly ISqlServerFhirModel _fhirModel; private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; private readonly ILogger _logger; public SqlServerSearchParameterStatusDataStore( - IScopeProvider scopedSqlConnectionWrapperFactory, + ISqlRetryService sqlRetryService, SchemaInformation schemaInformation, - SqlServerSortingValidator sortingValidator, ISqlServerFhirModel fhirModel, ISearchParameterDefinitionManager searchParameterDefinitionManager, ILogger logger) { - EnsureArg.IsNotNull(scopedSqlConnectionWrapperFactory, nameof(scopedSqlConnectionWrapperFactory)); + EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); EnsureArg.IsNotNull(schemaInformation, nameof(schemaInformation)); - EnsureArg.IsNotNull(sortingValidator, nameof(sortingValidator)); EnsureArg.IsNotNull(fhirModel, nameof(fhirModel)); EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); EnsureArg.IsNotNull(logger, nameof(logger)); - _scopedSqlConnectionWrapperFactory = scopedSqlConnectionWrapperFactory; + _sqlRetryService = sqlRetryService; _schemaInformation = schemaInformation; - _sortingValidator = sortingValidator; _fhirModel = fhirModel; _searchParameterDefinitionManager = searchParameterDefinitionManager; _logger = logger; } + public async Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken) + { + await _sqlRetryService.TryLogEvent(process, status, text, startDate, cancellationToken); + } + public async Task> GetSearchParameterStatuses(CancellationToken cancellationToken, DateTimeOffset? startLastUpdated = null) { - using (IScoped scopedSqlConnectionWrapperFactory = _scopedSqlConnectionWrapperFactory.Invoke()) - using (SqlConnectionWrapper sqlConnectionWrapper = await scopedSqlConnectionWrapperFactory.Value.ObtainSqlConnectionWrapperAsync(cancellationToken, true)) - using (SqlCommandWrapper cmd = sqlConnectionWrapper.CreateRetrySqlCommand()) + using var cmd = new SqlCommand(); + cmd.CommandType = CommandType.StoredProcedure; + cmd.CommandText = "dbo.GetSearchParamStatuses"; + if (startLastUpdated.HasValue) { - cmd.CommandType = CommandType.StoredProcedure; - cmd.CommandText = "dbo.GetSearchParamStatuses"; - if (_schemaInformation.Current.Value >= 103 && startLastUpdated.HasValue) - { - cmd.Parameters.AddWithValue("@StartLastUpdated", startLastUpdated.Value); - } - - var parameterStatuses = new List(); + cmd.Parameters.AddWithValue("@StartLastUpdated", startLastUpdated.Value); + } - // TODO: Bad reader. Use SQL retry - using (SqlDataReader sqlDataReader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) + var results = await cmd.ExecuteReaderAsync( + _sqlRetryService, + (reader) => { - while (await sqlDataReader.ReadAsync(cancellationToken)) - { - short id; - string uri; - string stringStatus; - DateTimeOffset? lastUpdated; - bool? isPartiallySupported; - - ResourceSearchParameterStatus resourceSearchParameterStatus; - - (id, uri, stringStatus, lastUpdated, isPartiallySupported) = sqlDataReader.ReadRow( - VLatest.SearchParam.SearchParamId, - VLatest.SearchParam.Uri, - VLatest.SearchParam.Status, - VLatest.SearchParam.LastUpdated, - VLatest.SearchParam.IsPartiallySupported); - - var status = Enum.Parse(stringStatus, true); - - resourceSearchParameterStatus = new SqlServerResourceSearchParameterStatus - { - Id = id, - Uri = new Uri(uri), - Status = status, - IsPartiallySupported = (bool)isPartiallySupported, - LastUpdated = (DateTimeOffset)lastUpdated, - }; - - // Check whether the corresponding type of the search parameter is supported. - SearchParameterInfo paramInfo = null; - try - { - paramInfo = _searchParameterDefinitionManager.GetSearchParameter(resourceSearchParameterStatus.Uri.OriginalString); - } - catch (SearchParameterNotSupportedException) - { - } + (short id, string uri, string stringStatus, DateTimeOffset lastUpdated, bool isPartiallySupported) = reader.ReadRow( + VLatest.SearchParam.SearchParamId, + VLatest.SearchParam.Uri, + VLatest.SearchParam.Status, + VLatest.SearchParam.LastUpdated, + VLatest.SearchParam.IsPartiallySupported); + + return (id, uri, stringStatus, lastUpdated, isPartiallySupported); + }, + _logger, + cancellationToken); + + var parameterStatuses = new List(); + foreach (var result in results) + { + var status = Enum.Parse(result.stringStatus, true); - if (paramInfo != null && SqlServerSortingValidator.SupportedSortParamTypes.Contains(paramInfo.Type)) - { - resourceSearchParameterStatus.SortStatus = SortParameterStatus.Enabled; - } - else - { - resourceSearchParameterStatus.SortStatus = SortParameterStatus.Disabled; - } + var resourceSearchParameterStatus = new SqlServerResourceSearchParameterStatus + { + Id = result.id, + Uri = new Uri(result.uri), + Status = status, + IsPartiallySupported = result.isPartiallySupported, + LastUpdated = result.lastUpdated, + }; + + // Check whether the corresponding type of the search parameter is supported. + SearchParameterInfo paramInfo = null; + try + { + paramInfo = _searchParameterDefinitionManager.GetSearchParameter(resourceSearchParameterStatus.Uri.OriginalString); + } + catch (SearchParameterNotSupportedException) + { + } - if (_schemaInformation.Current.Value < 103 && startLastUpdated.HasValue) - { - if (resourceSearchParameterStatus.LastUpdated > startLastUpdated.Value) // this is temporary as old stored proc does not have start last updated parameter - { - parameterStatuses.Add(resourceSearchParameterStatus); - } - } - else - { - parameterStatuses.Add(resourceSearchParameterStatus); - } - } + if (paramInfo != null && SqlServerSortingValidator.SupportedSortParamTypes.Contains(paramInfo.Type)) + { + resourceSearchParameterStatus.SortStatus = SortParameterStatus.Enabled; + } + else + { + resourceSearchParameterStatus.SortStatus = SortParameterStatus.Disabled; } - return parameterStatuses; + parameterStatuses.Add(resourceSearchParameterStatus); } + + return parameterStatuses; } public async Task UpsertStatuses(IReadOnlyCollection statuses, CancellationToken cancellationToken) @@ -200,40 +186,29 @@ private async Task UpsertStatusesWithRetry(IReadOnlyCollection statuses, CancellationToken cancellationToken) { - using (IScoped scopedSqlConnectionWrapperFactory = _scopedSqlConnectionWrapperFactory.Invoke()) - using (SqlConnectionWrapper sqlConnectionWrapper = await scopedSqlConnectionWrapperFactory.Value.ObtainSqlConnectionWrapperAsync(cancellationToken, true)) - using (SqlCommandWrapper cmd = sqlConnectionWrapper.CreateRetrySqlCommand()) + using var cmd = new SqlCommand(); + cmd.CommandType = CommandType.StoredProcedure; + cmd.CommandText = "dbo.MergeSearchParams"; + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(statuses.ToList())); + + var results = await cmd.ExecuteReaderAsync( + _sqlRetryService, + (reader) => { return reader.ReadRow(VLatest.SearchParam.SearchParamId, VLatest.SearchParam.Uri, VLatest.SearchParam.LastUpdated); }, + _logger, + cancellationToken); + + foreach (var result in results) { - cmd.CommandType = CommandType.StoredProcedure; - if (_schemaInformation.Current >= 103) - { - cmd.CommandText = "dbo.MergeSearchParams"; - } - else - { - cmd.CommandText = "dbo.UpsertSearchParamsWithOptimisticConcurrency"; - } + (short searchParamId, string searchParamUri, DateTimeOffset lastUpdated) = result; - new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(statuses.ToList())); + // Add the new search parameters to the FHIR model dictionary. + _fhirModel.TryAddSearchParamIdToUriMapping(searchParamUri, searchParamId); - // TODO: Reader is not propagating all failures to the code - using (SqlDataReader sqlDataReader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) + // Update the LastUpdated in our original collection for future operations + var matchingStatus = statuses.FirstOrDefault(s => s.Uri.OriginalString == searchParamUri); + if (matchingStatus != null) { - while (await sqlDataReader.ReadAsync(cancellationToken)) - { - // The procedure returns new search parameters. - (short searchParamId, string searchParamUri, DateTimeOffset lastUpdated) = sqlDataReader.ReadRow(VLatest.SearchParam.SearchParamId, VLatest.SearchParam.Uri, VLatest.SearchParam.LastUpdated); - - // Add the new search parameters to the FHIR model dictionary. - _fhirModel.TryAddSearchParamIdToUriMapping(searchParamUri, searchParamId); - - // Update the LastUpdated in our original collection for future operations - var matchingStatus = statuses.FirstOrDefault(s => s.Uri.OriginalString == searchParamUri); - if (matchingStatus != null) - { - matchingStatus.LastUpdated = lastUpdated; - } - } + matchingStatus.LastUpdated = lastUpdated; } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index b6591b7a41..79e784f999 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -178,7 +178,7 @@ public async Task InitializeAsync() SqlTransactionHandler = new SqlTransactionHandler(); SqlConnectionWrapperFactory = new SqlConnectionWrapperFactory(SqlTransactionHandler, SqlConnectionBuilder, sqlRetryLogicBaseProvider, SqlServerDataStoreConfiguration); - var sqlRetryService = new SqlRetryService(SqlConnectionBuilder, SqlServerDataStoreConfiguration, Options.Create(new SqlRetryServiceOptions()), new SqlRetryServiceDelegateOptions(), Options.Create(new CoreFeatureConfiguration())); + SqlRetryService = new SqlRetryService(SqlConnectionBuilder, SqlServerDataStoreConfiguration, Options.Create(new SqlRetryServiceOptions()), new SqlRetryServiceDelegateOptions(), Options.Create(new CoreFeatureConfiguration())); var sqlServerFhirModel = new SqlServerFhirModel( SchemaInformation, @@ -187,7 +187,7 @@ public async Task InitializeAsync() Options.Create(securityConfiguration), SqlConnectionWrapperFactory.CreateMockScopeProvider(), Substitute.For(), - sqlRetryService, + SqlRetryService, NullLogger.Instance); SqlServerFhirModel = sqlServerFhirModel; @@ -224,9 +224,8 @@ public async Task InitializeAsync() _supportedSearchParameterDefinitionManager = new SupportedSearchParameterDefinitionManager(_searchParameterDefinitionManager); SqlServerSearchParameterStatusDataStore = new SqlServerSearchParameterStatusDataStore( - SqlConnectionWrapperFactory.CreateMockScopeProvider(), + SqlRetryService, SchemaInformation, - sqlSortingValidator, sqlServerFhirModel, _searchParameterDefinitionManager, NullLogger.Instance); @@ -237,8 +236,6 @@ public async Task InitializeAsync() var bundleOrchestrator = new BundleOrchestrator(bundleOptions, NullLogger.Instance); - SqlRetryService = new SqlRetryService(SqlConnectionBuilder, SqlServerDataStoreConfiguration, Options.Create(new SqlRetryServiceOptions()), new SqlRetryServiceDelegateOptions(), Options.Create(new CoreFeatureConfiguration())); - var importErrorSerializer = new Shared.Core.Features.Operations.Import.ImportErrorSerializer(new Hl7.Fhir.Serialization.FhirJsonSerializer()); _fhirDataStore = new SqlServerFhirDataStore( From ac09feacdfb85926f13fe8290104056010edf243 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 16:11:29 -0700 Subject: [PATCH 06/23] Try log event --- .../Features/Search/Parameters/SearchParameterOperations.cs | 3 +-- .../Features/Search/Registry/SearchParameterStatusManager.cs | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index 14f16e0c63..73ffa9afa6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -379,8 +379,7 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati _searchParamLastUpdated = results.LastUpdated.Value; } - ////using var search = _searchServiceFactory.Invoke(); - ////await search.Value.TryLogEvent("GetAndApplySearchParameterUpdates", "Warn", $"HasResources={hasResources} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}", st, cancellationToken); + await _searchParameterStatusManager.TryLogEvent("GetAndApplySearchParameterUpdates", "Warn", $"Cache advanced={inCache && allHaveResources} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}", st, cancellationToken); } // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs index 69c4ad19d1..14e2035e42 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs @@ -49,6 +49,11 @@ public SearchParameterStatusManager( _logger = logger; } + internal async Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken) + { + await _searchParameterStatusDataStore.TryLogEvent(process, status, text, startDate, cancellationToken); + } + internal async Task EnsureInitializedAsync(CancellationToken cancellationToken) { var updated = new List(); From 3b3fe45f20e5ef210ca2e02008f035fd1c77b8e7 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 16:11:44 -0700 Subject: [PATCH 07/23] 200 --- .../Rest/Reindex/ReindexTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 8d0debe9d7..717546307f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -42,11 +42,11 @@ public ReindexTests(HttpIntegrationTestFixture fixture) } [Fact] - public async Task Given1000SearchParams_WhenReindexCompletes_ThenSearchParamsAreEnabled() + public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreEnabled() { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 500; + const int numberOfSearchParams = 200; // increase to 500 when cache is not updated by API calls. const string urlPrefix = "http://example.org/fhir/SearchParameter/"; var codes = new List(); var urls = new List(); From 366ea6be9f8e823193e3475a1f965999b9d1bbeb Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Fri, 13 Mar 2026 16:15:19 -0700 Subject: [PATCH 08/23] removed descr --- .../Features/Search/Parameters/SearchParameterOperations.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index 73ffa9afa6..afb406b4a6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -391,7 +391,7 @@ private bool ParametersAreInCache(IReadOnlyCollection Date: Fri, 13 Mar 2026 17:00:37 -0700 Subject: [PATCH 09/23] 100 --- .../Rest/Reindex/ReindexTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 717546307f..00764af2ff 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -46,7 +46,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 200; // increase to 500 when cache is not updated by API calls. + const int numberOfSearchParams = 100; // increase to 500 when cache is not updated by API calls. const string urlPrefix = "http://example.org/fhir/SearchParameter/"; var codes = new List(); var urls = new List(); From 5a41d0173423a72a47016382f84357d7aa47c470 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Sat, 14 Mar 2026 15:04:22 -0700 Subject: [PATCH 10/23] 50 --- .../Rest/Reindex/ReindexTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 00764af2ff..ce825769b9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -46,15 +46,15 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 100; // increase to 500 when cache is not updated by API calls. - const string urlPrefix = "http://example.org/fhir/SearchParameter/"; + const int numberOfSearchParams = 50; // increase to 500 when cache is not updated by API calls. + const string urlPrefix = "http://example.org/"; var codes = new List(); var urls = new List(); try { for (var i = 0; i < numberOfSearchParams; i++) { - var code = $"c-p-n-{i}"; + var code = $"c-id-{i}"; codes.Add(code); urls.Add($"{urlPrefix}{code}"); } @@ -86,7 +86,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE await Parallel.ForEachAsync(codes, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (code, cancel) => { - await VerifySearchParameterIsEnabledAsync($"Person?{code}=Test", code); + await VerifySearchParameterIsEnabledAsync($"Person?{code}=test", code); }); } finally @@ -115,8 +115,8 @@ async Task CreatePersonSearchParamsAsync() Name = code, Code = code, Status = PublicationStatus.Active, - Type = SearchParamType.String, - Expression = "Person.name.given", + Type = SearchParamType.Token, + Expression = "Person.id", Description = "any", Base = resourceTypes, }; From f8bb673952f96a7e2ff99722b634f78ccbbb37ee Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Sun, 15 Mar 2026 16:28:28 -0700 Subject: [PATCH 11/23] 10 --- .../Rest/Reindex/ReindexTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index ce825769b9..cfe069855f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -46,7 +46,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 50; // increase to 500 when cache is not updated by API calls. + const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls. const string urlPrefix = "http://example.org/"; var codes = new List(); var urls = new List(); From 7160789d93ce20b2020958496279fd53d0089dbc Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Sun, 15 Mar 2026 22:11:00 -0700 Subject: [PATCH 12/23] Removed 600 --- src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs b/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs index 92c9af2260..234df413d3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs +++ b/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs @@ -61,8 +61,6 @@ public FhirClient( { EnsureArg.IsNotNull(httpClient, nameof(httpClient)); - httpClient.Timeout = TimeSpan.FromSeconds(600); - if (httpClient.BaseAddress == null) { throw new ArgumentException(Resources.BaseAddressMustBeSpecified); From 46ce120edcd176d8464b5c93354e042653b9aa09 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Sun, 15 Mar 2026 22:17:37 -0700 Subject: [PATCH 13/23] using --- .../Registry/SqlServerSearchParameterStatusDataStore.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs index 5d2f11d921..c49b847a22 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs @@ -6,13 +6,10 @@ using System; using System.Collections.Generic; using System.Data; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; -using System.Transactions; using EnsureThat; -using Hl7.Fhir.Model; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Microsoft.Health.Extensions.DependencyInjection; From 719d4afb5f2fb16e668adaf3894753390736996f Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 16:07:46 -0700 Subject: [PATCH 14/23] using --- .../Rest/Reindex/ReindexTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index cfe069855f..6814011913 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -10,7 +10,6 @@ using System.Net; using System.Security.Cryptography; using System.Threading; -using System.Threading.Channels; using System.Threading.Tasks; using Hl7.Fhir.Model; using Microsoft.Health.Extensions.Xunit; From bc23714d6dfa47007ecc4ad5da27ba9f2d2ef98b Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 18:20:43 -0700 Subject: [PATCH 15/23] 50 --- .../Rest/Reindex/ReindexTests.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 6814011913..2b97701826 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -45,7 +45,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls. + const int numberOfSearchParams = 50; // increase to 500 when cache is not updated by API calls. const string urlPrefix = "http://example.org/"; var codes = new List(); var urls = new List(); @@ -69,14 +69,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE var response = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", urls)}"); Assert.True(response.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={response.Resource.Total}"); - var parameters = new Parameters - { - Parameter = - [ - new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerQuery", Value = new Integer(1) }, - new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerWrite", Value = new Integer(1) }, - ], - }; + var parameters = new Parameters { Parameter = [] }; var value = ((FhirResponse Response, Uri JobUri))await _fixture.TestFhirClient.PostReindexJobAsync(parameters); Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); @@ -142,11 +135,7 @@ async Task DeletePersonSearchParamsAsync() foreach (var code in codes) { - bundle.Entry.Add( - new EntryComponent - { - Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{code}" }, - }); + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{code}" } }); } var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); From 25dfa2c552764d7e39258312bbc398c7304a5043 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 20:46:28 -0700 Subject: [PATCH 16/23] 20 --- .../Rest/Reindex/ReindexTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 2b97701826..803263e606 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -45,7 +45,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 50; // increase to 500 when cache is not updated by API calls. + const int numberOfSearchParams = 20; // increase to 500 when cache is not updated by API calls. const string urlPrefix = "http://example.org/"; var codes = new List(); var urls = new List(); From c6df958101711d2b6470e311a5b0664ad548bc9d Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 20:53:36 -0700 Subject: [PATCH 17/23] value --- .../FhirClient.cs | 2 +- .../Rest/Reindex/ReindexTests.cs | 24 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs b/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs index 234df413d3..e46c2e3366 100644 --- a/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs +++ b/src/Microsoft.Health.Fhir.Shared.Client/FhirClient.cs @@ -569,7 +569,7 @@ public async Task> PostBundleAsync(Resource bundle, FhirBun return await CreateResponseAsync(response); } - public async Task<(FhirResponse reponse, Uri uri)> PostReindexJobAsync( + public async Task<(FhirResponse Response, Uri Uri)> PostReindexJobAsync( Parameters parameters, string uniqueResource = null, CancellationToken cancellationToken = default) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 803263e606..fca39814e3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -71,10 +71,10 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE var parameters = new Parameters { Parameter = [] }; - var value = ((FhirResponse Response, Uri JobUri))await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + var value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); - await WaitForJobCompletionAsync(value.JobUri, TimeSpan.FromSeconds(300)); + await WaitForJobCompletionAsync(value.Uri, TimeSpan.FromSeconds(300)); await Parallel.ForEachAsync(codes, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async (code, cancel) => { @@ -150,7 +150,6 @@ public async Task GivenReindexJobWithConcurrentUpdates_ThenReportedCountsAreLess var searchParam = new SearchParameter(); var testResources = new List<(string resourceType, string resourceId)>(); - (FhirResponse response, Uri jobUri) value = default; try { @@ -169,18 +168,18 @@ public async Task GivenReindexJobWithConcurrentUpdates_ThenReportedCountsAreLess ], }; - value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); - Assert.Equal(HttpStatusCode.Created, value.response.Response.StatusCode); + var value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); var tasks = new[] { - WaitForJobCompletionAsync(value.jobUri, TimeSpan.FromSeconds(300)), + WaitForJobCompletionAsync(value.Uri, TimeSpan.FromSeconds(300)), RandomPersonUpdate(testResources), }; await Task.WhenAll(tasks); // reported in reindex counts should be less than total resources created - await CheckReportedCounts(value.jobUri, testResources.Count, true); + await CheckReportedCounts(value.Uri, testResources.Count, true); } finally { @@ -204,7 +203,6 @@ public async Task GivenReindexJobWithMixedZeroAndNonZeroCountResources_WhenReind var testResources = new List<(string resourceType, string resourceId)>(); var supplyDeliveryCount = 40 * storageMultiplier; var personCount = 20 * storageMultiplier; - (FhirResponse response, Uri jobUri) value = default; try { @@ -253,13 +251,13 @@ public async Task GivenReindexJobWithMixedZeroAndNonZeroCountResources_WhenReind }, }; - value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + var value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); - Assert.Equal(HttpStatusCode.Created, value.response.Response.StatusCode); - Assert.NotNull(value.jobUri); + Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); + Assert.NotNull(value.Uri); // Wait for job to complete (this will wait for all sub-jobs to complete) - var jobStatus = await WaitForJobCompletionAsync(value.jobUri, TimeSpan.FromSeconds(300)); + var jobStatus = await WaitForJobCompletionAsync(value.Uri, TimeSpan.FromSeconds(300)); Assert.True( jobStatus == OperationStatus.Completed, $"Expected Completed, got {jobStatus}"); @@ -269,7 +267,7 @@ public async Task GivenReindexJobWithMixedZeroAndNonZeroCountResources_WhenReind await SearchCreatedResources("SupplyDelivery", supplyDeliveryCount); // check what reindex job reported - await CheckReportedCounts(value.jobUri, testResources.Count, false); + await CheckReportedCounts(value.Uri, testResources.Count, false); // Verify search parameter is working for SupplyDelivery (which has data) // Use the ACTUAL count we got, not the desired count From 39afef3c5c8519a78a4bb54229ffa6b30fd78e25 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 20:56:55 -0700 Subject: [PATCH 18/23] inline params --- .../Rest/Reindex/ReindexTests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index fca39814e3..a65842964f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -69,9 +69,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE var response = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", urls)}"); Assert.True(response.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={response.Resource.Total}"); - var parameters = new Parameters { Parameter = [] }; - - var value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); + var value = await _fixture.TestFhirClient.PostReindexJobAsync(new Parameters { Parameter = [] }); Assert.Equal(HttpStatusCode.Created, value.Response.Response.StatusCode); await WaitForJobCompletionAsync(value.Uri, TimeSpan.FromSeconds(300)); From 743a9d02dca9cdb25384bd1836c14821b2980e9c Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Mon, 16 Mar 2026 21:00:23 -0700 Subject: [PATCH 19/23] remove output bundle from delete --- .../Rest/Reindex/ReindexTests.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index a65842964f..f8c30f5e7e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -124,10 +124,9 @@ async Task VerifySearchParameterIsEnabledAsync(string searchQuery, string search Assert.NotNull(response); var error = HasNotSupportedError(response.Resource); Assert.False(error, $"Search param {searchParameterCode} is NOT supported after reindex."); - return; } - async Task DeletePersonSearchParamsAsync() + async Task DeletePersonSearchParamsAsync() { var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; @@ -136,8 +135,7 @@ async Task DeletePersonSearchParamsAsync() bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{code}" } }); } - var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); - return result; + await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); } } From 8b8e21c16246eb0d3c3544f5a54fec024a7a585c Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Tue, 17 Mar 2026 08:57:28 -0700 Subject: [PATCH 20/23] 10 --- .../Rest/Reindex/ReindexTests.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index f8c30f5e7e..b81204d62f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -45,17 +45,15 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 20; // increase to 500 when cache is not updated by API calls. - const string urlPrefix = "http://example.org/"; + const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls. + const string urlPrefix = "http://my.org/"; var codes = new List(); - var urls = new List(); try { for (var i = 0; i < numberOfSearchParams; i++) { var code = $"c-id-{i}"; codes.Add(code); - urls.Add($"{urlPrefix}{code}"); } var bundle = await CreatePersonSearchParamsAsync(); @@ -66,7 +64,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE } // check by urls - var response = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", urls)}"); + var response = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", codes.Select(_ => $"{urlPrefix}{_}"))}"); Assert.True(response.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={response.Resource.Total}"); var value = await _fixture.TestFhirClient.PostReindexJobAsync(new Parameters { Parameter = [] }); From cb4369a8c592d7a4a75285d5ba1c7b3be8e78b3a Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Tue, 17 Mar 2026 09:03:46 -0700 Subject: [PATCH 21/23] comment --- .../Rest/Reindex/ReindexTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index b81204d62f..d8df6b1400 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -45,7 +45,7 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE { await CancelAnyRunningReindexJobsAsync(); - const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls. + const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls and status is saved with resources in a single SQL transaction const string urlPrefix = "http://my.org/"; var codes = new List(); try From 0c375de644d64202732c9c6bb6cccb8aae9d122b Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Tue, 17 Mar 2026 14:32:01 -0700 Subject: [PATCH 22/23] log caller --- .../Reindex/CreateReindexRequestHandler.cs | 2 +- .../Parameters/ISearchParameterOperations.cs | 7 +++---- .../Parameters/SearchParameterOperations.cs | 18 ++++++++++-------- ...chParameterCacheRefreshBackgroundService.cs | 2 +- .../ReindexSingleResourceRequestHandler.cs | 2 +- .../Parameters/SearchParameterValidator.cs | 2 +- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs index 304725646b..d31054bdea 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs @@ -72,7 +72,7 @@ public async Task Handle(CreateReindexRequest request, Ca // We need to pull in latest search parameter updates from the data store before creating a reindex job. // There could be a potential delay of before // search parameter updates on one instance propagates to other instances. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "CreateReindexRequestHandler"); // What this handles is the scenario where a user is effectively forcing a reindex to run by passing // in a parameter of targetSearchParameterTypes. From those we can identify the base resource types. diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs index d2ce3f275a..6d0b3572a2 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs @@ -22,14 +22,13 @@ public interface ISearchParameterOperations Task UpdateSearchParameterAsync(ITypedElement searchParam, RawResource previousSearchParam, CancellationToken cancellationToken); /// - /// This method should be called periodically to get any updates to SearchParameters - /// added to the DB by other service instances. - /// It should also be called when a user starts a reindex job + /// This method should be called periodically to get any updates to SearchParameters added to the DB by other service instances. /// /// Cancellation token /// When true, forces a full refresh from database instead of incremental updates + /// If provided, indicates the caller of this method /// A task. - Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken, bool forceFullRefresh = false); + Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken, bool forceFullRefresh = false, string caller = null); string GetSearchParameterHash(string resourceType); } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index afb406b4a6..f401f654b4 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -89,7 +89,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to add // a search parameter. This is to avoid creating a duplicate search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken, caller: "AddSearchParameterAsync"); // verify the parameter is supported before continuing var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -166,7 +166,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to delete // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken, caller: "DeleteSearchParameterAsync"); // First we delete the status metadata from the data store as this function depends on // the in memory definition manager. Once complete we remove the SearchParameter from @@ -217,7 +217,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to update // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + await GetAndApplySearchParameterUpdates(cancellationToken, caller: "UpdateSearchParameterAsync"); var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -283,14 +283,13 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( } /// - /// This method should be called periodically to get any updates to SearchParameters - /// added to the DB by other service instances. - /// It should also be called when a user starts a reindex job + /// This method should be called periodically to get any updates to SearchParameters added to the DB by other service instances. /// /// Cancellation token /// When true, forces a full refresh from database instead of incremental updates + /// If provided, indicates the caller of this method /// A task. - public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken = default, bool forceFullRefresh = false) + public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken = default, bool forceFullRefresh = false, string caller = null) { var st = DateTime.UtcNow; var results = await _searchParameterStatusManager.GetSearchParameterStatusUpdates(cancellationToken, forceFullRefresh ? null : _searchParamLastUpdated); @@ -379,7 +378,10 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati _searchParamLastUpdated = results.LastUpdated.Value; } - await _searchParameterStatusManager.TryLogEvent("GetAndApplySearchParameterUpdates", "Warn", $"Cache advanced={inCache && allHaveResources} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}", st, cancellationToken); + var method = $"GetAndApplySearchParameterUpdates{(caller != null ? $".{caller}" : string.Empty)}"; + var msg = $"Cache in sync={inCache && allHaveResources} Processed params={statuses.Count} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}"; + _logger.LogInformation($"{method}: {msg}"); + await _searchParameterStatusManager.TryLogEvent(method, "Warn", msg, st, cancellationToken); } // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs index 10c0474dad..935a21997b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs @@ -118,7 +118,7 @@ private async void OnRefreshTimer(object state) try { _logger.LogInformation("Performing incremental SearchParameter cache refresh..."); - await _searchParameterOperations.GetAndApplySearchParameterUpdates(_stoppingToken, false); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(_stoppingToken, false, "SearchParameterCacheRefreshBackgroundService"); _logger.LogInformation("Completed incremental SearchParameter cache refresh."); } catch (OperationCanceledException) diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs index 0043ec0827..9800249aea 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs @@ -74,7 +74,7 @@ public async Task Handle(ReindexSingleResourceReq throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundById, request.ResourceType, request.ResourceId)); } - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "ReindexSingleResourceRequestHandler"); // We need to extract the "new" search indices since the assumption is that // a new search parameter has been added to the fhir server. diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index 7257a568e2..5b8b8e65bd 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -117,7 +117,7 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri else { // Refresh the search parameter cache in the search parameter definition manager before starting the validation. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "SearchParameterValidator"); // If a search parameter with the same uri exists already if (_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParameterInfo)) From 042dff62ea6f95be31c0ab090459e5ff64cfb584 Mon Sep 17 00:00:00 2001 From: Sergey Galuzo Date: Wed, 18 Mar 2026 11:37:40 -0700 Subject: [PATCH 23/23] removed caller --- .../Reindex/CreateReindexRequestHandler.cs | 2 +- .../Parameters/ISearchParameterOperations.cs | 3 +-- .../Search/Parameters/SearchParameterOperations.cs | 14 ++++++-------- ...SearchParameterCacheRefreshBackgroundService.cs | 2 +- .../Reindex/ReindexSingleResourceRequestHandler.cs | 2 +- .../Search/Parameters/SearchParameterValidator.cs | 2 +- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs index d31054bdea..304725646b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs @@ -72,7 +72,7 @@ public async Task Handle(CreateReindexRequest request, Ca // We need to pull in latest search parameter updates from the data store before creating a reindex job. // There could be a potential delay of before // search parameter updates on one instance propagates to other instances. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "CreateReindexRequestHandler"); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // What this handles is the scenario where a user is effectively forcing a reindex to run by passing // in a parameter of targetSearchParameterTypes. From those we can identify the base resource types. diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs index 6d0b3572a2..e3c69d2993 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs @@ -26,9 +26,8 @@ public interface ISearchParameterOperations /// /// Cancellation token /// When true, forces a full refresh from database instead of incremental updates - /// If provided, indicates the caller of this method /// A task. - Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken, bool forceFullRefresh = false, string caller = null); + Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken, bool forceFullRefresh = false); string GetSearchParameterHash(string resourceType); } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index f401f654b4..c7dec7a323 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -89,7 +89,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to add // a search parameter. This is to avoid creating a duplicate search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken, caller: "AddSearchParameterAsync"); + await GetAndApplySearchParameterUpdates(cancellationToken); // verify the parameter is supported before continuing var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -166,7 +166,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to delete // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken, caller: "DeleteSearchParameterAsync"); + await GetAndApplySearchParameterUpdates(cancellationToken); // First we delete the status metadata from the data store as this function depends on // the in memory definition manager. Once complete we remove the SearchParameter from @@ -217,7 +217,7 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( // We need to make sure we have the latest search parameters before trying to update // existing search parameter. This is to avoid trying to update a search parameter that // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken, caller: "UpdateSearchParameterAsync"); + await GetAndApplySearchParameterUpdates(cancellationToken); var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); @@ -287,9 +287,8 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( /// /// Cancellation token /// When true, forces a full refresh from database instead of incremental updates - /// If provided, indicates the caller of this method /// A task. - public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken = default, bool forceFullRefresh = false, string caller = null) + public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken = default, bool forceFullRefresh = false) { var st = DateTime.UtcNow; var results = await _searchParameterStatusManager.GetSearchParameterStatusUpdates(cancellationToken, forceFullRefresh ? null : _searchParamLastUpdated); @@ -378,10 +377,9 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati _searchParamLastUpdated = results.LastUpdated.Value; } - var method = $"GetAndApplySearchParameterUpdates{(caller != null ? $".{caller}" : string.Empty)}"; var msg = $"Cache in sync={inCache && allHaveResources} Processed params={statuses.Count} SearchParamLastUpdated={_searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fff")}"; - _logger.LogInformation($"{method}: {msg}"); - await _searchParameterStatusManager.TryLogEvent(method, "Warn", msg, st, cancellationToken); + _logger.LogInformation($"GetAndApplySearchParameterUpdates: {msg}"); + await _searchParameterStatusManager.TryLogEvent("GetAndApplySearchParameterUpdates", "Warn", msg, st, cancellationToken); } // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs index 935a21997b..10c0474dad 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs @@ -118,7 +118,7 @@ private async void OnRefreshTimer(object state) try { _logger.LogInformation("Performing incremental SearchParameter cache refresh..."); - await _searchParameterOperations.GetAndApplySearchParameterUpdates(_stoppingToken, false, "SearchParameterCacheRefreshBackgroundService"); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(_stoppingToken, false); _logger.LogInformation("Completed incremental SearchParameter cache refresh."); } catch (OperationCanceledException) diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs index 9800249aea..0043ec0827 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs @@ -74,7 +74,7 @@ public async Task Handle(ReindexSingleResourceReq throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundById, request.ResourceType, request.ResourceId)); } - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "ReindexSingleResourceRequestHandler"); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // We need to extract the "new" search indices since the assumption is that // a new search parameter has been added to the fhir server. diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index 5b8b8e65bd..7257a568e2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -117,7 +117,7 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri else { // Refresh the search parameter cache in the search parameter definition manager before starting the validation. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, caller: "SearchParameterValidator"); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // If a search parameter with the same uri exists already if (_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParameterInfo))