diff --git a/eng/Versions.props b/eng/Versions.props index 0621a70ae59..950907dd15d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 10.0.0-rc.1.25555.107 10.0.0-rc.1.25555.107 10.0.0-rc.1.25555.107 - 0.16.0-nightly.20260320.467 + 0.16.0-nightly.20260324.479 10.0.0-rc.1.25555.107 11.0.0-prerelease.26169.1 diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index ef8782db2aa..3bcb0a18441 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -4,6 +4,7 @@ netstandard2.0 enable true + $(NoWarn);NU1510 <_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)')) @@ -11,6 +12,14 @@ + + none + true + + + none + true + diff --git a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs index 7b3b2d38f3b..fadd193f8f8 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs @@ -21,15 +21,16 @@ public class MandatoryCategoryValidator : IValidator public MandatoryCategoryValidator(ImmutableHashSet categories) => _mandatoryCategories = categories; - public IEnumerable Validate(ValidationParameters validationParameters) - => validationParameters.Benchmarks + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) + { + return validationParameters.Benchmarks .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) .Distinct() - .Select(benchmarkId => - new ValidationError( - isCritical: TreatsWarningsAsErrors, - $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]") - ); + .Select(benchmarkId => new ValidationError( + isCritical: TreatsWarningsAsErrors, + $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]")) + .ToAsyncEnumerable(); + } } } \ No newline at end of file diff --git a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs index 5b2cfec63ca..d99620be0a2 100644 --- a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs @@ -23,16 +23,17 @@ public class NoWasmValidator : IValidator public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory; - public IEnumerable Validate(ValidationParameters validationParameters) - => validationParameters.Benchmarks + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) + { + return validationParameters.Benchmarks .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) .Distinct() - .Select(benchmarkId => - new ValidationError( - isCritical: TreatsWarningsAsErrors, - $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]") - ); + .Select(benchmarkId => new ValidationError( + isCritical: TreatsWarningsAsErrors, + $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]")) + .ToAsyncEnumerable(); + } private bool IsAsyncMethod(MethodInfo workloadMethod) { diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs index c1ed4268e0f..104828ff982 100644 --- a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs @@ -9,19 +9,25 @@ using Reporting; using System; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace BenchmarkDotNet.Extensions { - public class PerfLabExporter : ExporterBase + /// + /// Exports benchmark results to JSON format for PerfLab ingestion. + /// + public class PerfLabExporter : IExporter { - protected override string FileExtension => "json"; - protected override string FileCaption => "perf-lab-report"; + public string Name => "PerfLab"; - public PerfLabExporter() + public ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) { + Export(summary, logger); + return default; } - public override void ExportToLog(Summary summary, ILogger logger) + public void Export(Summary summary, ILogger logger) { var reporter = new Reporter(); @@ -132,7 +138,9 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit var jsonOutput = reporter.GetJson(); if (jsonOutput is not null) - logger.WriteLine(jsonOutput); + { + Console.WriteLine(jsonOutput); + } } } } diff --git a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs index aaf8a62317a..bc4803e4ca1 100644 --- a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs @@ -19,17 +19,17 @@ public class TooManyTestCasesValidator : IValidator public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) { - var byDescriptor = validationParameters.Benchmarks + return validationParameters.Benchmarks .Where(benchmark => !SkipValidation(benchmark.Descriptor.WorkloadMethod)) - .GroupBy(benchmark => (benchmark.Descriptor, benchmark.Job)); // descriptor = type + method - - return byDescriptor.Where(benchmarkCase => benchmarkCase.Count() > Limit).Select(group => - new ValidationError( + .GroupBy(benchmark => (benchmark.Descriptor, benchmark.Job)) + .Where(group => group.Count() > Limit) + .Select(group => new ValidationError( isCritical: true, message: $"{group.Key.Descriptor.Type.Name}.{group.Key.Descriptor.WorkloadMethod.Name} has {group.Count()} test cases. It MUST NOT have more than {Limit} test cases. We don't have inifinite amount of time to run all the benchmarks!!", - benchmarkCase: group.First())); + benchmarkCase: group.First())) + .ToAsyncEnumerable(); } private static bool SkipValidation(MemberInfo member) diff --git a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs index 532e9b003f5..00a29353eb3 100644 --- a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs @@ -14,8 +14,9 @@ public class UniqueArgumentsValidator : IValidator { public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) - => validationParameters.Benchmarks + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) + { + return validationParameters.Benchmarks .Where(benchmark => benchmark.HasArguments || benchmark.HasParameters) .GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job)) .Where(sameBenchmark => @@ -25,7 +26,9 @@ public IEnumerable Validate(ValidationParameters validationPara return numberOfTestCases != numberOfUniqueTestCases; }) - .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())); + .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())) + .ToAsyncEnumerable(); + } private class BenchmarkArgumentsComparer : IEqualityComparer { @@ -34,7 +37,7 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal)) return true; - return Enumerable.SequenceEqual( + return System.Linq.Enumerable.SequenceEqual( x.Parameters.Items.Select(argument => argument.Value), y.Parameters.Items.Select(argument => argument.Value)); }