Merge pull request #115 from saddam213/FeatureExtractor

Feature extractor update

Author: saddam213

OnnxStack.Console/Examples/ControlNetFeatureExample.cs

@@ -35,10 +35,10 @@ public async Task RunAsync()
             var inputImage = await InputImage.FromFileAsync("D:\\Repositories\\OnnxStack\\Assets\\Samples\\Img2Img_Start.bmp");
 
             // Create Annotation pipeline
-            var annotationPipeline = AnnotationPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators");
+            var annotationPipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true);
 
             // Create Depth Image
-            var controlImage = await annotationPipeline.DepthImage(inputImage);
+            var controlImage = await annotationPipeline.RunAsync(inputImage);
 
             // Save Depth Image (Debug Only)
             await controlImage.Image.SaveAsPngAsync(Path.Combine(_outputDirectory, $"Depth.png"));

OnnxStack.Console/Examples/FeatureExtractorExample.cs

@@ -0,0 +1,68 @@
+using OnnxStack.Core.Image;
+using OnnxStack.FeatureExtractor.Pipelines;
+using OnnxStack.StableDiffusion.Config;
+using SixLabors.ImageSharp;
+using System.Diagnostics;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class FeatureExtractorExample : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+
+        public FeatureExtractorExample(StableDiffusionConfig configuration)
+        {
+            _configuration = configuration;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(FeatureExtractorExample));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public int Index => 13;
+
+        public string Name => "Feature Extractor Example";
+
+        public string Description => "Simple exmaple using ControlNet feature extractors";
+
+        /// <summary>
+        /// ControlNet Example
+        /// </summary>
+        public async Task RunAsync()
+        {
+            // Load Control Image
+            var inputImage = await InputImage.FromFileAsync("D:\\Repositories\\OnnxStack\\Assets\\Samples\\Img2Img_Start.bmp");
+
+            var pipelines = new[]
+            {
+                FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\canny.onnx"),
+                FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\hed.onnx"),
+                FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true),
+
+               // FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\depth-anything-large-hf\\onnx\\model.onnx", normalize: true, sampleSize: 504),
+               // FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\sentis-MiDaS\\dpt_beit_large_512.onnx", normalize: true, sampleSize: 384),
+            };
+
+            foreach (var pipeline in pipelines)
+            {
+                var timestamp = Stopwatch.GetTimestamp();
+                OutputHelpers.WriteConsole($"Load pipeline`{pipeline.Name}`", ConsoleColor.Cyan);
+
+                // Run Pipeline
+                var imageFeature = await pipeline.RunAsync(inputImage);
+
+                OutputHelpers.WriteConsole($"Generating image", ConsoleColor.Cyan);
+
+                // Save Image
+                await imageFeature.Image.SaveAsPngAsync(Path.Combine(_outputDirectory, $"{pipeline.Name}.png"));
+
+
+                OutputHelpers.WriteConsole($"Unload pipeline", ConsoleColor.Cyan);
+
+                //Unload
+                await pipeline.UnloadAsync();
+
+                OutputHelpers.WriteConsole($"Elapsed: {Stopwatch.GetElapsedTime(timestamp)}ms", ConsoleColor.Yellow);
+            }
+        }
+    }
+}

OnnxStack.Core/Extensions/TensorExtension.cs

@@ -32,6 +32,28 @@ public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int cou
         }
 
 
+        /// <summary>
+        /// Normalize the data using Min-Max scaling to ensure all values are in the range [0, 1].
+        /// </summary>
+        /// <param name="tensor">The tensor.</param>
+        public static void NormalizeMinMax(this DenseTensor<float> tensor)
+        {
+            var values = tensor.Buffer.Span;
+            float min = float.PositiveInfinity, max = float.NegativeInfinity;
+            foreach (var val in values)
+            {
+                if (min > val) min = val;
+                if (max < val) max = val;
+            }
+
+            var range = max - min;
+            for (var i = 0; i < values.Length; i++)
+            {
+                values[i] = (values[i] - min) / range;
+            }
+        }
+
+
         /// <summary>
         /// Concatenates the specified tensors along the specified axis.
         /// </summary>

OnnxStack.Core/Video/VideoFrame.cs

@@ -2,5 +2,8 @@
 
 namespace OnnxStack.Core.Video
 {
-    public record VideoFrame(byte[] Frame, InputImage ControlImage = default);
+    public record VideoFrame(byte[] Frame)
+    {
+        public InputImage ExtraFrame { get; set; }
+    }
 }

OnnxStack.FeatureExtractor/Common/FeatureExtractorModel.cs

@@ -7,25 +7,35 @@ namespace OnnxStack.FeatureExtractor.Common
     public class FeatureExtractorModel : OnnxModelSession
     {
         private readonly int _sampleSize;
+        private readonly bool _normalize;
+        private readonly int _channels;
 
         public FeatureExtractorModel(FeatureExtractorModelConfig configuration)
             : base(configuration)
         {
             _sampleSize = configuration.SampleSize;
+            _normalize = configuration.Normalize;
+            _channels = configuration.Channels;
         }
 
         public int SampleSize => _sampleSize;
 
+        public bool Normalize => _normalize;
+
+        public int Channels => _channels;
+
         public static FeatureExtractorModel Create(FeatureExtractorModelConfig configuration)
         {
             return new FeatureExtractorModel(configuration);
         }
 
-        public static FeatureExtractorModel Create(string modelFile, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
+        public static FeatureExtractorModel Create(string modelFile, bool normalize = false, int sampleSize = 512, int channels = 3, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
         {
             var configuration = new FeatureExtractorModelConfig
             {
-                SampleSize = 512,
+                SampleSize = sampleSize,
+                Normalize = normalize,
+                Channels = channels,
                 DeviceId = deviceId,
                 ExecutionProvider = executionProvider,
                 ExecutionMode = ExecutionMode.ORT_SEQUENTIAL,
@@ -40,5 +50,7 @@ public static FeatureExtractorModel Create(string modelFile, int deviceId = 0, E
     public record FeatureExtractorModelConfig : OnnxModelConfig
     {
         public int SampleSize { get; set; }
+        public bool Normalize { get; set; }
+        public int Channels { get; set; }
     }
 }

…reExtractor/Common/AnnotationModelSet.cs …actor/Common/FeatureExtractorModelSet.csOnnxStack.FeatureExtractor/Common/AnnotationModelSet.cs renamed to OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs OnnxStack.FeatureExtractor/Common/AnnotationModelSet.cs renamed to OnnxStack.FeatureExtractor/Common/FeatureExtractorModelSet.cs

@@ -4,7 +4,7 @@
 
 namespace OnnxStack.FeatureExtractor.Common
 {
-    public record AnnotationModelSet : IOnnxModelSetConfig
+    public record FeatureExtractorModelSet : IOnnxModelSetConfig
     {
         public string Name { get; set; }
         public bool IsEnabled { get; set; }
@@ -15,14 +15,6 @@ public record AnnotationModelSet : IOnnxModelSetConfig
         public ExecutionProvider ExecutionProvider { get; set; }
 
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public FeatureExtractorModelConfig CannyImageConfig { get; set; }
-
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public FeatureExtractorModelConfig HedImageConfig { get; set; }
-
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public FeatureExtractorModelConfig DepthImageConfig { get; set; }
+        public FeatureExtractorModelConfig FeatureExtractorConfig { get; set; }
     }
-
-
 }