diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index 63131d05c03d5..8aedbc28eee11 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -1,4 +1,4 @@ - + Microsoft.ML.OnnxRuntime @@ -184,6 +184,10 @@ + + + + diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs index d38748c2f97cc..524f6af56e6bc 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs @@ -9,6 +9,14 @@ using System.Runtime.InteropServices; using System.Text; +#if NET8_0_OR_GREATER +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Runtime.CompilerServices; +using SystemNumericsTensors = System.Numerics.Tensors; +using TensorPrimitives = System.Numerics.Tensors.TensorPrimitives; +#endif + namespace Microsoft.ML.OnnxRuntime { /// @@ -205,6 +213,33 @@ public ReadOnlySpan GetTensorDataAsSpan() where T : unmanaged return MemoryMarshal.Cast(byteSpan); } +#if NET8_0_OR_GREATER + /// + /// Returns a ReadOnlyTensorSpan over tensor native buffer that + /// provides a read-only view. + /// + /// Note, that the memory may be device allocated and, therefore, not accessible from the CPU. + /// To get memory descriptor use GetTensorMemoryInfo(). + /// + /// OrtValue must contain a non-string tensor. + /// The span is valid as long as the OrtValue instance is alive (not disposed). + /// + /// + /// ReadOnlySpan + /// + [Experimental("SYSLIB5001")] + public SystemNumericsTensors.ReadOnlyTensorSpan GetTensorDataAsTensorSpan() where T : unmanaged + { + var byteSpan = GetTensorBufferRawData(typeof(T)); + + var typeSpan = MemoryMarshal.Cast(byteSpan); + var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape; + nint[] nArray = Array.ConvertAll(shape, new Converter(x => (nint)x)); + + return new SystemNumericsTensors.ReadOnlyTensorSpan(typeSpan, nArray, []); + } +#endif + /// /// Returns a Span over tensor native buffer. /// This enables you to safely and efficiently modify the underlying @@ -225,6 +260,32 @@ public Span GetTensorMutableDataAsSpan() where T : unmanaged return MemoryMarshal.Cast(byteSpan); } +#if NET8_0_OR_GREATER + /// + /// Returns a TensorSpan over tensor native buffer. + /// + /// Note, that the memory may be device allocated and, therefore, not accessible from the CPU. + /// To get memory descriptor use GetTensorMemoryInfo(). + /// + /// OrtValue must contain a non-string tensor. + /// The span is valid as long as the OrtValue instance is alive (not disposed). + /// + /// + /// ReadOnlySpan + /// + [Experimental("SYSLIB5001")] + public SystemNumericsTensors.TensorSpan GetTensorMutableDataAsTensorSpan() where T : unmanaged + { + var byteSpan = GetTensorBufferRawData(typeof(T)); + + var typeSpan = MemoryMarshal.Cast(byteSpan); + var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape; + nint[] nArray = Array.ConvertAll(shape, new Converter(x => (nint)x)); + + return new SystemNumericsTensors.TensorSpan(typeSpan, nArray, []); + } +#endif + /// /// Provides mutable raw native buffer access. /// @@ -234,6 +295,23 @@ public Span GetTensorMutableRawData() return GetTensorBufferRawData(typeof(byte)); } +#if NET8_0_OR_GREATER + /// + /// Provides mutable raw native buffer access. + /// + /// TensorSpan over the native buffer bytes + [Experimental("SYSLIB5001")] + public SystemNumericsTensors.TensorSpan GetTensorSpanMutableRawData() where T : unmanaged + { + var byteSpan = GetTensorBufferRawData(typeof(T)); + + var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape; + nint[] nArray = Array.ConvertAll(shape, new Converter(x => (nint)x)); + + return new SystemNumericsTensors.TensorSpan(byteSpan, nArray, []); + } +#endif + /// /// Fetch string tensor element buffer pointer at the specified index, /// convert/copy to UTF-16 char[] and return a ReadOnlyMemory{char} instance. @@ -605,6 +683,80 @@ public static OrtValue CreateTensorValueFromMemory(T[] data, long[] shape) wh return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, new Memory(data), shape); } +#if NET8_0_OR_GREATER + /// + /// This is a factory method creates a native Onnxruntime OrtValue containing a tensor. + /// The method will attempt to pin managed memory so no copying occurs when data is passed down + /// to native code. + /// + /// Tensor object + /// discovered tensor element type + /// And instance of OrtValue constructed on top of the object + [Experimental("SYSLIB5001")] + public static OrtValue CreateTensorValueFromSystemNumericsTensorObject(SystemNumericsTensors.Tensor tensor) where T : unmanaged + { + if (!IsContiguousAndDense(tensor)) + { + var newTensor = SystemNumericsTensors.Tensor.Create(tensor.Lengths); + tensor.CopyTo(newTensor); + tensor = newTensor; + } + unsafe + { + var backingData = (T[])tensor.GetType().GetField("_values", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(tensor); + GCHandle handle = GCHandle.Alloc(backingData, GCHandleType.Pinned); + var memHandle = new MemoryHandle(Unsafe.AsPointer(ref tensor.GetPinnableReference()), handle); + + try + { + IntPtr dataBufferPointer = IntPtr.Zero; + unsafe + { + dataBufferPointer = (IntPtr)memHandle.Pointer; + } + + var bufferLengthInBytes = tensor.FlattenedLength * sizeof(T); + long[] shape = Array.ConvertAll(tensor.Lengths.ToArray(), new Converter(x => (long)x)); + + var typeInfo = TensorBase.GetTypeInfo(typeof(T)) ?? + throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Tensor of type: {typeof(T)} is not supported"); + + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( + OrtMemoryInfo.DefaultInstance.Pointer, + dataBufferPointer, + (UIntPtr)(bufferLengthInBytes), + shape, + (UIntPtr)tensor.Rank, + typeInfo.ElementType, + out IntPtr nativeValue)); + + return new OrtValue(nativeValue, memHandle); + } + catch (Exception) + { + memHandle.Dispose(); + throw; + } + } + } + + [Experimental("SYSLIB5001")] + private static bool IsContiguousAndDense(SystemNumericsTensors.Tensor tensor) where T : unmanaged + { + // Right most dimension must be 1 for a dense tensor. + if (tensor.Strides[^1] != 1) + return false; + + // For other dimensions, the stride must be equal to the product of the dimensions to the right. + for (int i = tensor.Rank - 2; i >= 0; i--) + { + if (tensor.Strides[i] != TensorPrimitives.Product(tensor.Lengths.Slice(i + 1, tensor.Lengths.Length - i - 1))) + return false; + } + return true; + } +#endif + /// /// The factory API creates an OrtValue with memory allocated using the given allocator /// according to the specified shape and element type. The memory will be released when OrtValue diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index ff5fd2de54197..c67b474d49c0e 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -1,12 +1,19 @@ using Microsoft.ML.OnnxRuntime.Tensors; +using Microsoft.VisualStudio.TestPlatform.Utilities; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using System.Xml.Linq; using Xunit; +#if NET8_0_OR_GREATER +using DotnetTensors = System.Numerics.Tensors; +using TensorPrimitives = System.Numerics.Tensors.TensorPrimitives; +#endif + namespace Microsoft.ML.OnnxRuntime.Tests { /// @@ -67,6 +74,196 @@ public void CanCreateAndDisposeSessionWithModelPath() } } +#if NET8_0_OR_GREATER +#pragma warning disable SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback + [Theory] + [InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, true)] + [InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, false)] + [InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, true)] + [InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, false)] + private void CanRunInferenceOnAModelDotnetTensors(GraphOptimizationLevel graphOptimizationLevel, bool enableParallelExecution) + { + var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx"); + + using (var cleanUp = new DisposableListTest()) + { + // Set the graph optimization level for this session. + SessionOptions options = new SessionOptions(); + cleanUp.Add(options); + options.GraphOptimizationLevel = graphOptimizationLevel; + if (enableParallelExecution) options.ExecutionMode = ExecutionMode.ORT_PARALLEL; + + var session = new InferenceSession(model, options); + cleanUp.Add(session); + + using var runOptions = new RunOptions(); + using var inputOrtValues = new DisposableListTest>(session.InputMetadata.Count); + var inputMeta = session.InputMetadata; + var outputMeta = session.OutputMetadata; + + float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out"); + long[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data + ReadOnlySpan expectedOutputDimensions = expectedDimensions; + string[] expectedOutputNames = new string[] { "softmaxout_1" }; + + float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model + + foreach (var name in inputMeta.Keys) + { + Assert.Equal(typeof(float), inputMeta[name].ElementType); + Assert.True(inputMeta[name].IsTensor); + var tensor = DotnetTensors.Tensor.Create(inputData, inputMeta[name].Dimensions.Select(x => (nint)x).ToArray()); + inputOrtValues.Add(new DisposableTestPair(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + + } + + runOptions.LogId = "CsharpTest"; + runOptions.Terminate = false; // TODO: Test terminate = true, it currently crashes + runOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR; + // Run inference with named inputs and outputs created with in Run() + using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List(["softmaxout_1"]))) // results is an IDisposableReadOnlyCollection container + { + // validate the results + foreach (var r in results) + { + Assert.Single(results); + + ValidateRunResult(r, expectedOutput, expectedDimensions); + } + } + + session.Dispose(); + } + } + + [Fact] + public void InferenceSessionDisposedDotnetTensors() + { + var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx"); + + // Set the graph optimization level for this session. + using (SessionOptions options = new SessionOptions()) + { + options.ProfileOutputPathPrefix = "Ort_P_"; + options.EnableProfiling = true; + using (var session = new InferenceSession(model, options)) + { + var inputMeta = session.InputMetadata; + var container = new List(); + + float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model + + using (var runOptions = new RunOptions()) + using (var inputOrtValues = new DisposableListTest>(session.InputMetadata.Count)) + using (var outputOrtValues = new DisposableListTest>(session.OutputMetadata.Count)) + { + + foreach (var name in inputMeta.Keys) + { + Assert.Equal(typeof(float), inputMeta[name].ElementType); + Assert.True(inputMeta[name].IsTensor); + var tensor = DotnetTensors.Tensor.Create(inputData, inputMeta[name].Dimensions.Select(x => (nint) x).ToArray()); + inputOrtValues.Add(new DisposableTestPair(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + } + + // Run inference with named inputs and outputs created with in Run() + using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List(["softmaxout_1"]))) // results is an IDisposableReadOnlyCollection container + { + // validate the results + foreach (var r in results) + { + Assert.Single(results); + + float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out"); + long[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data + ValidateRunResult(r, expectedOutput, expectedDimensions); + } + } + } + + string profile_file = session.EndProfiling(); + + // Profile file should have the output path prefix in it + Assert.Contains("Ort_P_", profile_file); + } + } + } + + [Fact] + private void ThrowWrongOutputNameDotnetTensors() + { + var tuple = OpenSessionSqueezeNet(); + var session = tuple.Item1; + var inputData = tuple.Item2; + var inputTensor = tuple.Item3; + + using (var runOptions = new RunOptions()) + using (var inputOrtValues = new DisposableListTest>(session.InputMetadata.Count)) + using (var outputOrtValues = new DisposableListTest>(session.OutputMetadata.Count)) + { + var tensor = DotnetTensors.Tensor.Create(inputData, Array.ConvertAll(inputTensor.Dimensions.ToArray(), x => (nint)x)); + + inputOrtValues.Add(new DisposableTestPair("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + outputOrtValues.Add(new DisposableTestPair("bad_output_name", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + + var ex = Assert.Throws(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["bad_output_name"], [outputOrtValues[0].Value])); + Assert.Contains("Output name: 'bad_output_name' is not in the metadata", ex.Message); + } + + session.Dispose(); + } + + [Fact] + private void ThrowWrongOutputDimensionDotnetTensors() + { + var tuple = OpenSessionSqueezeNet(); + var session = tuple.Item1; + var inputData = tuple.Item2; + var inputTensor = tuple.Item3; + var outputTensor = DotnetTensors.Tensor.Create([1, 1001, 1, 1]); + + using (var runOptions = new RunOptions()) + using (var inputOrtValues = new DisposableListTest>(session.InputMetadata.Count)) + using (var outputOrtValues = new DisposableListTest>(session.OutputMetadata.Count)) + { + var tensor = DotnetTensors.Tensor.Create(inputData, Array.ConvertAll(inputTensor.Dimensions.ToArray(), x => (nint)x)); + + inputOrtValues.Add(new DisposableTestPair("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + outputOrtValues.Add(new DisposableTestPair("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor))); + + var ex = Assert.Throws(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], [outputOrtValues[0].Value])); + } + + session.Dispose(); + } + + [Fact] + private void ThrowInconsistentPinnedOutputsDotnetTensors() + { + var tuple = OpenSessionSqueezeNet(); + var session = tuple.Item1; + var inputData = tuple.Item2; + var inputTensor = tuple.Item3; + var outputTensor = DotnetTensors.Tensor.Create([1, 1001, 1, 1], [4]); + + using (var runOptions = new RunOptions()) + using (var inputOrtValues = new DisposableListTest>(session.InputMetadata.Count)) + using (var outputOrtValues = new DisposableListTest>(session.OutputMetadata.Count)) + { + var tensor = DotnetTensors.Tensor.Create(inputData, Array.ConvertAll(inputTensor.Dimensions.ToArray(), x => (nint)x)); + + inputOrtValues.Add(new DisposableTestPair("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor))); + outputOrtValues.Add(new DisposableTestPair("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor))); + OrtValue[] outputs = []; + var ex = Assert.Throws(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], outputs)); + Assert.StartsWith("Length of outputNames (1) must match that of outputValues (0).", ex.Message); + } + session.Dispose(); + } +#pragma warning restore SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback +#endif + + #if USE_CUDA [Fact(DisplayName = "TestCUDAProviderOptions")] private void TestCUDAProviderOptions() @@ -1416,6 +1613,25 @@ private void VerifyNativeMethodsExist() } } +#if NET8_0_OR_GREATER +#pragma warning disable SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + private void ValidateRunResultData(DotnetTensors.Tensor resultTensor, float[] expectedOutput, int[] expectedDimensions) + { + Assert.Equal(expectedDimensions.Length, resultTensor.Rank); + + var resultDimensions = resultTensor.Lengths; + for (int i = 0; i < expectedDimensions.Length; i++) + { + Assert.Equal(expectedDimensions[i], resultDimensions[i]); + } + + var resultArray = resultTensor.ToArray(); + Assert.Equal(expectedOutput.Length, resultArray.Length); + Assert.Equal(expectedOutput, resultArray, new FloatComparer()); + } +#pragma warning restore SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +#endif + static string GetTestModelsDir() { // get build directory, append downloaded models location diff --git a/csharp/tools/linux_pack/LinuxPackNativeNuget.csproj b/csharp/tools/linux_pack/LinuxPackNativeNuget.csproj index 098078d2e3683..b814f99b05ae1 100644 --- a/csharp/tools/linux_pack/LinuxPackNativeNuget.csproj +++ b/csharp/tools/linux_pack/LinuxPackNativeNuget.csproj @@ -7,7 +7,7 @@ If you need a more sophisticated package for testing, you can run the production packaging pipeline against your branch and download the resulting nuget package from the build artifacts. --> - + netstandard2.0 $(OnnxRuntimeBuildDirectory)/NativeNuget.nuspec