From 81b3852e232559bea8bc34b20da7374408a70696 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Fri, 25 Oct 2024 10:59:33 -0700 Subject: [PATCH 01/12] WIP to enable sending OTLP over UDS. --- .../Metrics/GenevaMetricExporter.cs | 56 +++++++++++++++++-- .../OtlpProtobufMetricExporter.cs | 28 ++++------ .../OtlpProtobuf/OtlpProtobufSerializer.cs | 24 ++++---- .../MetricUnixDomainSocketDataTransport.cs | 10 ++-- src/OpenTelemetry.Exporter.Geneva/README.md | 2 +- .../OtlpProtobufMetricExporterTests.cs | 51 ++++++++++++----- 6 files changed, 116 insertions(+), 55 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs index ed78dda6ee..2d3811b683 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 using System.Reflection; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using OpenTelemetry.Exporter.Geneva.Metrics; using OpenTelemetry.Internal; @@ -52,14 +53,38 @@ public GenevaMetricExporter(GenevaMetricExporterOptions options) if (connectionStringBuilder.PrivatePreviewEnableOtlpProtobufEncoding) { - var otlpProtobufExporter = new OtlpProtobufMetricExporter( - () => { return this.Resource; }, - connectionStringBuilder, - options.PrepopulatedMetricDimensions); +#if NET6_0_OR_GREATER + IMetricDataTransport transport = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? MetricUnixUserEventsDataTransport.Instance + : MetricWindowsEventTracingDataTransport.Instance; +#else + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new NotSupportedException("Exporting data in protobuf format is not supported on Linux."); + } + + var transport = MetricWindowsEventTracingDataTransport.Instance; +#endif - this.exporter = otlpProtobufExporter; + var response = this.InitializeOtlpProtobufExport(options, connectionStringBuilder, transport); - this.exportMetrics = otlpProtobufExporter.Export; + this.exporter = response.Exporter; + this.exportMetrics = response.ExportFunc; + } +#if !NET + else if (connectionStringBuilder.Endpoint.IndexOf("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase) >= 0) +#else + else if (connectionStringBuilder.Endpoint.Contains("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase)) +#endif + { + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); + + IMetricDataTransport transport = new MetricUnixDomainSocketDataTransport(unixDomainSocketPath); + + var response = this.InitializeOtlpProtobufExport(options, connectionStringBuilder, transport); + + this.exporter = response.Exporter; + this.exportMetrics = response.ExportFunc; } else { @@ -120,4 +145,23 @@ protected override void Dispose(bool disposing) #else private static Regex GetDisableRegexPattern() => new(DisableRegexPattern, RegexOptions.Compiled); #endif + + private (IDisposable Exporter, ExportMetricsFunc ExportFunc) InitializeOtlpProtobufExport( + GenevaMetricExporterOptions options, + ConnectionStringBuilder connectionStringBuilder, + IMetricDataTransport transport) + { + connectionStringBuilder.TryGetMetricsAccountAndNamespace( + out var metricsAccount, + out var metricsNamespace); + + var otlpProtobufExporter = new OtlpProtobufMetricExporter( + () => { return this.Resource; }, + transport, + metricsAccount, + metricsNamespace, + options.PrepopulatedMetricDimensions); + + return (otlpProtobufExporter, otlpProtobufExporter.Export); + } } diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs index 174e5ded5c..68f8d3087a 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 using System.Diagnostics; -using System.Runtime.InteropServices; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; @@ -18,29 +17,20 @@ internal sealed class OtlpProtobufMetricExporter : IDisposable public OtlpProtobufMetricExporter( Func getResource, - ConnectionStringBuilder connectionStringBuilder, + IMetricDataTransport transport, + string? metricsAccount, + string? metricsNamespace, IReadOnlyDictionary? prepopulatedMetricDimensions) { Debug.Assert(getResource != null, "getResource was null"); + Debug.Assert(transport != null, "transport was null"); this.getResource = getResource!; -#if NET6_0_OR_GREATER - IMetricDataTransport transport = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? MetricUnixUserEventsDataTransport.Instance - : MetricWindowsEventTracingDataTransport.Instance; -#else - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - throw new NotSupportedException("Exporting data in protobuf format is not supported on Linux."); - } - - var transport = MetricWindowsEventTracingDataTransport.Instance; -#endif - this.otlpProtobufSerializer = new OtlpProtobufSerializer( - transport, - connectionStringBuilder, + transport!, + metricsAccount, + metricsNamespace, prepopulatedMetricDimensions); } @@ -59,5 +49,9 @@ public ExportResult Export(in Batch batch) public void Dispose() { + if (this.otlpProtobufSerializer.MetricDataTransport is MetricUnixDomainSocketDataTransport udsTransport) + { + udsTransport.Dispose(); + } } } diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs index 73123d6b5f..f92cd6558d 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs @@ -1,6 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +using System.Buffers.Binary; using System.Diagnostics; using System.Globalization; using OpenTelemetry.Metrics; @@ -17,6 +18,7 @@ internal sealed class OtlpProtobufSerializer private readonly Dictionary> scopeMetrics = new(); private readonly string? metricNamespace; private readonly string? metricAccount; + private readonly bool prefixBufferWithUInt32LittleEndianLength; private readonly byte[]? prepopulatedNumberDataPointAttributes; private readonly int prepopulatedNumberDataPointAttributesLength; private readonly byte[]? prepopulatedHistogramDataPointAttributes; @@ -37,12 +39,16 @@ internal sealed class OtlpProtobufSerializer public OtlpProtobufSerializer( IMetricDataTransport metricDataTransport, - ConnectionStringBuilder? connectionStringBuilder, + string? metricsAccount, + string? metricsNamespace, IReadOnlyDictionary? prepopulatedMetricDimensions) { Debug.Assert(metricDataTransport != null, "metricDataTransport was null"); this.MetricDataTransport = metricDataTransport!; + this.metricAccount = metricsAccount; + this.metricNamespace = metricsNamespace; + this.prefixBufferWithUInt32LittleEndianLength = metricDataTransport is MetricUnixDomainSocketDataTransport; // Taking a arbitrary number here for writing attributes. byte[] temp = new byte[20000]; @@ -68,14 +74,6 @@ public OtlpProtobufSerializer( Array.Copy(temp, this.prepopulatedExponentialHistogramDataPointAttributes, cursor); this.prepopulatedExponentialHistogramDataPointAttributesLength = cursor; } - - if (connectionStringBuilder?.TryGetMetricsAccountAndNamespace( - out var metricsAccount, - out var metricsNamespace) == true) - { - this.metricAccount = metricsAccount; - this.metricNamespace = metricsNamespace; - } } internal static void WriteInstrumentDetails(byte[] buffer, ref int cursor, Metric metric) @@ -224,7 +222,7 @@ internal void SerializeResourceMetrics(byte[] buffer, Resource resource) { int cursor = 0; - this.resourceMetricTagAndLengthIndex = cursor; + this.resourceMetricTagAndLengthIndex = cursor + (this.prefixBufferWithUInt32LittleEndianLength ? 4 : 0); this.resourceMetricValueIndex = cursor + TagAndLengthSize; @@ -734,11 +732,15 @@ private void WriteIndividualMessageTagsAndLength(byte[] buffer, ref int cursor, // Write resource metric tag and length ProtobufSerializerHelper.WriteTagAndLengthPrefix(buffer, ref resourceMetricIndex, cursor - this.resourceMetricValueIndex, FieldNumberConstants.ResourceMetrics_resource, WireType.LEN); + + if (this.prefixBufferWithUInt32LittleEndianLength) + { + BinaryPrimitives.WriteUInt32LittleEndian(buffer, (uint)cursor); + } } private void SendMetricPoint(byte[] buffer, ref int cursor) { - // TODO: Extend this for user_events. this.MetricDataTransport.SendOtlpProtobufEvent(buffer, cursor); } diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricUnixDomainSocketDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricUnixDomainSocketDataTransport.cs index 4faa04560f..70856642c1 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricUnixDomainSocketDataTransport.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricUnixDomainSocketDataTransport.cs @@ -28,6 +28,11 @@ public void Send(MetricEventType eventType, byte[] body, int size) this.udsDataTransport.Send(body, size + this.fixedPayloadLength); } + public void SendOtlpProtobufEvent(byte[] body, int size) + { + this.udsDataTransport.Send(body, size); + } + public void Dispose() { if (this.isDisposed) @@ -38,9 +43,4 @@ public void Dispose() this.udsDataTransport.Dispose(); this.isDisposed = true; } - - public void SendOtlpProtobufEvent(byte[] body, int size) - { - throw new NotImplementedException(); - } } diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index 2ba0725906..090e66d0c6 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -279,7 +279,7 @@ specification](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.1.0 As of `1.10.0` `PrivatePreviewEnableOtlpProtobufEncoding=true` is also supported on Linux. On Linux when using `PrivatePreviewEnableOtlpProtobufEncoding=true` an `Endpoint` is **NOT** required to be provided on `ConnectionString`. For -example: `Endpoint=unix:Account={MetricAccount};Namespace={MetricNamespace}`. +example: `Account={MetricAccount};Namespace={MetricNamespace}`. > [!IMPORTANT] > When `PrivatePreviewEnableOtlpProtobufEncoding` is enabled on Linux metrics diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs index cc8d84b9cc..9d2ab5ba27 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs @@ -238,9 +238,11 @@ public void CounterSerializationSingleMetricPoint(string instrumentName, long? l var testTransport = new TestTransport(); - ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder($"Account={expectedAccount};Namespace={expectedNamespace}"); - - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, addAccountAndNamespace ? connectionStringBuilder : null, addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + addAccountAndNamespace ? expectedAccount : null, + addAccountAndNamespace ? expectedNamespace : null, + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -410,7 +412,8 @@ public void CounterSerializationMultipleMetricPoints(string instrumentName, long var buffer = new byte[65360]; var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -535,8 +538,12 @@ public void UpdownCounterSerializationSingleMetricPoint(string instrumentName, l var buffer = new byte[65360]; var testTransport = new TestTransport(); - ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder($"Account={expectedAccount};Namespace={expectedNamespace}"); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, addAccountAndNamespace ? connectionStringBuilder : null, addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + addAccountAndNamespace ? expectedAccount : null, + addAccountAndNamespace ? expectedNamespace : null, + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -656,7 +663,8 @@ public void UpdownCounterSerializationMultipleMetricPoints(string instrumentName var buffer = new byte[65360]; var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -811,8 +819,12 @@ public void HistogramSerializationSingleMetricPoint(double doubleValue, bool add var buffer = new byte[65360]; var testTransport = new TestTransport(); - ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder($"Account={expectedAccount};Namespace={expectedNamespace}"); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, addAccountAndNamespace ? connectionStringBuilder : null, addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + addAccountAndNamespace ? expectedAccount : null, + addAccountAndNamespace ? expectedNamespace : null, + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -982,7 +994,8 @@ public void HistogramSerializationMultipleMetricPoints(double[] doubleValues) var buffer = new byte[65360]; var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -1133,8 +1146,12 @@ public void GaugeSerializationSingleMetricPoint(string instrumentName, long? lon var buffer = new byte[65360]; var testTransport = new TestTransport(); - ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder($"Account={expectedAccount};Namespace={expectedNamespace}"); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, addAccountAndNamespace ? connectionStringBuilder : null, addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + addAccountAndNamespace ? expectedAccount : null, + addAccountAndNamespace ? expectedNamespace : null, + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -1264,7 +1281,8 @@ public void GaugeSerializationMultipleMetricPoints(string instrumentName, long[] var buffer = new byte[65360]; var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null); + + var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -1424,8 +1442,11 @@ public void ExponentialHistogramSerializationSingleMetricPoint(double? doubleVal var testTransport = new TestTransport(); - ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder($"Account={expectedAccount};Namespace={expectedNamespace}"); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, addAccountAndNamespace ? connectionStringBuilder : null, addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + addAccountAndNamespace ? expectedAccount : null, + addAccountAndNamespace ? expectedNamespace : null, + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); From 13626ea6fdd3bc2663a78f87e413788b41535e2e Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Fri, 25 Oct 2024 21:28:07 -0700 Subject: [PATCH 02/12] Bug fix. --- .../Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs index f92cd6558d..aaccef41e8 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs @@ -735,7 +735,7 @@ private void WriteIndividualMessageTagsAndLength(byte[] buffer, ref int cursor, if (this.prefixBufferWithUInt32LittleEndianLength) { - BinaryPrimitives.WriteUInt32LittleEndian(buffer, (uint)cursor); + BinaryPrimitives.WriteUInt32LittleEndian(buffer, (uint)cursor - 4); } } From 675b186ea66417af40e3a6ec4998518a8969e19c Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Sun, 27 Oct 2024 21:28:34 -0700 Subject: [PATCH 03/12] Bug fixes and tests. --- .../Metrics/GenevaMetricExporter.cs | 12 +- .../OtlpProtobufMetricExporter.cs | 3 +- .../OtlpProtobuf/OtlpProtobufSerializer.cs | 9 +- .../OtlpProtobufMetricExporterTests.cs | 113 ++++++++++++------ ...lpProtobufMetricExporterWithPrefixTests.cs | 12 ++ ...rotobufMetricExporterWithoutPrefixTests.cs | 12 ++ 6 files changed, 116 insertions(+), 45 deletions(-) create mode 100644 test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithPrefixTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithoutPrefixTests.cs diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs index 2d3811b683..2af76917c5 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs @@ -72,11 +72,19 @@ public GenevaMetricExporter(GenevaMetricExporterOptions options) this.exportMetrics = response.ExportFunc; } #if !NET - else if (connectionStringBuilder.Endpoint.IndexOf("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase) >= 0) + else if ( + connectionStringBuilder.Protocol == TransportProtocol.Unix + && connectionStringBuilder.Endpoint.IndexOf("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase) >= 0) #else - else if (connectionStringBuilder.Endpoint.Contains("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase)) + else if (connectionStringBuilder.Protocol == TransportProtocol.Unix + && connectionStringBuilder.Endpoint.Contains("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase)) #endif { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("Unix domain socket should not be used on Windows."); + } + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); IMetricDataTransport transport = new MetricUnixDomainSocketDataTransport(unixDomainSocketPath); diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs index 68f8d3087a..1931cf2dd4 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs @@ -31,7 +31,8 @@ public OtlpProtobufMetricExporter( transport!, metricsAccount, metricsNamespace, - prepopulatedMetricDimensions); + prepopulatedMetricDimensions, + prefixBufferWithUInt32LittleEndianLength: transport is MetricUnixDomainSocketDataTransport); } public ExportResult Export(in Batch batch) diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs index aaccef41e8..e837fc327c 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufSerializer.cs @@ -41,14 +41,15 @@ public OtlpProtobufSerializer( IMetricDataTransport metricDataTransport, string? metricsAccount, string? metricsNamespace, - IReadOnlyDictionary? prepopulatedMetricDimensions) + IReadOnlyDictionary? prepopulatedMetricDimensions, + bool prefixBufferWithUInt32LittleEndianLength = false) { Debug.Assert(metricDataTransport != null, "metricDataTransport was null"); this.MetricDataTransport = metricDataTransport!; this.metricAccount = metricsAccount; this.metricNamespace = metricsNamespace; - this.prefixBufferWithUInt32LittleEndianLength = metricDataTransport is MetricUnixDomainSocketDataTransport; + this.prefixBufferWithUInt32LittleEndianLength = prefixBufferWithUInt32LittleEndianLength; // Taking a arbitrary number here for writing attributes. byte[] temp = new byte[20000]; @@ -220,9 +221,9 @@ internal void ClearScopeMetrics() internal void SerializeResourceMetrics(byte[] buffer, Resource resource) { - int cursor = 0; + int cursor = this.prefixBufferWithUInt32LittleEndianLength ? 4 : 0; - this.resourceMetricTagAndLengthIndex = cursor + (this.prefixBufferWithUInt32LittleEndianLength ? 4 : 0); + this.resourceMetricTagAndLengthIndex = cursor; this.resourceMetricValueIndex = cursor + TagAndLengthSize; diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs index 9d2ab5ba27..4870b6aee1 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterTests.cs @@ -14,7 +14,7 @@ namespace OpenTelemetry.Exporter.Geneva.Tests; -public class OtlpProtobufMetricExporterTests +public abstract class OtlpProtobufMetricExporterTests { public TagList TagList; @@ -50,7 +50,7 @@ public class OtlpProtobufMetricExporterTests private TagList exemplarTagList; - public OtlpProtobufMetricExporterTests() + protected OtlpProtobufMetricExporterTests() { this.TagList = default; @@ -95,6 +95,8 @@ public OtlpProtobufMetricExporterTests() this.exemplarTagList.Add(new("zfilteredKey1", "zfilteredValue1")); } + protected abstract bool PrefixBufferWithUInt32LittleEndianLength { get; } + [Theory] [InlineData("longcounter", 123L, null, true, true, true, true)] [InlineData("longcounter", 123L, null, true, true, true, false)] @@ -242,15 +244,15 @@ public void CounterSerializationSingleMetricPoint(string instrumentName, long? l testTransport, addAccountAndNamespace ? expectedAccount : null, addAccountAndNamespace ? expectedNamespace : null, - addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); Assert.Single(testTransport.ExportedItems); - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[0]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[0]); Assert.Single(request.ResourceMetrics); @@ -413,7 +415,12 @@ public void CounterSerializationMultipleMetricPoints(string instrumentName, long var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + metricsAccount: null, + metricsNamespace: null, + prepopulatedMetricDimensions: null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -424,9 +431,8 @@ public void CounterSerializationMultipleMetricPoints(string instrumentName, long for (int i = 0; i < expectedMetricPoints; i++) { - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[i]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[i]); Assert.Single(request.ResourceMetrics); @@ -543,15 +549,15 @@ public void UpdownCounterSerializationSingleMetricPoint(string instrumentName, l testTransport, addAccountAndNamespace ? expectedAccount : null, addAccountAndNamespace ? expectedNamespace : null, - addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); Assert.Single(testTransport.ExportedItems); - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[0]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[0]); Assert.Single(request.ResourceMetrics); @@ -664,7 +670,12 @@ public void UpdownCounterSerializationMultipleMetricPoints(string instrumentName var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + metricsAccount: null, + metricsNamespace: null, + prepopulatedMetricDimensions: null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -675,9 +686,8 @@ public void UpdownCounterSerializationMultipleMetricPoints(string instrumentName for (int i = 0; i < expectedMetricPoints; i++) { - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[i]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[i]); Assert.Single(request.ResourceMetrics); @@ -824,15 +834,15 @@ public void HistogramSerializationSingleMetricPoint(double doubleValue, bool add testTransport, addAccountAndNamespace ? expectedAccount : null, addAccountAndNamespace ? expectedNamespace : null, - addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); Assert.Single(testTransport.ExportedItems); - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[0]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[0]); Assert.Single(request.ResourceMetrics); @@ -995,7 +1005,12 @@ public void HistogramSerializationMultipleMetricPoints(double[] doubleValues) var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + metricsAccount: null, + metricsNamespace: null, + prepopulatedMetricDimensions: null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -1005,9 +1020,8 @@ public void HistogramSerializationMultipleMetricPoints(double[] doubleValues) for (int i = 0; i < expectedMetricPointCount; i++) { - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[i]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[i]); Assert.NotNull(request.ResourceMetrics[0].Resource); @@ -1151,15 +1165,15 @@ public void GaugeSerializationSingleMetricPoint(string instrumentName, long? lon testTransport, addAccountAndNamespace ? expectedAccount : null, addAccountAndNamespace ? expectedNamespace : null, - addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); Assert.Single(testTransport.ExportedItems); - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[0]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[0]); Assert.Single(request.ResourceMetrics); @@ -1282,7 +1296,12 @@ public void GaugeSerializationMultipleMetricPoints(string instrumentName, long[] var testTransport = new TestTransport(); - var otlpProtobufSerializer = new OtlpProtobufSerializer(testTransport, null, null, null); + var otlpProtobufSerializer = new OtlpProtobufSerializer( + testTransport, + metricsAccount: null, + metricsNamespace: null, + prepopulatedMetricDimensions: null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, Resource.Empty, new Batch(exportedItems.ToArray(), exportedItems.Count)); @@ -1293,9 +1312,8 @@ public void GaugeSerializationMultipleMetricPoints(string instrumentName, long[] for (int i = 0; i < expectedMetricPoints; i++) { - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[i]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[i]); Assert.Single(request.ResourceMetrics); @@ -1446,15 +1464,15 @@ public void ExponentialHistogramSerializationSingleMetricPoint(double? doubleVal testTransport, addAccountAndNamespace ? expectedAccount : null, addAccountAndNamespace ? expectedNamespace : null, - addPrepopulatedDimensions ? prepopulatedMetricDimensions : null); + addPrepopulatedDimensions ? prepopulatedMetricDimensions : null, + prefixBufferWithUInt32LittleEndianLength: this.PrefixBufferWithUInt32LittleEndianLength); otlpProtobufSerializer.SerializeAndSendMetrics(buffer, meterProvider.GetResource(), new Batch(exportedItems.ToArray(), exportedItems.Count)); Assert.Single(testTransport.ExportedItems); - var request = new OtlpCollector.ExportMetricsServiceRequest(); - - request.MergeFrom(testTransport.ExportedItems[0]); + var request = this.AssertAndConvertExportedBlobToRequest( + testTransport.ExportedItems[0]); Assert.Single(request.ResourceMetrics); @@ -1644,6 +1662,25 @@ private static void AssertOtlpAttributeValue(object expected, OtlpCommon.AnyValu } } + private OtlpCollector.ExportMetricsServiceRequest AssertAndConvertExportedBlobToRequest( + byte[] blob) + { + var request = new OtlpCollector.ExportMetricsServiceRequest(); + + if (this.PrefixBufferWithUInt32LittleEndianLength) + { + var content = blob.AsSpan().Slice(4); + Assert.Equal((uint)content.Length, BitConverter.ToUInt32(blob, 0)); + request.MergeFrom(content); + } + else + { + request.MergeFrom(blob); + } + + return request; + } + private class TestTransport : IMetricDataTransport { public List ExportedItems = new(); diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithPrefixTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithPrefixTests.cs new file mode 100644 index 0000000000..9421c8de95 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithPrefixTests.cs @@ -0,0 +1,12 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.Tests; + +[Collection("OtlpProtobufMetricExporterTests")] +public class OtlpProtobufMetricExporterWithPrefixTests : OtlpProtobufMetricExporterTests +{ + protected override bool PrefixBufferWithUInt32LittleEndianLength => true; +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithoutPrefixTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithoutPrefixTests.cs new file mode 100644 index 0000000000..f21059620f --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/OtlpProtobufMetricExporterWithoutPrefixTests.cs @@ -0,0 +1,12 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.Tests; + +[Collection("OtlpProtobufMetricExporterTests")] +public class OtlpProtobufMetricExporterWithoutPrefixTests : OtlpProtobufMetricExporterTests +{ + protected override bool PrefixBufferWithUInt32LittleEndianLength => false; +} From ea20880cc48748091a8d31960d6e6a8a50b4e3f4 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Sun, 27 Oct 2024 22:15:47 -0700 Subject: [PATCH 04/12] Tweaks. --- .../Metrics/GenevaMetricExporter.cs | 82 ++++++++----------- 1 file changed, 33 insertions(+), 49 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs index 2af76917c5..8607744849 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs @@ -53,46 +53,49 @@ public GenevaMetricExporter(GenevaMetricExporterOptions options) if (connectionStringBuilder.PrivatePreviewEnableOtlpProtobufEncoding) { -#if NET6_0_OR_GREATER - IMetricDataTransport transport = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? MetricUnixUserEventsDataTransport.Instance - : MetricWindowsEventTracingDataTransport.Instance; -#else - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - throw new NotSupportedException("Exporting data in protobuf format is not supported on Linux."); - } + IMetricDataTransport transport; - var transport = MetricWindowsEventTracingDataTransport.Instance; -#endif + if (connectionStringBuilder.Protocol == TransportProtocol.Unix) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("Unix domain socket should not be used on Windows."); + } - var response = this.InitializeOtlpProtobufExport(options, connectionStringBuilder, transport); + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); - this.exporter = response.Exporter; - this.exportMetrics = response.ExportFunc; - } -#if !NET - else if ( - connectionStringBuilder.Protocol == TransportProtocol.Unix - && connectionStringBuilder.Endpoint.IndexOf("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase) >= 0) + transport = new MetricUnixDomainSocketDataTransport(unixDomainSocketPath); + } + else + { +#if NET6_0_OR_GREATER + transport = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? MetricUnixUserEventsDataTransport.Instance + : MetricWindowsEventTracingDataTransport.Instance; #else - else if (connectionStringBuilder.Protocol == TransportProtocol.Unix - && connectionStringBuilder.Endpoint.Contains("mdm_otlp.socket", StringComparison.OrdinalIgnoreCase)) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new NotSupportedException("Exporting data in protobuf format is not supported on Linux."); + } + + transport = MetricWindowsEventTracingDataTransport.Instance; #endif - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - throw new ArgumentException("Unix domain socket should not be used on Windows."); } - var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); + connectionStringBuilder.TryGetMetricsAccountAndNamespace( + out var metricsAccount, + out var metricsNamespace); - IMetricDataTransport transport = new MetricUnixDomainSocketDataTransport(unixDomainSocketPath); + var otlpProtobufExporter = new OtlpProtobufMetricExporter( + () => { return this.Resource; }, + transport, + metricsAccount, + metricsNamespace, + options.PrepopulatedMetricDimensions); - var response = this.InitializeOtlpProtobufExport(options, connectionStringBuilder, transport); + this.exportMetrics = otlpProtobufExporter.Export; - this.exporter = response.Exporter; - this.exportMetrics = response.ExportFunc; + this.exporter = otlpProtobufExporter; } else { @@ -153,23 +156,4 @@ protected override void Dispose(bool disposing) #else private static Regex GetDisableRegexPattern() => new(DisableRegexPattern, RegexOptions.Compiled); #endif - - private (IDisposable Exporter, ExportMetricsFunc ExportFunc) InitializeOtlpProtobufExport( - GenevaMetricExporterOptions options, - ConnectionStringBuilder connectionStringBuilder, - IMetricDataTransport transport) - { - connectionStringBuilder.TryGetMetricsAccountAndNamespace( - out var metricsAccount, - out var metricsNamespace); - - var otlpProtobufExporter = new OtlpProtobufMetricExporter( - () => { return this.Resource; }, - transport, - metricsAccount, - metricsNamespace, - options.PrepopulatedMetricDimensions); - - return (otlpProtobufExporter, otlpProtobufExporter.Export); - } } From 3193be503af82447ed2e2bcb72ce2b5f7af2063f Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Mon, 28 Oct 2024 11:10:46 -0700 Subject: [PATCH 05/12] Doc updates. --- src/OpenTelemetry.Exporter.Geneva/README.md | 43 +++++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index 090e66d0c6..26de570083 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -270,23 +270,42 @@ For example: ##### OtlpProtobufEncoding -On Windows set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the -`ConnectionString` to opt-in to the experimental feature for changing the -underlying serialization format to binary protobuf following the schema defined -in [OTLP +An experimental feature flag is available to opt-into changing the underlying +serialization format to binary protobuf following the schema defined in [OTLP specification](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.1.0/opentelemetry/proto/metrics/v1/metrics.proto). +###### Windows + +To send metric data over ETW in OTLP format set +`PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`: +`Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. + +###### Linux + As of `1.10.0` `PrivatePreviewEnableOtlpProtobufEncoding=true` is also supported -on Linux. On Linux when using `PrivatePreviewEnableOtlpProtobufEncoding=true` an -`Endpoint` is **NOT** required to be provided on `ConnectionString`. For -example: `Account={MetricAccount};Namespace={MetricNamespace}`. +on Linux. + +###### When using unix domain socket + +To send metric data over UDS in OTLP format set the `Endpoint` to use the +correct `OtlpSocketPath` path and set +`PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`: +`Endpoint=unix:{OTLP UDS +Path};Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. > [!IMPORTANT] -> When `PrivatePreviewEnableOtlpProtobufEncoding` is enabled on Linux metrics -> are written using -> [user_events](https://docs.kernel.org/trace/user_events.html). `user_events` -> are a newer feature of the Linux kernel and require a distro with the feature -> enabled. +> OTLP over UDS requires a different socket path than TLV over UDS. + +###### When using user_events + +> [!IMPORTANT] +> [user_events](https://docs.kernel.org/trace/user_events.html) are a newer +> feature of the Linux kernel and require a distro with the feature enabled. + +To send metric data over user_events in OTLP format do **NOT** specify an +`Endpoint` and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the +`ConnectionString`: +`Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. #### `MetricExportIntervalMilliseconds` (optional) From 636bc2b7515759e4b81cf6df2ffed389ca8aec29 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Mon, 28 Oct 2024 11:31:46 -0700 Subject: [PATCH 06/12] Code fixes. --- .../Exporter/MetricExporterBenchmarks.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/MetricExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/MetricExporterBenchmarks.cs index 0e45b72a37..70c343a0fc 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/MetricExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/MetricExporterBenchmarks.cs @@ -111,12 +111,12 @@ public void Setup() this.tlvMetricsExporter = new TlvMetricExporter(connectionStringBuilder, exporterOptions.PrepopulatedMetricDimensions); // Using test transport here with noop to benchmark just the serialization part. - this.otlpProtobufSerializer = new OtlpProtobufSerializer(new TestTransport(), null, null); + this.otlpProtobufSerializer = new OtlpProtobufSerializer(new TestTransport(), metricsAccount: null, metricsNamespace: null, prepopulatedMetricDimensions: null); var resourceBuilder = ResourceBuilder.CreateDefault().Clear() .AddAttributes(new[] { new KeyValuePair("TestResourceKey", "TestResourceValue") }); this.resource = resourceBuilder.Build(); - this.otlpProtobufMetricExporter = new OtlpProtobufMetricExporter(() => { return this.resource; }, null, null); + this.otlpProtobufMetricExporter = new OtlpProtobufMetricExporter(() => { return this.resource; }, new TestTransport(), metricsAccount: null, metricsNamespace: null, prepopulatedMetricDimensions: null); this.buffer = new byte[GenevaMetricExporter.BufferSize]; this.counterMetricPointWith3Dimensions = this.GenerateCounterMetricItemWith3Dimensions(out this.counterMetricDataWith3Dimensions); From 1abccb5503ff6baec25fcd9b18c74ba691bf1353 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Mon, 28 Oct 2024 11:39:58 -0700 Subject: [PATCH 07/12] CHANGELOG stub. --- .../CHANGELOG.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index e1fbedc7df..643ae3b5a9 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -5,14 +5,19 @@ * Drop support for .NET 6 as this target is no longer supported. ([#2117](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2117)) -* Added support for exporting metrics via - [user_events](https://docs.kernel.org/trace/user_events.html) on Linux when - OTLP protobuf encoding is enabled via the - `PrivatePreviewEnableOtlpProtobufEncoding=true` connection string switch. With - this, `PrivatePreviewEnableOtlpProtobufEncoding=true` is now supported on both - Widows and Linux. Windows uses ETW as transport, while Linux uses user_events - as transport. - ([#2113](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2113)) +* Added support for exporting metrics on Linux when OTLP protobuf encoding is + enabled via the `PrivatePreviewEnableOtlpProtobufEncoding=true` connection + string switch. `PrivatePreviewEnableOtlpProtobufEncoding=true` is now + supported on both Widows and Linux. + + * `user_events` transport: + [#2113](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2113). + + * Unix domain socket transport: + [#XXXX](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/XXXX). + + For configuration details see: + [OtlpProtobufEncoding](./README.md#otlpprotobufencoding). ## 1.9.0 From e939c7183be4b22d2893c757b7e32518012568c4 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Mon, 28 Oct 2024 11:43:09 -0700 Subject: [PATCH 08/12] CHANGELOG patch. --- src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 643ae3b5a9..0775b0bda9 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -14,7 +14,7 @@ [#2113](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2113). * Unix domain socket transport: - [#XXXX](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/XXXX). + [#2261](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2261). For configuration details see: [OtlpProtobufEncoding](./README.md#otlpprotobufencoding). From f238a510d3556f039ee47c870408520a1203e274 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Tue, 29 Oct 2024 12:45:19 -0700 Subject: [PATCH 09/12] Update src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md Co-authored-by: Cijo Thomas --- src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 0775b0bda9..7ca97b842c 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -8,7 +8,7 @@ * Added support for exporting metrics on Linux when OTLP protobuf encoding is enabled via the `PrivatePreviewEnableOtlpProtobufEncoding=true` connection string switch. `PrivatePreviewEnableOtlpProtobufEncoding=true` is now - supported on both Widows and Linux. + supported on both Windows and Linux. * `user_events` transport: [#2113](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2113). From 63211d863a2192f2b4c956463be1937d1fa65588 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Tue, 29 Oct 2024 13:32:49 -0700 Subject: [PATCH 10/12] Code review. --- src/OpenTelemetry.Exporter.Geneva/README.md | 42 ++++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index 26de570083..fcc4cb52e8 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -274,11 +274,43 @@ An experimental feature flag is available to opt-into changing the underlying serialization format to binary protobuf following the schema defined in [OTLP specification](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.1.0/opentelemetry/proto/metrics/v1/metrics.proto). +When using OTLP format `Account` and `Namespace` are **NOT** required to be set +on the `ConnectionString`. The recommended approach is to use OpenTelemetry +Resource instead: + +```csharp +using var meterProvider = Sdk.CreateMeterProviderBuilder() + // Other configuration not shown + .ConfigureResource(r => r.AddAttributes( + new Dictionary() + { + ["_microsoft_metrics_account"] = "MetricsAccountGoesHere", + ["_microsoft_metrics_namespace"] = "MetricsNamespaceGoesHere", + })) + .AddGenevaMetricExporter(options => + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + options.ConnectionString = "PrivatePreviewEnableOtlpProtobufEncoding=true"; + } + else + { + // Note: 1.10.0+ version required to use OTLP format on Linux + + // Use Unix domain socket mode + options.ConnectionString = "Endpoint=unix:{OTLP UDS Path};PrivatePreviewEnableOtlpProtobufEncoding=true"; + + // Use user_events mode (preferred but considered experimental) + // options.ConnectionString = "PrivatePreviewEnableOtlpProtobufEncoding=true"; + } + }) + .Build(); +``` + ###### Windows To send metric data over ETW in OTLP format set -`PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`: -`Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. +`PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`. ###### Linux @@ -290,8 +322,7 @@ on Linux. To send metric data over UDS in OTLP format set the `Endpoint` to use the correct `OtlpSocketPath` path and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`: -`Endpoint=unix:{OTLP UDS -Path};Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. +`Endpoint=unix:{OTLP UDS Path};PrivatePreviewEnableOtlpProtobufEncoding=true`. > [!IMPORTANT] > OTLP over UDS requires a different socket path than TLV over UDS. @@ -304,8 +335,7 @@ Path};Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOt To send metric data over user_events in OTLP format do **NOT** specify an `Endpoint` and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the -`ConnectionString`: -`Account={MetricAccount};Namespace={MetricNamespace};PrivatePreviewEnableOtlpProtobufEncoding=true`. +`ConnectionString`. #### `MetricExportIntervalMilliseconds` (optional) From c167d334e35b78c923ceddb2bae3c821ee06b49e Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Wed, 30 Oct 2024 10:06:23 -0700 Subject: [PATCH 11/12] Code review. --- src/OpenTelemetry.Exporter.Geneva/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index fcc4cb52e8..011c32dfce 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -300,7 +300,7 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder() // Use Unix domain socket mode options.ConnectionString = "Endpoint=unix:{OTLP UDS Path};PrivatePreviewEnableOtlpProtobufEncoding=true"; - // Use user_events mode (preferred but considered experimental) + // Use user_events mode (preferred but considered experimental as this is a new capability in Linux kernel) // options.ConnectionString = "PrivatePreviewEnableOtlpProtobufEncoding=true"; } }) From 311c4129a33ebabd7d6a75b5c3b40b8a6cc0209a Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Wed, 30 Oct 2024 10:08:28 -0700 Subject: [PATCH 12/12] Code review. --- src/OpenTelemetry.Exporter.Geneva/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md index 011c32dfce..2410808596 100644 --- a/src/OpenTelemetry.Exporter.Geneva/README.md +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -274,9 +274,9 @@ An experimental feature flag is available to opt-into changing the underlying serialization format to binary protobuf following the schema defined in [OTLP specification](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.1.0/opentelemetry/proto/metrics/v1/metrics.proto). -When using OTLP format `Account` and `Namespace` are **NOT** required to be set -on the `ConnectionString`. The recommended approach is to use OpenTelemetry -Resource instead: +When using OTLP protobuf encoding `Account` and `Namespace` are **NOT** required +to be set on the `ConnectionString`. The recommended approach is to use +OpenTelemetry Resource instead: ```csharp using var meterProvider = Sdk.CreateMeterProviderBuilder() @@ -295,7 +295,7 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder() } else { - // Note: 1.10.0+ version required to use OTLP format on Linux + // Note: 1.10.0+ version required to use OTLP protobuf encoding on Linux // Use Unix domain socket mode options.ConnectionString = "Endpoint=unix:{OTLP UDS Path};PrivatePreviewEnableOtlpProtobufEncoding=true"; @@ -309,7 +309,7 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder() ###### Windows -To send metric data over ETW in OTLP format set +To send metric data over ETW using OTLP protobuf encoding set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`. ###### Linux @@ -319,8 +319,8 @@ on Linux. ###### When using unix domain socket -To send metric data over UDS in OTLP format set the `Endpoint` to use the -correct `OtlpSocketPath` path and set +To send metric data over UDS using OTLP protobuf encoding set the `Endpoint` to +use the correct `OtlpSocketPath` path and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the `ConnectionString`: `Endpoint=unix:{OTLP UDS Path};PrivatePreviewEnableOtlpProtobufEncoding=true`. @@ -333,9 +333,9 @@ correct `OtlpSocketPath` path and set > [user_events](https://docs.kernel.org/trace/user_events.html) are a newer > feature of the Linux kernel and require a distro with the feature enabled. -To send metric data over user_events in OTLP format do **NOT** specify an -`Endpoint` and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the -`ConnectionString`. +To send metric data over user_events using OTLP protobuf encoding do **NOT** +specify an `Endpoint` and set `PrivatePreviewEnableOtlpProtobufEncoding=true` on +the `ConnectionString`. #### `MetricExportIntervalMilliseconds` (optional)