diff --git a/NLog.Targets.Network.sln b/NLog.Targets.Network.sln index 9ebbf56..c13daf3 100644 --- a/NLog.Targets.Network.sln +++ b/NLog.Targets.Network.sln @@ -1,18 +1,29 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.14.36414.22 d17.14 +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 18 +VisualStudioVersion = 18.5.11723.231 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.Network", "src\NLog.Targets.Network\NLog.Targets.Network.csproj", "{99EEF3AA-C3E7-C176-CDA5-ED7F3C4125D8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.HttpClient", "src\NLog.Targets.HttpClient\NLog.Targets.HttpClient.csproj", "{C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.Network.Tests", "tests\NLog.Targets.Network.Tests\NLog.Targets.Network.Tests.csproj", "{A1863F0C-6306-30C4-5DD5-70FA0B102107}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.HttpClient.Tests", "tests\NLog.Targets.HttpClient.Tests\NLog.Targets.HttpClient.Tests.csproj", "{D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8EC462FD-D22E-90A8-E5CE-7E832BA40C5D}" ProjectSection(SolutionItems) = preProject appveyor.yml = appveyor.yml README.md = README.md EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.OpenTelemetryHttp", "src\NLog.Targets.OpenTelemetryHttp\NLog.Targets.OpenTelemetryHttp.csproj", "{6C5EB13D-8029-04AD-0415-C75DB4FBE959}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NLog.Targets.OpenTelemetryHttp.Tests", "tests\NLog.Targets.OpenTelemetryHttp.Tests\NLog.Targets.OpenTelemetryHttp.Tests.csproj", "{1D918000-3071-BB27-FD3B-2342B4B6E1FB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -23,15 +34,39 @@ Global {99EEF3AA-C3E7-C176-CDA5-ED7F3C4125D8}.Debug|Any CPU.Build.0 = Debug|Any CPU {99EEF3AA-C3E7-C176-CDA5-ED7F3C4125D8}.Release|Any CPU.ActiveCfg = Release|Any CPU {99EEF3AA-C3E7-C176-CDA5-ED7F3C4125D8}.Release|Any CPU.Build.0 = Release|Any CPU + {C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F}.Release|Any CPU.Build.0 = Release|Any CPU {A1863F0C-6306-30C4-5DD5-70FA0B102107}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A1863F0C-6306-30C4-5DD5-70FA0B102107}.Debug|Any CPU.Build.0 = Debug|Any CPU {A1863F0C-6306-30C4-5DD5-70FA0B102107}.Release|Any CPU.ActiveCfg = Release|Any CPU {A1863F0C-6306-30C4-5DD5-70FA0B102107}.Release|Any CPU.Build.0 = Release|Any CPU + {D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B}.Release|Any CPU.Build.0 = Release|Any CPU + {6C5EB13D-8029-04AD-0415-C75DB4FBE959}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C5EB13D-8029-04AD-0415-C75DB4FBE959}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C5EB13D-8029-04AD-0415-C75DB4FBE959}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C5EB13D-8029-04AD-0415-C75DB4FBE959}.Release|Any CPU.Build.0 = Release|Any CPU + {1D918000-3071-BB27-FD3B-2342B4B6E1FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1D918000-3071-BB27-FD3B-2342B4B6E1FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1D918000-3071-BB27-FD3B-2342B4B6E1FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1D918000-3071-BB27-FD3B-2342B4B6E1FB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {99EEF3AA-C3E7-C176-CDA5-ED7F3C4125D8} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C1D5F2A3-7E4B-4C8D-9F6E-3B2A1D0E5C7F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A1863F0C-6306-30C4-5DD5-70FA0B102107} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {D2E4A1B3-8F5C-4D9E-A7B2-1C3E0F4D6A8B} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {6C5EB13D-8029-04AD-0415-C75DB4FBE959} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1D918000-3071-BB27-FD3B-2342B4B6E1FB} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {A8C623A8-9FF0-46A3-8098-E88489136CA2} + SolutionGuid = {70CA2AC6-E9C1-47B7-9A93-598B06A187B5} EndGlobalSection EndGlobal diff --git a/appveyor.yml b/appveyor.yml index 8c16855..f9f7de8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,15 +1,21 @@ version: 6.0.0.{build} -image: Visual Studio 2022 +image: Visual Studio 2026 configuration: Release platform: Any CPU skip_tags: true skip_branch_with_pr: true +environment: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_NOLOGO: true nuget: disable_publish_on_pr: true build_script: - ps: msbuild /t:restore,pack /p:Configuration=Release /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg /p:ContinuousIntegrationBuild=true /p:EmbedUntrackedSources=true /p:PublishRepositoryUrl=true /verbosity:minimal test_script: - ps: dotnet test tests\NLog.Targets.Network.Tests\NLog.Targets.Network.Tests.csproj +- ps: dotnet test tests\NLog.Targets.HttpClient.Tests\NLog.Targets.HttpClient.Tests.csproj +- ps: dotnet test tests\NLog.Targets.OpenTelemetryHttp.Tests\NLog.Targets.OpenTelemetryHttp.Tests.csproj artifacts: - path: '**\NLog.*.nupkg' - path: '**\NLog.*.snupkg' diff --git a/src/NLog.Targets.HttpClient/HttpClientTarget.cs b/src/NLog.Targets.HttpClient/HttpClientTarget.cs new file mode 100644 index 0000000..1e243f0 --- /dev/null +++ b/src/NLog.Targets.HttpClient/HttpClientTarget.cs @@ -0,0 +1,802 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Compression; + using System.Net; + using System.Net.Http; + using System.Net.Http.Headers; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using NLog.Config; + using NLog.Layouts; + + /// + /// Sends log events to an HTTP or HTTPS endpoint, with support for batching and compression. + /// + /// + /// See NLog Wiki + /// + /// Documentation on NLog Wiki + [Target("HttpClient")] + [Target("Http")] + public class HttpClientTarget : AsyncTaskTarget + { + private static readonly Encoding _utf8Encoding = new UTF8Encoding(false); // No PreAmble BOM + private readonly char[] _reusableEncodingBuffer = new char[40 * 1024]; // Avoid large-object-heap + private readonly StringBuilder _reusableEncodingBuilder = new StringBuilder(); + private MemoryStream _reusableMemoryStream = new MemoryStream(4096); + private volatile HttpClient? _httpClient; + private const int _httpClientLifeTimeTicks = 5 * 60 * 1000; + private volatile int _httpClientCreatedTicks = 0; +#if !NETFRAMEWORK || NET471_OR_GREATER + private readonly NLog.Internal.SslCertificateCache _sslCertificateCache = new NLog.Internal.SslCertificateCache(); +#endif + + /// + /// Initializes a new instance of the class with default settings. + /// + public HttpClientTarget() + { + RetryDelayMilliseconds = 2500; // Delay before retry when transient failures. Ex Rate-Limited responses (e.g. HTTP 429) + } + + /// + /// Gets or sets the EndPoint destination URL for HTTP requests. + /// + public Layout Url + { + get => _url; + set + { + if (ReferenceEquals(value, _url)) return; + _url = value; + SignalHttpClientReset(); + } + } + private Layout _url = Layout.Empty; + + /// + /// Gets or sets the HTTP method used for the request. + /// + /// Default: + public string HttpMethod + { + get => _httpMethod.ToString(); + set + { + var httpMethod = value?.Trim() ?? string.Empty; + if (string.Equals(httpMethod, nameof(System.Net.Http.HttpMethod.Post), StringComparison.OrdinalIgnoreCase) || string.IsNullOrEmpty(httpMethod)) + _httpMethod = System.Net.Http.HttpMethod.Post; + else if (string.Equals(httpMethod, nameof(System.Net.Http.HttpMethod.Get), StringComparison.OrdinalIgnoreCase)) + _httpMethod = System.Net.Http.HttpMethod.Get; + else + _httpMethod = new System.Net.Http.HttpMethod(httpMethod.ToUpperInvariant()); + } + } + private System.Net.Http.HttpMethod _httpMethod = System.Net.Http.HttpMethod.Post; + + /// + /// Get or sets the content-type header to use for the http-request. + /// + /// Default: application/json + public string ContentType + { + get => _contentType; + set + { + if (value == _contentType) return; + _contentType = string.IsNullOrWhiteSpace(value) ? "application/json" : value; + var isTextContentType = _contentType.IndexOf("text", StringComparison.OrdinalIgnoreCase) >= 0 || _contentType.IndexOf("json", StringComparison.OrdinalIgnoreCase) >= 0 || _contentType.IndexOf("xml", StringComparison.OrdinalIgnoreCase) >= 0; + _contentTypeHeader = new MediaTypeHeaderValue(_contentType) { CharSet = isTextContentType ? _utf8Encoding.WebName : null }; + } + } + private string _contentType = "application/json"; + private MediaTypeHeaderValue _contentTypeHeader = new MediaTypeHeaderValue("application/json") { CharSet = _utf8Encoding.WebName }; + + /// + /// Gets or sets whether HTTP persistent connections (Keep-Alive) are enabled. + /// + /// Default: + public bool KeepAlive + { + get => _keepAlive; + set + { + if (value == _keepAlive) return; + _keepAlive = value; + SignalHttpClientReset(); + } + } + private bool _keepAlive = true; + + /// + /// Get or sets whether to expect http 100-Continue behavior, where the client sends headers and expects the http-server to reply with http-status 100-continue before sending the http-request body. + /// + /// This can introduce additional latency for the http-request, especially when http-server does not support the protocol. + /// + public bool? Expect100Continue + { + get => _expect100Continue; + set + { + if (value == _expect100Continue) return; + _expect100Continue = value; + SignalHttpClientReset(); + } + } + private bool? _expect100Continue +#if NETFRAMEWORK + = false +#endif + ; + + /// + /// Gets or sets the line ending mode to use when batching log events. + /// + /// Remember to assign to enable batching. Has no effect when using = + public LineEndingMode LineEnding { get; set; } = LineEndingMode.LF; + + /// + /// Gets or sets whether batched log events are wrapped in a JSON array. (Overrides ) + /// + /// Default: (Remember to assign to enable batching) + public bool BatchAsJsonArray { get; set; } + + /// + /// Gets or sets the timeout duration, in seconds, for HTTP requests. + /// + /// Default: secs + public int SendTimeoutSeconds + { + get => _sendTimeoutSeconds; + set + { + if (value == _sendTimeoutSeconds) return; + _sendTimeoutSeconds = value; + SignalHttpClientReset(); + } + } + private int _sendTimeoutSeconds = 30; + + /// + /// Gets or sets the username used for HTTP authentication. + /// + /// Explicit Empty/Blank String means use default network credentials (NTLM Windows Authentication) + public Layout? NetworkUserName + { + get => _networkUserName; + set + { + if (ReferenceEquals(value, _networkUserName)) return; + _networkUserName = value; + SignalHttpClientReset(); + } + } + private Layout? _networkUserName; + + /// + /// Gets or sets the password used for HTTP authentication. + /// + /// Empty/Blank String means use default credentials + public Layout? NetworkPassword + { + get => _networkPassword; + set + { + if (ReferenceEquals(value, _networkPassword)) return; + _networkPassword = value; + SignalHttpClientReset(); + } + } + private Layout? _networkPassword; + +#if !NETFRAMEWORK || NET471_OR_GREATER + /// + /// Gets or sets the file path to a client SSL certificate for mutual TLS (mTLS) authentication. + /// + public Layout? SslCertificateFile + { + get => _sslCertificateFile; + set + { + if (ReferenceEquals(value, _sslCertificateFile)) return; + _sslCertificateFile = value; + SignalHttpClientReset(); + } + } + private Layout? _sslCertificateFile; + + /// + /// Gets or sets the password for the client SSL certificate specified by . + /// + public Layout? SslCertificatePassword + { + get => _sslCertificatePassword; + set + { + if (ReferenceEquals(value, _sslCertificatePassword)) return; + _sslCertificatePassword = value; + SignalHttpClientReset(); + } + } + private Layout? _sslCertificatePassword; +#endif + + /// + /// Gets or sets the maximum payload size (in bytes) before batched log events are split into multiple HTTP payloads. + /// + /// Default: bytes. Remember to assign to enable batching. + public int MaxPayloadSizeBytes { get; set; } = 40 * 1024; + + /// + /// Gets or sets the compression mode used for HTTP request payloads. (None / GZip / GZipFast) + /// + /// Default: + public HttpCompressionType Compress { get; set; } + + /// + /// Gets or sets the collection of header properties to be included in the http-request. + /// + [ArrayParameter(typeof(TargetPropertyWithContext), "header")] + public IList Headers { get; set; } = new List(); + + /// + /// Gets or sets the URL of the proxy server used for HTTP requests. + /// + /// Explicit Empty/Blank String means default proxy + public Layout? ProxyUrl + { + get => _proxyUrl; + set + { + if (ReferenceEquals(value, _proxyUrl)) return; + _proxyUrl = value; + SignalHttpClientReset(); + } + } + private Layout? _proxyUrl; + + /// + /// Gets or sets the username used when authenticating with the proxy server. + /// + public Layout? ProxyUser + { + get => _proxyUser; + set + { + if (ReferenceEquals(value, _proxyUser)) return; + _proxyUser = value; + SignalHttpClientReset(); + } + } + private Layout? _proxyUser; + + /// + /// Gets or sets the password used when authenticating with the proxy server. + /// + public Layout? ProxyPassword + { + get => _proxyPassword; + set + { + if (ReferenceEquals(value, _proxyPassword)) return; + _proxyPassword = value; + SignalHttpClientReset(); + } + } + private Layout? _proxyPassword; + + /// + protected override void InitializeTarget() + { + if (Url is null || ReferenceEquals(Url, Layout.Empty)) + throw new NLogConfigurationException($"{nameof(Url)} layout must be specified for {GetType()}"); + + string baseUrl = Url?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (!string.IsNullOrWhiteSpace(baseUrl)) + { + if (!Uri.TryCreate(baseUrl, UriKind.Absolute, out var _)) + throw new NLogConfigurationException($"Invalid {nameof(Url)} specified for {GetType()}: {baseUrl}"); + } + + var proxyUrl = ProxyUrl?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty; + if (!string.IsNullOrWhiteSpace(proxyUrl)) + { + if (!Uri.TryCreate(proxyUrl, UriKind.Absolute, out var _)) + throw new NLogConfigurationException($"Invalid {nameof(ProxyUrl)} specified for {GetType()}: {proxyUrl}"); + } + + base.InitializeTarget(); + } + + /// + protected override void CloseTarget() + { + var oldHttpClient = _httpClient; + _httpClient = null; + _httpClientCreatedTicks = 0; + oldHttpClient?.Dispose(); + base.CloseTarget(); + } + + /// + protected sealed override Task WriteAsyncTask(LogEventInfo logEvent, CancellationToken cancellationToken) + { + throw new NotSupportedException(); // Never called + } + + /// + protected override async Task WriteAsyncTask(IList logEvents, CancellationToken cancellationToken) + { + if (logEvents.Count == 0) + return; + + var output = _reusableMemoryStream; + try + { + int lastBatchSize = 0; + var httpContent = Compress == HttpCompressionType.None ? BuildChunk(output, logEvents, 0, out lastBatchSize) : GZipCompressChunk(output, logEvents, 0, out lastBatchSize); + { + using var _ = await HttpClientSendAsync(null, httpContent, cancellationToken).ConfigureAwait(false); + } + if (lastBatchSize != logEvents.Count) + { + await HttpSendBatchesAsync(output, lastBatchSize, logEvents, cancellationToken).ConfigureAwait(false); + } + } + catch + { + ResetReusableCompressStream(output); + throw; + } + + if (output.Capacity > 1024 * 1024) + { + ResetReusableCompressStream(output); + } + } + + private void ResetReusableCompressStream(MemoryStream oldStream) + { + if (ReferenceEquals(_reusableMemoryStream, oldStream)) + _reusableMemoryStream = new MemoryStream(4096); + oldStream.Dispose(); + } + + private async Task HttpSendBatchesAsync(MemoryStream output, int lastBatchSize, IList logEvents, CancellationToken cancellationToken) + { + int batchStartIndex = lastBatchSize; + while (batchStartIndex < logEvents.Count) + { + cancellationToken.ThrowIfCancellationRequested(); + var httpContent = Compress == HttpCompressionType.None ? BuildChunk(output, logEvents, batchStartIndex, out lastBatchSize) : GZipCompressChunk(output, logEvents, batchStartIndex, out lastBatchSize); + using var _ = await HttpClientSendAsync(null, httpContent, cancellationToken).ConfigureAwait(false); + batchStartIndex += lastBatchSize; + } + } + + /// + /// Send an HTTP request as an asynchronous operation. + /// + /// Support custom overrides of WriteAsyncTask, that calls with custom ByteArrayContent / StreamContent + /// Override the default + /// The contents of the HTTP message + /// The cancellation token to cancel operation. + /// HTTP response with status-code and data (Remember to Dispose the response) + protected async Task HttpClientSendAsync(Uri? url, HttpContent httpContent, CancellationToken cancellationToken) + { + var httpClient = ResetHttpClientIfNeeded(url); + + HttpStatusCode httpStatusCode = default(HttpStatusCode); + + try + { + using var httpRequest = new HttpRequestMessage(_httpMethod, string.Empty) { Content = httpContent }; + httpRequest.Content.Headers.ContentType = _contentTypeHeader; + + var startTickCount = Environment.TickCount; + + var httpResponseMessage = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false); + httpStatusCode = httpResponseMessage.StatusCode; + Common.InternalLogger.Debug("{0}: HTTP request completed after {1}ms with http-status-code {2}", this, (Environment.TickCount - startTickCount), (int)httpStatusCode); + + try + { + httpResponseMessage.EnsureSuccessStatusCode(); // Throw if not a success code to trigger retry + } + catch (HttpRequestException ex) + { +#if NET || NETSTANDARD2_1_OR_GREATER + if (httpStatusCode == HttpStatusCode.TooManyRequests || httpStatusCode == HttpStatusCode.RequestTimeout || ((int)httpStatusCode >= 500 && httpStatusCode != HttpStatusCode.NetworkAuthenticationRequired)) +#else + if ((int)httpStatusCode == 429 || httpStatusCode == HttpStatusCode.RequestTimeout || ((int)httpStatusCode >= 500 && (int)httpStatusCode != 511)) +#endif + { + // Retry 429 + 408 + 5xx (server errors, typically transient) + throw; + } + + // Swallow other failures (e.g. 400 Bad Request) without retrying + NLog.Common.InternalLogger.Error(ex, "{0}: HTTP request failed with status code {1}", this, (int)httpStatusCode); + } + return httpResponseMessage; + } + catch (Exception ex) + { + NLog.Common.InternalLogger.Error(ex, "{0}: HTTP request failed with status code {1}", this, (int)httpStatusCode); + if (httpStatusCode == 0 && HttpClientLifeTimeExpired(Environment.TickCount, 5000)) + SignalHttpClientReset(); // Reset HttpClient immediately on transport-level failures (e.g. DNS failure, network failure) to clear the stale HttpClient TCP connection pool. + throw; + } + } + + private HttpContent BuildChunk(MemoryStream output, IList logEvents, int startIndex, out int batchSize) + { + var newlineDelimiter = BatchAsJsonArray ? ", " : LineEnding.NewLineCharacters; + var endIndex = logEvents.Count; + + batchSize = 1; + + lock (_reusableEncodingBuilder) + { + try + { + var sb = _reusableEncodingBuilder; + sb.Length = 0; + + if (BatchAsJsonArray) + sb.Append('['); + + Layout.Render(logEvents[startIndex], sb); + if (sb.Length < MaxPayloadSizeBytes) + { + batchSize = endIndex - startIndex; + for (int i = startIndex + 1; i < endIndex; ++i) + { + var orgLength = sb.Length; + sb.Append(newlineDelimiter); + Layout.Render(logEvents[i], sb); + if (sb.Length >= MaxPayloadSizeBytes) + { + sb.Length = orgLength; // Remove last rendered log that caused overflow + batchSize = i - startIndex; + break; + } + } + } + + if (BatchAsJsonArray) + sb.Append(']'); + + EncodePayload(_utf8Encoding, sb, output); + return new ByteArrayContent(output.GetBuffer(), 0, (int)output.Position); + } + finally + { + if (_reusableEncodingBuilder.Length > _reusableEncodingBuffer.Length) + _reusableEncodingBuilder.Remove(0, _reusableEncodingBuilder.Length - 1); // Attempt soft clear that skips Large-Object-Heap (LOH) re-allocation + _reusableEncodingBuilder.Length = 0; + } + } + } + + private HttpContent GZipCompressChunk(MemoryStream output, IList logEvents, int startIndex, out int batchSize) + { + var newlineDelimiter = BatchAsJsonArray ? ", " : LineEnding.NewLineCharacters; + var endIndex = logEvents.Count; + batchSize = endIndex - startIndex; + + output.Position = 0; + output.SetLength(0); + var compressionLevel = Compress == HttpCompressionType.GZipFast ? CompressionLevel.Fastest : CompressionLevel.Optimal; + + using (var gzipStream = new GZipStream(output, compressionLevel, leaveOpen: true)) + using (var streamWriter = new StreamWriter(gzipStream, _utf8Encoding, 1024, leaveOpen: true)) + { + if (BatchAsJsonArray) + streamWriter.Write('['); + + for (int i = startIndex; i < endIndex; ++i) + { + if (i > startIndex) + { + if (output.Position >= MaxPayloadSizeBytes) + { + batchSize = i - startIndex; + break; + } + streamWriter.Write(newlineDelimiter); + } + + RenderLogEventForChunk(logEvents[i], streamWriter); + } + + if (BatchAsJsonArray) + streamWriter.Write(']'); + } + + var content = new ByteArrayContent(output.GetBuffer(), 0, (int)output.Position); + content.Headers.ContentEncoding.Add("gzip"); + return content; + } + + private void RenderLogEventForChunk(LogEventInfo logEvent, StreamWriter streamWriter) + { + lock (_reusableEncodingBuilder) + { + try + { + var sb = _reusableEncodingBuilder; + sb.Length = 0; + + Layout.Render(logEvent, sb); + + if (sb.Length < _reusableEncodingBuffer.Length) + { + lock (_reusableEncodingBuffer) + { + sb.CopyTo(0, _reusableEncodingBuffer, 0, sb.Length); + streamWriter.Write(_reusableEncodingBuffer, 0, sb.Length); + } + } + else + { + streamWriter.Write(sb.ToString()); + } + } + finally + { + if (_reusableEncodingBuilder.Length > _reusableEncodingBuffer.Length) + _reusableEncodingBuilder.Remove(0, _reusableEncodingBuilder.Length - 1); // Attempt soft clear that skips Large-Object-Heap (LOH) re-allocation + _reusableEncodingBuilder.Length = 0; + } + } + } + + private void EncodePayload(Encoding encoder, StringBuilder payload, MemoryStream output) + { + output.Position = 0; + + var totalLength = payload.Length; + lock (_reusableEncodingBuffer) + { + if (totalLength < _reusableEncodingBuffer.Length) + { + payload.CopyTo(0, _reusableEncodingBuffer, 0, totalLength); + var maxByteCount = ((encoder.GetMaxByteCount(totalLength) / 4096) + 1) * 4096; + output.SetLength(maxByteCount); + var byteCount = encoder.GetBytes(_reusableEncodingBuffer, 0, totalLength, output.GetBuffer(), 0); + output.SetLength(byteCount); + output.Position = byteCount; + } + else + { + var payloadString = payload.ToString(); + var maxByteCount = encoder.GetMaxByteCount(payloadString.Length); + output.SetLength(maxByteCount); + var byteCount = encoder.GetBytes(payloadString, 0, payloadString.Length, output.GetBuffer(), 0); + output.SetLength(byteCount); + output.Position = byteCount; + } + } + } + + private HttpClient ResetHttpClientIfNeeded(Uri? url) + { + var oldHttpClient = _httpClient; + + int nowTickCount = Environment.TickCount; + if (!HttpClientLifeTimeExpired(nowTickCount, _httpClientLifeTimeTicks) && oldHttpClient != null) + { + if (url is null || oldHttpClient.BaseAddress.Equals(url)) + return oldHttpClient; + } + + // HttpClient is intended to be long-lived, but DNS changes can cause it to fail. Periodically recycle it to mitigate this. + lock (_reusableEncodingBuffer) + { + oldHttpClient = _httpClient; + if (!HttpClientLifeTimeExpired(nowTickCount, _httpClientLifeTimeTicks) && oldHttpClient != null && (url is null || oldHttpClient.BaseAddress.Equals(url))) + return oldHttpClient; + + _httpClient = null; + oldHttpClient?.Dispose(); + _httpClient = oldHttpClient = CreateNewHttpClient(url); + _httpClientCreatedTicks = nowTickCount; + } + + return oldHttpClient; + } + + private bool HttpClientLifeTimeExpired(int nowTickCount, int lifetimeTicks) + { + var deltaTicks = nowTickCount - _httpClientCreatedTicks; + return deltaTicks > lifetimeTicks || deltaTicks < -lifetimeTicks; + } + + private void SignalHttpClientReset() + { + if (_httpClientCreatedTicks != 0) + NLog.Common.InternalLogger.Debug("{0}: Signal HttpClient reset after config change", this); + lock (_reusableEncodingBuffer) + { + _httpClientCreatedTicks = 0; + } + } + + private HttpClient CreateNewHttpClient(Uri? url) + { + var nullEvent = LogEventInfo.CreateNullEvent(); + + var baseAddress = url?.ToString() ?? Url?.Render(nullEvent); + if (_httpClientCreatedTicks == 0) + NLog.Common.InternalLogger.Info("{0}: Creating HttpClient for BaseAddress: {1}", this, baseAddress); + else + NLog.Common.InternalLogger.Debug("{0}: Creating HttpClient for BaseAddress: {1}", this, baseAddress); + if (!Uri.TryCreate(baseAddress, UriKind.Absolute, out var baseAddressUri)) + throw new NLogRuntimeException($"Invalid {nameof(Url)} specified for {GetType()}: {baseAddress}"); + + var handler = new HttpClientHandler(); + +#if !NETFRAMEWORK || NET471_OR_GREATER + if (SslCertificateFile != null) + { + var sslCertificateFile = SslCertificateFile.Render(nullEvent) ?? string.Empty; + if (!_sslCertificateCache.TryGetCertificate(sslCertificateFile, out var clientCertificates)) + { + var sslCertificatePassword = SslCertificatePassword?.Render(nullEvent) ?? string.Empty; + try + { + clientCertificates = _sslCertificateCache.LoadCertificate(sslCertificateFile, sslCertificatePassword); + } + catch (Exception ex) + { + Common.InternalLogger.Error(ex, "{0}: Failed loading SSL certificate from file: {1}", this, sslCertificateFile); + throw new NLogRuntimeException($"{GetType()}: Failed loading SSL certificate from file: {sslCertificateFile}", ex); + } + } + + if (clientCertificates?.Count > 0) + { + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.ClientCertificates.AddRange(clientCertificates); + } + handler.ServerCertificateCustomValidationCallback = static (message, certificate, chain, sslPolicyErrors) => + { + if (sslPolicyErrors == System.Net.Security.SslPolicyErrors.None) + return true; + + Common.InternalLogger.Warn("SSL certificate errors were encountered when establishing connection to the server: {0}, Certificate: {1}", sslPolicyErrors, certificate); + if (certificate is null) + return false; + + return true; + }; + } +#endif + + var networkUserName = NetworkUserName?.Render(nullEvent)?.Trim() ?? string.Empty; + if (NetworkUserName != null) + { + handler.PreAuthenticate = true; // Authorization header included upfront (instead of waiting for 401-challenge from server) to avoid extra round-trip and latency + if (string.IsNullOrWhiteSpace(networkUserName)) + { + handler.Credentials = CredentialCache.DefaultCredentials; + } + else + { + var networkPassword = NetworkPassword?.Render(nullEvent) ?? string.Empty; + handler.Credentials = new NetworkCredential(networkUserName, networkPassword); + } + } + + if (ProxyUrl != null) + { + var proxyAddress = ProxyUrl?.Render(nullEvent) ?? string.Empty; + var proxyUser = ProxyUser?.Render(nullEvent) ?? string.Empty; + var proxyPassword = ProxyPassword?.Render(nullEvent) ?? string.Empty; + handler.UseProxy = true; + handler.Proxy = CreateWebProxy(proxyAddress, proxyUser, proxyPassword); + } + else + { + handler.UseProxy = false; + } + + var newHttpClient = new HttpClient(handler) + { + BaseAddress = baseAddressUri, + }; + if (SendTimeoutSeconds > 0) + newHttpClient.Timeout = TimeSpan.FromSeconds(SendTimeoutSeconds); + + if (KeepAlive) + newHttpClient.DefaultRequestHeaders.Connection.Add("keep-alive"); + else + newHttpClient.DefaultRequestHeaders.ConnectionClose = true; // Closes TCP connection after each request (Disables HTTP Keep-Alive) + + if (Expect100Continue.HasValue) + newHttpClient.DefaultRequestHeaders.ExpectContinue = Expect100Continue.Value; + + foreach (var header in Headers) + { + var headerName = header.Name?.Trim(); + if (string.IsNullOrEmpty(headerName)) + continue; + var headerValue = header.Layout?.Render(nullEvent) ?? string.Empty; + if (string.IsNullOrWhiteSpace(headerValue) && !header.IncludeEmptyValue) + continue; + newHttpClient.DefaultRequestHeaders.TryAddWithoutValidation(headerName, headerValue); + } + + + return newHttpClient; + } + + private IWebProxy CreateWebProxy(string proxyAddress, string proxyUser, string proxyPassword) + { + if (string.IsNullOrEmpty(proxyAddress)) + return WebRequest.DefaultWebProxy; + + if (!Uri.TryCreate(proxyAddress, UriKind.Absolute, out var proxyUri)) + throw new NLogRuntimeException($"Invalid {nameof(ProxyUrl)} specified for {GetType()}: {proxyAddress}"); + + var proxy = new WebProxy(proxyUri); + if (string.IsNullOrEmpty(proxyUser)) + { + proxy.UseDefaultCredentials = true; + } + else + { + var cred = proxyUser.Split(new[] { '\\' }, StringSplitOptions.RemoveEmptyEntries); + proxy.Credentials = cred.Length == 1 + ? new NetworkCredential + { UserName = proxyUser, Password = proxyPassword } + : new NetworkCredential + { + Domain = cred[0], + UserName = cred[1], + Password = proxyPassword + }; + } + + return proxy; + } + } +} diff --git a/src/NLog.Targets.HttpClient/HttpCompressionType.cs b/src/NLog.Targets.HttpClient/HttpCompressionType.cs new file mode 100644 index 0000000..86ab018 --- /dev/null +++ b/src/NLog.Targets.HttpClient/HttpCompressionType.cs @@ -0,0 +1,54 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets +{ + /// + /// Type of compression for HTTP payload + /// + public enum HttpCompressionType + { + /// + /// No compression + /// + None = 0, + /// + /// GZip optimal compression + /// + GZip = 1, + /// + /// GZip fastest compression + /// + GZipFast = 2, + } +} diff --git a/src/NLog.Targets.HttpClient/NLog.Targets.HttpClient.csproj b/src/NLog.Targets.HttpClient/NLog.Targets.HttpClient.csproj new file mode 100644 index 0000000..a17713c --- /dev/null +++ b/src/NLog.Targets.HttpClient/NLog.Targets.HttpClient.csproj @@ -0,0 +1,92 @@ + + + + 18.0 + net471;netstandard2.0 + net471;netstandard2.0;netstandard2.1 + + 6.0.5 + Preview1 + 6.0.0.0 + + $(APPVEYOR_BUILD_NUMBER) + 0 + $(VersionPrefix).$(AppVeyorBuildNumber) + + NLog.Targets.HttpClient + NLog + NLog HttpClientTarget for sending log messages to HTTP Web-server using either HTTP or HTTPS with support for batching and compression. + NLog.Targets.HttpClient v$(ProductVersion) + $(ProductVersion) + Jarek Kowalski,Kim Christensen,Julian Verdurmen + $([System.DateTime]::Now.ToString(yyyy)) + Copyright (c) 2004-$(CurrentYear) NLog Project - https://nlog-project.org/ + + +Changelog: + +- Preview1 Release of HttpClientTarget. + + README.md + NLog;HTTP;HTTPS;HttpClient;logging;log + N.png + https://github.com/NLog/NLog.Targets.Network + BSD-3-Clause + git + https://github.com/NLog/NLog.Targets.Network.git + + true + NLog.snk + true + + true + true + true + enable + 9 + true + true + true + + + + NLog.Targets.HttpClient for .NET Standard 2.1 + + + + NLog.Targets.HttpClient for .NET Standard 2.0 + + + + NLog.Targets.HttpClient for .NET Framework 4.7.1 + true + + + + + + + + + + + + + + + + + + + $(Title) + + + + + + + + + + + diff --git a/src/NLog.Targets.HttpClient/NLog.snk b/src/NLog.Targets.HttpClient/NLog.snk new file mode 100644 index 0000000..ae6cb7d Binary files /dev/null and b/src/NLog.Targets.HttpClient/NLog.snk differ diff --git a/src/NLog.Targets.HttpClient/Properties/AssemblyInfo.cs b/src/NLog.Targets.HttpClient/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..d0994b7 --- /dev/null +++ b/src/NLog.Targets.HttpClient/Properties/AssemblyInfo.cs @@ -0,0 +1,47 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +using System; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security; + +[assembly: AssemblyCulture("")] +[assembly: CLSCompliant(true)] +[assembly: ComVisible(false)] +[assembly: InternalsVisibleTo("NLog.Targets.HttpClient.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ef8eab4fbdeb511eeb475e1659fe53f00ec1c1340700f1aa347bf3438455d71993b28b1efbed44c8d97a989e0cb6f01bcb5e78f0b055d311546f63de0a969e04cf04450f43834db9f909e566545a67e42822036860075a1576e90e1c43d43e023a24c22a427f85592ae56cac26f13b7ec2625cbc01f9490d60f16cfbb1bc34d9")] +[assembly: AllowPartiallyTrustedCallers] +#if !NET35 +[assembly: SecurityRules(SecurityRuleSet.Level1)] +#endif diff --git a/src/NLog.Targets.HttpClient/README.md b/src/NLog.Targets.HttpClient/README.md new file mode 100644 index 0000000..f811d17 --- /dev/null +++ b/src/NLog.Targets.HttpClient/README.md @@ -0,0 +1,125 @@ +# NLog.Targets.HttpClient + +[![Version](https://badge.fury.io/nu/NLog.Targets.HttpClient.svg)](https://www.nuget.org/packages/NLog.Targets.HttpClient) +[![AppVeyor](https://img.shields.io/appveyor/ci/NLog/NLog-Targets-Network/master.svg)](https://ci.appveyor.com/project/NLog/NLog-Targets-Network/branch/master) + +NLog `HttpClient` target for sending log events to an HTTP or HTTPS endpoint. + +* HTTP POST, GET, and custom HTTP methods +* Batching of multiple log events +* JSON array batching +* GZip compression +* Custom request headers +* HTTP authentication +* Client certificates (mTLS) +* Proxy servers + +If having trouble with output, then check [NLog InternalLogger](https://github.com/NLog/NLog/wiki/Internal-Logging) for clues. See also [Troubleshooting NLog](https://github.com/NLog/NLog/wiki/Logging-Troubleshooting). + +## Register Extension + +NLog will only recognize the type-alias `HttpClient` when loading from an `NLog.config` file after registering the extension: + +```xml + + + +``` + +Alternative - register from code using the [fluent configuration API](https://github.com/NLog/NLog/wiki/Fluent-Configuration-API): + +```csharp +LogManager.Setup().SetupExtensions(ext => { + ext.RegisterTarget(); +}); +``` + +## Configuration Example + +```xml + + + + + + + +``` + +## Parameters + +## Parameters + +| Parameter | Default | Description | +| ------------------------ | ------------------- | ----------------------------------------------------------------------------------| +| `url` | Required | Destination URL for HTTP requests. | +| `httpMethod` | `POST` | HTTP method used when sending requests. | +| `contentType` | `application/json` | Value of the HTTP Content-Type header. | +| `keepAlive` | `true` | Reuses TCP connections between requests. | +| `expect100Continue` | `false` | Controls HTTP 100-Continue behavior. | +| `lineEnding` | `LF` | Line separator used when batching log events. | +| `batchAsJsonArray` | `false` | Wraps batched log events in a JSON array. Disables `lineEnding` value | +| `sendTimeoutSeconds` | `30` | HTTP request timeout in seconds. | +| `networkUserName` | | Username for HTTP authentication. Empty value uses default system credentials. | +| `networkPassword` | | Password for HTTP authentication. | +| `sslCertificateFile` | | Client certificate file used for mutual TLS authentication. | +| `sslCertificatePassword` | | Password for the client certificate. | +| `maxPayloadSizeBytes` | `40960` | Max payload size before splitting into multiple HTTP requests. Remember `BatchSize` | +| `compress` | `None` | Payload compression mode (`None`, `GZip`, `GZipFast`). | +| `proxyUrl` | | Proxy server URL. | +| `proxyUser` | | Proxy authentication username. | +| `proxyPassword` | | Proxy authentication password. | +| `headers` | | Additional HTTP request headers. | +| `batchSize` | `1` | Maximum number of log events to send in a single HTTP payload. | +| `taskDelayMilliseconds` | `1` | Delay before processing queued log events. Higher value can improve batching | +| `taskTimeoutSeconds` | `150` | Maximum execution time in seconds before cancellation of HTTP request. | +| `retryCount` | `0` | Number of retry attempts for failed write operations. | +| `retryDelayMilliseconds` | `2500` | Initial delay before retry after failed request. Delay doubles for each retry. | +| `queueLimit` | `10000` | Maximum number of pending requests allowed in the internal queue. | +| `overflowAction` | `Discard` | Action taken when the internal request queue reaches its limit. | + + +## Custom Headers + +Additional HTTP headers can be configured: + +```xml + + +
+
+ + +``` + +## Client Certificates (mTLS) + +Mutual TLS authentication can be enabled using a client certificate: + +```xml + +``` + +## Retry Behavior + +The target treats the following status codes as transient failures, that can be retried: + +* 408 Request Timeout +* 429 Too Many Requests +* 5xx Server Errors + +Client-side failures such as `400 Bad Request` are not retried. + +## Notes + +* The target internally reuses HttpClient instance for connection pooling. +* HttpClient instance is periodically recycled every 5 mins to handle DNS changes. diff --git a/src/NLog.Targets.Network/NLog.Targets.Network.csproj b/src/NLog.Targets.Network/NLog.Targets.Network.csproj index 5fdbbba..0de8c41 100644 --- a/src/NLog.Targets.Network/NLog.Targets.Network.csproj +++ b/src/NLog.Targets.Network/NLog.Targets.Network.csproj @@ -1,7 +1,7 @@  - 17.0 + 18.0 net35;net46;net471;netstandard2.0 net35;net46;net471;netstandard2.0;netstandard2.1 @@ -80,7 +80,7 @@ Changelog: - + diff --git a/src/NLog.Targets.OpenTelemetryHttp/Internal/OtlpProtobufSerializer.cs b/src/NLog.Targets.OpenTelemetryHttp/Internal/OtlpProtobufSerializer.cs new file mode 100644 index 0000000..95bba6e --- /dev/null +++ b/src/NLog.Targets.OpenTelemetryHttp/Internal/OtlpProtobufSerializer.cs @@ -0,0 +1,747 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Internal +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.IO; + using System.Text; + using NLog.Layouts; + + /// + /// Serializes NLog log events into the OTLP ExportLogsServiceRequest protobuf binary format. + /// + /// + /// Implements manual protobuf encoding without any external proto library dependency. + /// The output is a complete ExportLogsServiceRequest message as defined in the + /// OpenTelemetry Log Data Model proto schema. + /// See OTLP logs.proto + /// + internal sealed class OtlpProtobufSerializer + { + private static readonly long UnixEpochTicks = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).Ticks; + + /// Gets or sets the layout for capturing the W3C TraceId for each LogRecord (Hexadecimal String: 32 chars). + public Layout? TraceId { get; set; } + + /// Gets or sets the layout for capturing the W3C SpanId for each LogRecord (Hexadecimal String: 16 chars). + public Layout? SpanId { get; set; } + + /// + /// Returns an that writes a single OTLP + /// + public OtlpLogRecordBuilder BeginRecord(MemoryStream output) + { + return new OtlpLogRecordBuilder(this, output); + } + + /// + /// Builds a ScopeLogs protobuf message containing the instrumentation scope and a single log record. + /// + private void BuildScopeLogs(MemoryStream stream, string scopeName, LogEventInfo logEvent, string logMessage, IEnumerable>? logProperties) + { + // ScopeLogs { InstrumentationScope scope = 1; repeated LogRecord log_records = 2 } + if (!string.IsNullOrEmpty(scopeName)) + { + // InstrumentationScope { string name = 1; string version = 2 } + var scopeNameMaxBytes = Encoding.UTF8.GetMaxByteCount(scopeName.Length); + using (BeginSubmessageField(stream, 1, scopeNameMaxBytes)) + { + WriteStringField(stream, 1, scopeName, scopeNameMaxBytes); + } + } + + using (BeginSubmessageField(stream, 2)) + { + BuildLogRecord(stream, logEvent, logMessage, logProperties); + } + } + + /// + /// Builds a LogRecord protobuf message for a single log event. + /// + private void BuildLogRecord(MemoryStream stream, LogEventInfo logEvent, string logMessage, IEnumerable>? logProperties) + { + // LogRecord proto field numbers: + // fixed64 time_unix_nano = 1 + // SeverityNumber severity_number = 2 + // string severity_text = 3 + // AnyValue body = 5 + // repeated KeyValue attributes = 6 + // bytes trace_id = 9 + // bytes span_id = 10 + var unixNano = ToUnixNano(logEvent.TimeStamp); + WriteFixed64Field(stream, 1, unixNano); + + var severityNumber = MapSeverityNumber(logEvent.Level); + if (severityNumber != 0) + { + WriteVarintField(stream, 2, (ulong)severityNumber); + WriteStringField(stream, 3, logEvent.Level.ToString()); + } + + // body = AnyValue { string string_value = 1 } + if (!string.IsNullOrEmpty(logMessage)) + { + BuildAnyValueString(stream, 5, logMessage); + } + + if (!string.IsNullOrEmpty(logEvent.LoggerName)) + { + WriteKeyStringValue(stream, 6, "LoggerName", logEvent.LoggerName); + } + + if (logEvent.Exception != null) + { + WriteKeyStringValue(stream, 6, "exception.type", logEvent.Exception.GetType().ToString()); + WriteKeyStringValue(stream, 6, "exception.message", logEvent.Exception.Message); + WriteKeyStringValue(stream, 6, "exception.stacktrace", logEvent.Exception.ToString()); + } + + if (logProperties != null) + { + var enumerator = logProperties.GetEnumerator(); + try + { + while (enumerator.MoveNext()) + { + var prop = enumerator.Current; + var key = prop.Key?.ToString(); + if (key is null || string.IsNullOrEmpty(key)) + continue; + + WriteKeyValue(stream, 6, key, prop.Value); + } + } + finally + { + (enumerator as IDisposable)?.Dispose(); + } + } + + // trace_id (field 9) and span_id (field 10) + var traceId = TraceId?.RenderValue(logEvent); + if (traceId.HasValue) + { + WriteTraceIdField(stream, 9, traceId.Value); + } + var spanId = SpanId?.RenderValue(logEvent); + if (spanId.HasValue) + { + WriteSpanIdField(stream, 10, spanId.Value); + } + } + + /// + /// Builder for incrementally constructing a single OTLP ResourceLogs entry. + /// Obtain via ; dispose to finalize. + /// + internal struct OtlpLogRecordBuilder : IDisposable + { + private readonly OtlpProtobufSerializer _serializer; + private readonly MemoryStream _stream; + private readonly SubmessageWriter _requestWriter; + private readonly SubmessageWriter _resourceWriter; + private bool _completed; + + internal OtlpLogRecordBuilder(OtlpProtobufSerializer serializer, MemoryStream stream) + { + _serializer = serializer; + _stream = stream; + _completed = false; + + // ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1 } + _requestWriter = BeginSubmessageField(stream, 1); + // ResourceLogs { Resource resource = 1; ... } + _resourceWriter = BeginSubmessageField(stream, 1); + } + + /// Writes a resource attribute KeyValue into the open Resource submessage. + public void AddResourceAttribute(string attributeName, string attributeValue) + { + WriteKeyStringValue(_stream, 1, attributeName, attributeValue); + } + + /// Closes the Resource submessage, writes ScopeLogs, and finalizes the OTLP message. + public void AddScopeLogs(string scopeName, LogEventInfo logEvent, string logMessage, IEnumerable>? logProperties = null) + { + if (_completed) return; + _completed = true; + + _resourceWriter.Dispose(); + + // ResourceLogs { ... repeated ScopeLogs scope_logs = 2 } + using (BeginSubmessageField(_stream, 2)) + { + _serializer.BuildScopeLogs(_stream, scopeName, logEvent, logMessage, logProperties); + } + + _requestWriter.Dispose(); + } + + public void Complete() + { + if (_completed) return; + _completed = true; + _resourceWriter.Dispose(); + _requestWriter.Dispose(); + } + + /// + public void Dispose() => Complete(); + } + + private static void BuildAnyValueString(MemoryStream stream, int fieldNumber, string value) + { + var maxByteCount = Encoding.UTF8.GetMaxByteCount(value.Length); + BuildAnyValueString(stream, fieldNumber, value, maxByteCount); + } + + private static void BuildAnyValueString(MemoryStream stream, int fieldNumber, string value, int maxByteCount) + { + if (string.IsNullOrEmpty(value)) + return; + + using (BeginSubmessageField(stream, fieldNumber, maxByteCount)) + { + WriteStringField(stream, 1, value, maxByteCount); + } + } + +#if NET8_0_OR_GREATER + /// + /// Formats into a stack-allocated char buffer via , + /// then encodes the result as UTF-8 directly into — no intermediate string allocation. + /// Falls back to a string allocation when the value is too wide for the stack buffer. + /// Using a generic constraint avoids boxing value-type implementations (e.g. DateTime, DateTimeOffset, Guid). + /// + private static void BuildAnyValueSpanFormattable(MemoryStream stream, int fieldNumber, T value, string? format) where T : ISpanFormattable + { + const int StackCharBufferSize = 128; + Span charBuf = stackalloc char[StackCharBufferSize]; + if (value.TryFormat(charBuf, out int charsWritten, format, System.Globalization.CultureInfo.InvariantCulture)) + { + var formatted = charBuf.Slice(0, charsWritten); + var maxByteCount = Encoding.UTF8.GetMaxByteCount(charsWritten); + using (BeginSubmessageField(stream, fieldNumber, maxByteCount)) + { + WriteStringFieldSpan(stream, 1, formatted, maxByteCount); + } + } + else + { + BuildAnyValueString(stream, fieldNumber, value.ToString(format, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); + } + } +#endif + + internal static void BuildAnyValue(MemoryStream stream, int fieldNumber, object? value) + { + // AnyValue { string string_value = 1; bool bool_value = 2; int64 int_value = 3; double double_value = 4; ArrayValue array_value = 5; KeyValueList kvlist_value = 6 } + if (value is Enum) + { + BuildAnyValueString(stream, fieldNumber, value?.ToString() ?? string.Empty); + return; + } + + if (value is IConvertible convertible) + { + switch (convertible.GetTypeCode()) + { + case TypeCode.Boolean: + // bool_value field: tag(1) + varint(1) = 2 bytes + using (BeginSubmessageField(stream, fieldNumber, 2)) + { + WriteVarintField(stream, 2, convertible.ToBoolean(System.Globalization.CultureInfo.InvariantCulture) ? 1UL : 0UL); + } + return; + case TypeCode.SByte: + case TypeCode.Int16: + case TypeCode.Int32: + case TypeCode.Int64: + // int_value field: negative values sign-extend to 10 varint bytes: tag(1) + 10 = 11 bytes + var varInt = unchecked((ulong)convertible.ToInt64(System.Globalization.CultureInfo.InvariantCulture)); + using (BeginSubmessageField(stream, fieldNumber, 11)) + { + WriteVarintField(stream, 3, varInt); + } + return; + case TypeCode.Byte: + case TypeCode.UInt16: + case TypeCode.UInt32: + case TypeCode.UInt64: + // int_value field: max ulong → 10 varint bytes: tag(1) + 10 = 11 bytes + var varUInt = convertible.ToUInt64(System.Globalization.CultureInfo.InvariantCulture); + using (BeginSubmessageField(stream, fieldNumber, 11)) + { + WriteVarintField(stream, 3, varUInt); + } + return; + case TypeCode.Single: + case TypeCode.Double: + case TypeCode.Decimal: + // double_value field: fixed64 → tag(1) + 8 bytes = 9 bytes + var doubleVal = convertible.ToDouble(System.Globalization.CultureInfo.InvariantCulture); + using (BeginSubmessageField(stream, fieldNumber, 9)) + { + WriteFixed64Field(stream, 4, unchecked((ulong)BitConverter.DoubleToInt64Bits(doubleVal))); + } + return; + case TypeCode.DateTime: +#if NET8_0_OR_GREATER + BuildAnyValueSpanFormattable(stream, fieldNumber, convertible.ToDateTime(System.Globalization.CultureInfo.InvariantCulture), "o"); +#else + BuildAnyValueString(stream, fieldNumber, convertible.ToDateTime(System.Globalization.CultureInfo.InvariantCulture).ToString("o", System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); +#endif + return; + default: + BuildAnyValueString(stream, fieldNumber, convertible.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); + return; + } + } + + if (value is DateTimeOffset dateTimeOffset) + { +#if NET8_0_OR_GREATER + BuildAnyValueSpanFormattable(stream, fieldNumber, dateTimeOffset, "o"); +#else + BuildAnyValueString(stream, fieldNumber, dateTimeOffset.ToString("o", System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); +#endif + return; + } + +#if NET8_0_OR_GREATER + if (value is ISpanFormattable spanFormattable) + { + BuildAnyValueSpanFormattable(stream, fieldNumber, spanFormattable, format: null); + return; + } +#endif + + if (value is IFormattable formattable) + { + BuildAnyValueString(stream, fieldNumber, formattable.ToString(null, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); + return; + } + + if (value is IList list) + { + using (BeginSubmessageField(stream, fieldNumber)) + { + BuildAnyValueArray(stream, list); + } + return; + } + + if (value is IDictionary dict) + { + using (BeginSubmessageField(stream, fieldNumber)) + { + BuildAnyValueKvList(stream, dict); + } + return; + } + + if (value is IEnumerable enumerable) + { + using (BeginSubmessageField(stream, fieldNumber)) + { + BuildAnyValueArray(stream, enumerable); + } + return; + } + + BuildAnyValueString(stream, fieldNumber, value?.ToString() ?? string.Empty); + } + + private static void BuildAnyValueKvList(MemoryStream stream, IDictionary dict) + { + // AnyValue { KeyValueList kvlist_value = 6 } + // KeyValueList { repeated KeyValue values = 1 } + using (BeginSubmessageField(stream, 6)) + { + var enumerator = dict.GetEnumerator(); + try + { + while (enumerator.MoveNext()) + { + var key = enumerator.Key?.ToString(); + if (key is null || string.IsNullOrEmpty(key)) + continue; + + WriteKeyValue(stream, 1, key, enumerator.Value); + } + } + finally + { + (enumerator as IDisposable)?.Dispose(); + } + } + } + + private static void BuildAnyValueArray(MemoryStream stream, IList list) + { + // AnyValue { ArrayValue array_value = 5 } + // ArrayValue { repeated AnyValue values = 1 } + using (BeginSubmessageField(stream, 5)) + { + for (int i = 0; i < list.Count; i++) + { + BuildAnyValue(stream, 1, list[i]); + } + } + } + + private static void BuildAnyValueArray(MemoryStream stream, IEnumerable enumerable) + { + // AnyValue { ArrayValue array_value = 5 } + // ArrayValue { repeated AnyValue values = 1 } + using (BeginSubmessageField(stream, 5)) + { + var enumerator = enumerable.GetEnumerator(); + try + { + while (enumerator.MoveNext()) + { + BuildAnyValue(stream, 1, enumerator.Current); + } + } + finally + { + (enumerator as IDisposable)?.Dispose(); + } + } + } + + private static void WriteKeyStringValue(MemoryStream parent, int fieldNumber, string key, string value) + { + // KeyValue { string key = 1; AnyValue value = 2 } + var keyMaxBytes = Encoding.UTF8.GetMaxByteCount(key.Length); + var valueMaxBytes = Encoding.UTF8.GetMaxByteCount(value.Length); + // +16 accounts for field tags and inner length-prefix bytes of the key string field and AnyValue wrapper + using (BeginSubmessageField(parent, fieldNumber, keyMaxBytes + valueMaxBytes + 16)) + { + WriteStringField(parent, 1, key, keyMaxBytes); + BuildAnyValueString(parent, 2, value, valueMaxBytes); + } + } + + private static void WriteKeyValue(MemoryStream parent, int fieldNumber, string key, object? value) + { + if (value is string stringValue) + { + WriteKeyStringValue(parent, fieldNumber, key, stringValue); + } + else + { + // KeyValue { string key = 1; AnyValue value = 2 } + var keyMaxBytes = Encoding.UTF8.GetMaxByteCount(key.Length); + // key string field: tag(1)+length(up to 2)+keyMaxBytes; AnyValue wrapper: tag(1)+length(1); largest scalar (int64/uint64): tag(1)+10 varint bytes + var maxByteCount = value is IConvertible ? keyMaxBytes + 16 : int.MaxValue; + using (BeginSubmessageField(parent, fieldNumber, maxByteCount)) + { + WriteStringField(parent, 1, key, keyMaxBytes); + BuildAnyValue(parent, 2, value); + } + } + } + + private static void WriteVarint(MemoryStream stream, ulong value) + { + while (value > 0x7F) + { + stream.WriteByte((byte)((value & 0x7F) | 0x80)); + value >>= 7; + } + stream.WriteByte((byte)value); + } + + private static void WriteTag(MemoryStream stream, int fieldNumber, int wireType) + { + WriteVarint(stream, (ulong)((fieldNumber << 3) | wireType)); + } + + private static void WriteVarintField(MemoryStream stream, int fieldNumber, ulong value) + { + WriteTag(stream, fieldNumber, 0); + WriteVarint(stream, value); + } + + private static void WriteFixed64Field(MemoryStream stream, int fieldNumber, ulong value) + { + WriteTag(stream, fieldNumber, 1); +#if NET || NETSTANDARD2_1_OR_GREATER + var pos = (int)stream.Position; + stream.SetLength(pos + 8); + System.Buffers.Binary.BinaryPrimitives.WriteUInt64LittleEndian(stream.GetBuffer().AsSpan(pos, 8), value); + stream.Position = pos + 8; +#else + for (int i = 0; i < 8; i++) + { + stream.WriteByte((byte)(value & 0xFF)); + value >>= 8; + } +#endif + } + + private static void WriteStringField(MemoryStream stream, int fieldNumber, string value) + { + WriteStringField(stream, fieldNumber, value, Encoding.UTF8.GetMaxByteCount(value.Length)); + } + + private static void WriteStringField(MemoryStream stream, int fieldNumber, string value, int maxByteCount) + { +#if NET || NETSTANDARD2_1_OR_GREATER + WriteStringFieldSpan(stream, fieldNumber, value.AsSpan(), maxByteCount); +#else + if (maxByteCount >= SubmessageWriter.MaxContentLength) + throw new InvalidOperationException($"String field {fieldNumber} is too large ({maxByteCount} bytes exceeds the {SubmessageWriter.MaxContentLength}-byte protobuf limit)."); + + // 1-byte length for short strings (≤127 bytes), 2-byte padded varint for medium, 4-byte for large. + // actualByteCount <= maxByteCount so the selected size always fits. + var varintSize = maxByteCount > SubmessageWriter.MaxContent2ByteThreshold ? 4 : (maxByteCount > 127 ? 2 : 1); + WriteTag(stream, fieldNumber, 2); + var varintPos = (int)stream.Position; + var contentStart = varintPos + varintSize; + stream.SetLength(contentStart + maxByteCount); + var buf = stream.GetBuffer(); + var length = Encoding.UTF8.GetBytes(value, 0, value.Length, buf, contentStart); + if (varintSize == 1) + { + buf[varintPos] = (byte)length; + } + else if (varintSize == 2) + { + buf[varintPos] = (byte)((length & 0x7F) | 0x80); + buf[varintPos + 1] = (byte)(length >> 7); + } + else + { + buf[varintPos] = (byte)((length & 0x7F) | 0x80); + buf[varintPos + 1] = (byte)(((length >> 7) & 0x7F) | 0x80); + buf[varintPos + 2] = (byte)(((length >> 14) & 0x7F) | 0x80); + buf[varintPos + 3] = (byte)(length >> 21); + } + stream.SetLength(contentStart + length); + stream.Position = contentStart + length; +#endif + } + +#if NET || NETSTANDARD2_1_OR_GREATER + private static void WriteStringFieldSpan(MemoryStream stream, int fieldNumber, ReadOnlySpan value, int maxByteCount) + { + if (maxByteCount >= SubmessageWriter.MaxContentLength) + throw new InvalidOperationException($"String field {fieldNumber} is too large ({maxByteCount} bytes exceeds the {SubmessageWriter.MaxContentLength}-byte protobuf limit)."); + + // 1-byte length for short strings (≤127 bytes), 2-byte padded varint for medium, 4-byte for large. + // actualByteCount <= maxByteCount so the selected size always fits. + var varintSize = maxByteCount > SubmessageWriter.MaxContent2ByteThreshold ? 4 : (maxByteCount > 127 ? 2 : 1); + WriteTag(stream, fieldNumber, 2); + var varintPos = (int)stream.Position; + var contentStart = varintPos + varintSize; + stream.SetLength(contentStart + maxByteCount); + var buf = stream.GetBuffer(); + var length = Encoding.UTF8.GetBytes(value, buf.AsSpan(contentStart)); + if (varintSize == 1) + { + buf[varintPos] = (byte)length; + } + else if (varintSize == 2) + { + buf[varintPos] = (byte)((length & 0x7F) | 0x80); + buf[varintPos + 1] = (byte)(length >> 7); + } + else + { + buf[varintPos] = (byte)((length & 0x7F) | 0x80); + buf[varintPos + 1] = (byte)(((length >> 7) & 0x7F) | 0x80); + buf[varintPos + 2] = (byte)(((length >> 14) & 0x7F) | 0x80); + buf[varintPos + 3] = (byte)(length >> 21); + } + stream.SetLength(contentStart + length); + stream.Position = contentStart + length; + } +#endif + + private static SubmessageWriter BeginSubmessageField(MemoryStream stream, int fieldNumber, int maxByteCount = int.MaxValue) + { + return new SubmessageWriter(stream, fieldNumber, maxByteCount); + } + + private readonly struct SubmessageWriter : IDisposable + { + // Protobuf allows non-minimal (padded) varints — decoders must accept them. + // fixedLength=false reserves 2 bytes (up to 16,383 bytes content) for small leaf nodes. + // fixedLength=true reserves 4 bytes (up to ~256 MB content) for large containers. + internal const int MaxContentLength = (1 << 28) - 1; + internal const int MaxContent2ByteLength = (1 << 14) - 1; // 16,383 bytes — the true 2-byte padded varint limit + internal const int MaxContent2ByteThreshold = MaxContent2ByteLength - 16; // Safety margin for tag + length-prefix overhead of nested fields + + private readonly MemoryStream _stream; + private readonly ulong _tagValue; + private readonly int _headerPos; + private readonly int _tagSize; + private readonly int _paddedLengthSize; + + internal SubmessageWriter(MemoryStream stream, int fieldNumber, int maxByteCount) + { + _tagValue = (ulong)((fieldNumber << 3) | 2); + _tagSize = _tagValue < 0x80 ? 1 : 2; // field 1-15 = 1 byte tag; field 16+ = 2 bytes + _stream = stream; + _headerPos = (int)stream.Position; + _paddedLengthSize = maxByteCount <= 127 ? 1 : (maxByteCount <= MaxContent2ByteThreshold ? 2 : 4); + stream.Position = _headerPos + _tagSize + _paddedLengthSize; + } + + public void Dispose() + { + var endPos = (int)_stream.Position; + var contentLength = endPos - _headerPos - _tagSize - _paddedLengthSize; + if (contentLength <= 0) + { + _stream.Position = _headerPos; + _stream.SetLength(_headerPos); + return; + } + + WriteHeader(contentLength); + } + + private void WriteHeader(int contentLength) + { +#if NET || NETSTANDARD2_1_OR_GREATER + var header = _stream.GetBuffer().AsSpan(_headerPos); + WriteTagToBuffer(header, _tagValue, 0); + header = header.Slice(_tagSize, _paddedLengthSize); + const int offset = 0; +#else + var header = _stream.GetBuffer(); + WriteTagToBuffer(header, _tagValue, _headerPos); + int offset = _headerPos + _tagSize; +#endif + if (_paddedLengthSize == 1) + { + if (contentLength > 127) + throw new InvalidOperationException($"Protobuf content length {contentLength} exceeds the 1-byte varint limit of 127 bytes."); + + header[offset] = (byte)contentLength; + } + else if (_paddedLengthSize == 2) + { + if (contentLength > MaxContent2ByteLength) + throw new InvalidOperationException($"Protobuf content length {contentLength} exceeds the 2-byte padded varint limit of {MaxContent2ByteLength} bytes."); + + // Encode as exactly 2 varint bytes (padded) — valid per the protobuf spec + header[offset] = (byte)((contentLength & 0x7F) | 0x80); + header[offset + 1] = (byte)(contentLength >> 7); + } + else + { + if (contentLength >= MaxContentLength) + throw new InvalidOperationException($"Protobuf content length {contentLength} exceeds the 4-byte padded varint limit of {MaxContentLength} bytes."); + + // Encode as exactly 4 varint bytes (padded) — valid per the protobuf spec + header[offset] = (byte)((contentLength & 0x7F) | 0x80); + header[offset + 1] = (byte)(((contentLength >> 7) & 0x7F) | 0x80); + header[offset + 2] = (byte)(((contentLength >> 14) & 0x7F) | 0x80); + header[offset + 3] = (byte)(contentLength >> 21); + } + } + +#if NET || NETSTANDARD2_1_OR_GREATER + private static void WriteTagToBuffer(Span buffer, ulong value, int offset) +#else + private static void WriteTagToBuffer(byte[] buffer, ulong value, int offset) +#endif + { + while (value > 0x7F) + { + buffer[offset++] = (byte)((value & 0x7F) | 0x80); + value >>= 7; + } + buffer[offset] = (byte)value; + } + } + + private static void WriteTraceIdField(MemoryStream stream, int fieldNumber, System.Diagnostics.ActivityTraceId traceId) + { + if (traceId == default) + return; + + WriteTag(stream, fieldNumber, 2); + WriteVarint(stream, 16); // TraceId = 16 bytes + + var pos = (int)stream.Position; + stream.SetLength(pos + 16); + + traceId.CopyTo(stream.GetBuffer().AsSpan(pos, 16)); + + stream.Position = pos + 16; + } + + private static void WriteSpanIdField(MemoryStream stream, int fieldNumber, System.Diagnostics.ActivitySpanId spanId) + { + if (spanId == default) + return; + + WriteTag(stream, fieldNumber, 2); + WriteVarint(stream, 8); // SpanId = 8 bytes + + var pos = (int)stream.Position; + stream.SetLength(pos + 8); + + spanId.CopyTo(stream.GetBuffer().AsSpan(pos, 8)); + + stream.Position = pos + 8; + } + + internal static ulong ToUnixNano(DateTime timestamp) + { + var utcTicks = timestamp.ToUniversalTime().Ticks - UnixEpochTicks; + return utcTicks > 0 ? (ulong)utcTicks * 100UL : 0UL; + } + + internal static int MapSeverityNumber(LogLevel? level) + { + if (level == LogLevel.Trace) return 1; // SEVERITY_NUMBER_TRACE + if (level == LogLevel.Debug) return 5; // SEVERITY_NUMBER_DEBUG + if (level == LogLevel.Info) return 9; // SEVERITY_NUMBER_INFO + if (level == LogLevel.Warn) return 13; // SEVERITY_NUMBER_WARN + if (level == LogLevel.Error) return 17; // SEVERITY_NUMBER_ERROR + if (level == LogLevel.Fatal) return 21; // SEVERITY_NUMBER_FATAL + return 0; + } + } +} diff --git a/src/NLog.Targets.OpenTelemetryHttp/NLog.Targets.OpenTelemetryHttp.csproj b/src/NLog.Targets.OpenTelemetryHttp/NLog.Targets.OpenTelemetryHttp.csproj new file mode 100644 index 0000000..85f6955 --- /dev/null +++ b/src/NLog.Targets.OpenTelemetryHttp/NLog.Targets.OpenTelemetryHttp.csproj @@ -0,0 +1,97 @@ + + + + 18.0 + net471;netstandard2.0 + net471;netstandard2.0;netstandard2.1;net8.0 + net471;netstandard2.0;netstandard2.1;net8.0;net10.0 + + 6.0.5 + Preview1 + 6.0.0.0 + + $(APPVEYOR_BUILD_NUMBER) + 0 + $(VersionPrefix).$(AppVeyorBuildNumber) + + NLog.Targets.OpenTelemetryHttp + NLog + NLog OpenTelemetryHttpTarget for sending log messages to an OpenTelemetry OTLP endpoint using protobuf encoding over HTTP. + NLog.Targets.OpenTelemetryHttp v$(ProductVersion) + $(ProductVersion) + Jarek Kowalski,Kim Christensen,Julian Verdurmen + $([System.DateTime]::Now.ToString(yyyy)) + Copyright (c) 2004-$(CurrentYear) NLog Project - https://nlog-project.org/ + + +Changelog: + +- Preview1 Release of OpenTelemetryHttpTarget. + + README.md + NLog;OpenTelemetry;OTLP;logging;log + N.png + https://github.com/NLog/NLog.Targets.Network + BSD-3-Clause + git + https://github.com/NLog/NLog.Targets.Network.git + + true + NLog.snk + true + + true + true + true + enable + 9 + true + true + true + + + + NLog.Targets.OpenTelemetryHttp for .NET Standard 2.1 + + + + NLog.Targets.OpenTelemetryHttp for .NET Standard 2.0 + + + + NLog.Targets.OpenTelemetryHttp for .NET Framework 4.7.1 + true + + + + + + + + + + + + + + + + + + + + + + + $(Title) + + + + + + + + + + + diff --git a/src/NLog.Targets.OpenTelemetryHttp/NLog.snk b/src/NLog.Targets.OpenTelemetryHttp/NLog.snk new file mode 100644 index 0000000..ae6cb7d Binary files /dev/null and b/src/NLog.Targets.OpenTelemetryHttp/NLog.snk differ diff --git a/src/NLog.Targets.OpenTelemetryHttp/OpenTelemetryHttpTarget.cs b/src/NLog.Targets.OpenTelemetryHttp/OpenTelemetryHttpTarget.cs new file mode 100644 index 0000000..ca59b4e --- /dev/null +++ b/src/NLog.Targets.OpenTelemetryHttp/OpenTelemetryHttpTarget.cs @@ -0,0 +1,511 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Compression; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using NLog.Config; + using NLog.Internal; + using NLog.Layouts; + + /// + /// Sends log messages to an OpenTelemetry OTLP endpoint using protobuf encoding over HTTP. + /// + /// + /// + /// The target serializes log events into the OTLP ExportLogsServiceRequest protobuf format + /// and sends them via HTTP POST. Typical endpoint URL is http://localhost:4318/v1/logs. + /// + /// + /// Supports standard OpenTelemetry environment variables as fallback defaults: + /// OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, + /// OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_HEADERS, + /// OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, + /// OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, + /// OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_LOGS_PROTOCOL, + /// OTEL_SERVICE_NAME, OTEL_RESOURCE_ATTRIBUTES. + /// Environment variables are resolved using NLog ${environment} layout renderer. + /// + /// See OTLP/HTTP Specification + /// See OTLP Exporter Configuration + /// + [Target("OpenTelemetry")] + [Target("OpenTelemetryHttp")] + public class OpenTelemetryHttpTarget : HttpClientTarget + { + private static readonly SimpleLayout _defaultServiceName = new SimpleLayout("${appdomain:format=Friendly}"); + private static readonly string EmptyTraceIdToHexString = default(System.Diagnostics.ActivityTraceId).ToHexString(); + private static readonly string EmptySpanIdToHexString = default(System.Diagnostics.ActivitySpanId).ToHexString(); + + private OtlpProtobufSerializer _serializer = new OtlpProtobufSerializer(); + private Uri? _logsUrl; + private readonly Stack _memoryStreamPool = new Stack(); + private List? _activeChunkStreams = new List(); + + /// + /// Initializes a new instance of the class. + /// + public OpenTelemetryHttpTarget() + { + ContentType = "application/x-protobuf"; // application/json not supported, only protobuf as OTLP_PROTOCOL + IncludeEventProperties = true; + BatchSize = 200; // Consider doubling this if enabling compression + TaskDelayMilliseconds = 50; // Small delay to improve chance of batching and reducing number of HTTP requests. + RetryDelayMilliseconds = 2500; // Notice OTel SDK initial backoff is 5 secs + RetryCount = 3; // Notice OTel SDK default max 5 retries + } + + /// + /// Gets or sets the OTLP resource service.name attribute value. + /// + /// Default: ${appdomain:format=Friendly}. Defaults to Unknown when empty value + public Layout ServiceName { get; set; } = _defaultServiceName; + + /// + /// Gets or sets the OpenTelemetry instrumentation scope name. + /// + /// Default: + public string ScopeName { get; set; } = "NLog"; + + /// + /// Gets or sets additional OpenTelemetry resource attributes. + /// These attributes are included with every exported OTLP log record. + /// + [ArrayParameter(typeof(TargetPropertyWithContext), "resourceattribute")] + public IList ResourceAttributes { get; set; } = new List(); + + /// + /// Gets or sets the layout used to populate the OTLP LogRecord TraceId field. + /// + /// Default: ${activity:property=TraceId} — captures Activity.Current TraceId on the logging thread. + public Layout? TraceId { get; set; } = Layout.FromMethod(static evt => System.Diagnostics.Activity.Current?.TraceId is System.Diagnostics.ActivityTraceId activityTraceId && !ReferenceEquals(EmptyTraceIdToHexString, activityTraceId.ToHexString()) ? activityTraceId : null); + + /// + /// Gets or sets the layout used to populate the OTLP LogRecord SpanId field. + /// + /// Default: ${activity:property=SpanId} — captures Activity.Current SpanId on the logging thread. + public Layout? SpanId { get; set; } = Layout.FromMethod(static evt => System.Diagnostics.Activity.Current?.SpanId is System.Diagnostics.ActivitySpanId activitySpanId && !ReferenceEquals(EmptySpanIdToHexString, activitySpanId.ToHexString()) ? activitySpanId : null); + + /// + protected override void InitializeTarget() + { + ResolveOtlpEnvironmentVariables(); + _serializer = new OtlpProtobufSerializer + { + TraceId = TraceId, + SpanId = SpanId, + }; + _logsUrl = null; + base.InitializeTarget(); + } + + /// + protected override async Task WriteAsyncTask(IList logEvents, CancellationToken cancellationToken) + { + if (logEvents.Count == 0) + return; + + if (_logsUrl is null) + { + var urlStr = RenderLogEvent(Url, logEvents[0]); + if (!urlStr.EndsWith("/logs/", StringComparison.OrdinalIgnoreCase) && !urlStr.EndsWith("/logs", StringComparison.OrdinalIgnoreCase)) + urlStr = urlStr.TrimEnd('/') + "/v1/logs"; + if (!Uri.TryCreate(urlStr, UriKind.Absolute, out _logsUrl)) + NLog.Common.InternalLogger.Warn("{0}: Invalid OTLP endpoint URL: {1}", this, urlStr); + } + + var chunks = Interlocked.Exchange(ref _activeChunkStreams, null) ?? new List(); + try + { + if (Compress != HttpCompressionType.None) + await WriteCompressedAsync(logEvents, chunks, cancellationToken).ConfigureAwait(false); + else + await WriteRawAsync(logEvents, chunks, cancellationToken).ConfigureAwait(false); + } + finally + { + for (int i = 0; i < chunks.Count; i++) + ReturnMemoryStream(chunks[i]); + chunks.Clear(); + Interlocked.Exchange(ref _activeChunkStreams, chunks); + } + } + + private async Task WriteRawAsync(IList logEvents, List chunks, CancellationToken cancellationToken) + { + List>? pendingTasks = null; + var output = RentMemoryStream(); + chunks.Add(output); + + for (int i = 0; i < logEvents.Count; i++) + { + SerializeLogEvent(logEvents[i], output); + + if (output.Length >= MaxPayloadSizeBytes && i < logEvents.Count - 1) + { + var sendTask = HttpClientSendAsync(_logsUrl, BuildHttpContent(output), cancellationToken); + pendingTasks ??= new List>(); + pendingTasks.Add(sendTask); + output = RentMemoryStream(); + chunks.Add(output); + } + } + + await SendLastChunkAndAwaitPendingAsync(BuildHttpContent(output), pendingTasks, cancellationToken).ConfigureAwait(false); + } + + private async Task SendLastChunkAndAwaitPendingAsync(HttpContent lastContent, List>? pendingTasks, CancellationToken cancellationToken) + { + using var lastResponse = await HttpClientSendAsync(_logsUrl, lastContent, cancellationToken).ConfigureAwait(false); + if (pendingTasks != null) + { + if (pendingTasks.Count == 1) + { + using var response = await pendingTasks[0].ConfigureAwait(false); + } + else + { + var responses = await Task.WhenAll(pendingTasks).ConfigureAwait(false); + foreach (var response in responses) + response?.Dispose(); + } + } + } + + private static ByteArrayContent BuildHttpContent(MemoryStream output) + { + return new ByteArrayContent(output.GetBuffer(), 0, (int)output.Length); + } + + private async Task WriteCompressedAsync(IList logEvents, List chunks, CancellationToken cancellationToken) + { + List>? pendingTasks = null; + var compressionLevel = Compress == HttpCompressionType.GZipFast ? CompressionLevel.Fastest : CompressionLevel.Optimal; + var eventBuffer = RentMemoryStream(); + var compressedOutput = RentMemoryStream(); + chunks.Add(compressedOutput); + GZipStream? gzipStream = new GZipStream(compressedOutput, compressionLevel, leaveOpen: true); + try + { + for (int i = 0; i < logEvents.Count; i++) + { + eventBuffer.Position = 0; + eventBuffer.SetLength(0); + SerializeLogEvent(logEvents[i], eventBuffer); + gzipStream.Write(eventBuffer.GetBuffer(), 0, (int)eventBuffer.Length); + + if (compressedOutput.Length >= MaxPayloadSizeBytes && i < logEvents.Count - 1) + { + gzipStream.Dispose(); + gzipStream = null; + var sendTask = HttpClientSendAsync(_logsUrl, BuildGzipContent(compressedOutput), cancellationToken); + pendingTasks ??= new List>(); + pendingTasks.Add(sendTask); + compressedOutput = RentMemoryStream(); + chunks.Add(compressedOutput); + gzipStream = new GZipStream(compressedOutput, compressionLevel, leaveOpen: true); + } + } + + gzipStream.Dispose(); + gzipStream = null; + } + finally + { + gzipStream?.Dispose(); + ReturnMemoryStream(eventBuffer); + } + + await SendLastChunkAndAwaitPendingAsync(BuildGzipContent(compressedOutput), pendingTasks, cancellationToken).ConfigureAwait(false); + } + + private static HttpContent BuildGzipContent(MemoryStream compressedPayload) + { + var content = new ByteArrayContent(compressedPayload.GetBuffer(), 0, (int)compressedPayload.Length); + content.Headers.ContentEncoding.Add("gzip"); + return content; + } + + private void SerializeLogEvent(LogEventInfo logEvent, MemoryStream output) + { + using var builder = _serializer.BeginRecord(output); + + var serviceName = RenderLogEvent(ServiceName, logEvent); + if (serviceName is null || string.IsNullOrWhiteSpace(serviceName)) + serviceName = "Unknown"; + builder.AddResourceAttribute("service.name", serviceName); + + for (int j = 0; j < ResourceAttributes.Count; j++) + { + var attr = ResourceAttributes[j]; + if (attr.Name is null || string.IsNullOrWhiteSpace(attr.Name)) + continue; + var value = RenderLogEvent(attr.Layout, logEvent); + if (!attr.IncludeEmptyValue && string.IsNullOrWhiteSpace(value)) + continue; + builder.AddResourceAttribute(attr.Name, value); + } + + var properties = GetLogEventProperties(logEvent); + if (properties is null && (IncludeEventProperties && logEvent.HasProperties)) + { + builder.AddScopeLogs(ScopeName, logEvent, RenderLogEvent(Layout, logEvent), logEvent.Properties); + } + else + { + builder.AddScopeLogs(ScopeName, logEvent, RenderLogEvent(Layout, logEvent), properties); + } + } + + IEnumerable>? GetLogEventProperties(LogEventInfo logEvent) + { + if (IncludeScopeProperties || IncludeGdc || (ContextProperties.Count > 0 && IncludeEventProperties && logEvent.HasProperties)) + { + return GetAllProperties(logEvent); + } + else if (ContextProperties.Count > 0) + { + return GetLogEventContextProperties(logEvent); + } + else + { + return null; + } + } + + private IEnumerable> GetLogEventContextProperties(LogEventInfo logEvent) + { + for (int i = 0; i < ContextProperties.Count; i++) + { + var prop = ContextProperties[i]; + var value = RenderLogEvent(prop.Layout, logEvent); + if (!prop.IncludeEmptyValue && string.IsNullOrWhiteSpace(value)) + continue; + yield return new KeyValuePair(prop.Name, value); + } + } + + private MemoryStream RentMemoryStream() + { + lock (_memoryStreamPool) + return _memoryStreamPool.Count > 0 ? _memoryStreamPool.Pop() : new MemoryStream(4096); + } + + private void ReturnMemoryStream(MemoryStream stream) + { + if (stream.Capacity < 1_000_000) + { + stream.Position = 0; + stream.SetLength(0); + lock (_memoryStreamPool) + _memoryStreamPool.Push(stream); + } + else + { + stream.Dispose(); + } + } + + + #region OTLP Environment Variable Support + + /// + /// Resolves standard OpenTelemetry environment variables as fallback defaults. + /// Signal-specific variables (e.g. OTEL_EXPORTER_OTLP_LOGS_ENDPOINT) take precedence + /// over generic ones (e.g. OTEL_EXPORTER_OTLP_ENDPOINT). + /// Properties that have been explicitly configured are not overridden. + /// + private void ResolveOtlpEnvironmentVariables() + { + // Capture whether ServiceName is still at its default value before any env var processing. + // OTEL_SERVICE_NAME must take precedence over service.name in OTEL_RESOURCE_ATTRIBUTES, + // but must not override an explicitly configured ServiceName. + bool serviceNameIsDefault = ReferenceEquals(ServiceName, _defaultServiceName); + + // Endpoint: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT > OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/logs" + if (Url is null || ReferenceEquals(Url, Layout.Empty)) + { + var logsEndpoint = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"); + if (string.IsNullOrWhiteSpace(logsEndpoint)) + { + var baseEndpoint = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_ENDPOINT"); + if (!string.IsNullOrWhiteSpace(baseEndpoint)) + { + logsEndpoint = baseEndpoint!.TrimEnd('/') + "/v1/logs"; + } + } + if (!string.IsNullOrWhiteSpace(logsEndpoint)) + { + Url = logsEndpoint; + } + } + + // Headers: OTEL_EXPORTER_OTLP_LOGS_HEADERS > OTEL_EXPORTER_OTLP_HEADERS + if (Headers.Count == 0) + { + var headersStr = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + if (string.IsNullOrWhiteSpace(headersStr)) + headersStr = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_HEADERS"); + if (!string.IsNullOrWhiteSpace(headersStr)) + ParseOtlpHeaders(headersStr!); + } + + // Compression: OTEL_EXPORTER_OTLP_LOGS_COMPRESSION > OTEL_EXPORTER_OTLP_COMPRESSION + if (Compress == HttpCompressionType.None) + { + var compression = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_LOGS_COMPRESSION"); + if (string.IsNullOrWhiteSpace(compression)) + compression = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_COMPRESSION"); + if (string.Equals(compression?.Trim(), "gzip", StringComparison.OrdinalIgnoreCase)) + Compress = HttpCompressionType.GZip; + else if (string.Equals(compression?.Trim(), "none", StringComparison.OrdinalIgnoreCase)) + Compress = HttpCompressionType.None; + } + + // Timeout: OTEL_EXPORTER_OTLP_LOGS_TIMEOUT > OTEL_EXPORTER_OTLP_TIMEOUT (milliseconds) + { + var timeoutStr = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_LOGS_TIMEOUT"); + if (string.IsNullOrWhiteSpace(timeoutStr)) + timeoutStr = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_TIMEOUT"); + if (!string.IsNullOrWhiteSpace(timeoutStr) && int.TryParse(timeoutStr!.Trim(), out var timeoutMs) && timeoutMs > 0) + SendTimeoutSeconds = Math.Max(1, timeoutMs / 1000); + } + + // Resource Attributes: OTEL_RESOURCE_ATTRIBUTES (format: key1=value1,key2=value2) + if (ResourceAttributes.Count == 0) + { + var resourceAttrs = GetEnvironmentValueFromLayout("OTEL_RESOURCE_ATTRIBUTES"); + if (!string.IsNullOrWhiteSpace(resourceAttrs)) + ParseOtlpResourceAttributes(resourceAttrs!); + } + + // Service Name: OTEL_SERVICE_NAME takes precedence over service.name in OTEL_RESOURCE_ATTRIBUTES + if (serviceNameIsDefault) + { + var otelServiceName = GetEnvironmentValueFromLayout("OTEL_SERVICE_NAME"); + if (!string.IsNullOrWhiteSpace(otelServiceName)) + { + ServiceName = otelServiceName; + } + } + +#if !NETFRAMEWORK || NET471_OR_GREATER + // Client Certificate: OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE > OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE + if (SslCertificateFile is null) + { + var clientCert = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE"); + if (string.IsNullOrWhiteSpace(clientCert)) + clientCert = GetEnvironmentValueFromLayout("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE"); + if (!string.IsNullOrWhiteSpace(clientCert)) + SslCertificateFile = clientCert; + } +#endif + } + + /// + /// Reads an environment variable using NLog ${environment} layout renderer. + /// + private static string GetEnvironmentValueFromLayout(string variableName) + { + try + { + Layout layout = "${environment:" + variableName + "}"; + return layout.Render(LogEventInfo.CreateNullEvent()); + } + catch (Exception ex) + { + NLog.Common.InternalLogger.Debug(ex, "OpenTelemetryHttpTarget: Failed to read environment variable '{0}'", variableName); + return string.Empty; + } + } + + /// + /// Parses OTLP headers from a W3C Baggage-like format (key1=value1,key2=value2) + /// and adds them to . + /// + private void ParseOtlpHeaders(string headersStr) + { + foreach (var (key, value) in ParseKeyValuePairs(headersStr)) + Headers.Add(new TargetPropertyWithContext { Name = key, Layout = value }); + } + + /// + /// Parses OTLP resource attributes from key1=value1,key2=value2 format. + /// The service.name key is extracted into . + /// + private void ParseOtlpResourceAttributes(string attrsStr) + { + foreach (var (key, value) in ParseKeyValuePairs(attrsStr)) + { + if (string.Equals(key, "service.name", StringComparison.Ordinal)) + { + if (ReferenceEquals(ServiceName, _defaultServiceName) && !string.IsNullOrWhiteSpace(value)) + ServiceName = value; + } + else + { + ResourceAttributes.Add(new TargetPropertyWithContext { Name = key, Layout = value }); + } + } + } + + /// + /// Splits a comma-separated key=value string, percent-decodes each token, + /// and yields only entries with a non-empty key. + /// + private static IEnumerable<(string key, string value)> ParseKeyValuePairs(string input) + { + var pairs = input.Split(','); + for (int i = 0; i < pairs.Length; i++) + { + var pair = pairs[i]; + var eqIdx = pair.IndexOf('='); + if (eqIdx <= 0) + continue; + var key = Uri.UnescapeDataString(pair.Substring(0, eqIdx).Trim()); + if (string.IsNullOrEmpty(key)) + continue; + var value = Uri.UnescapeDataString(pair.Substring(eqIdx + 1).Trim()); + yield return (key, value); + } + } + + #endregion + } +} diff --git a/src/NLog.Targets.OpenTelemetryHttp/Properties/AssemblyInfo.cs b/src/NLog.Targets.OpenTelemetryHttp/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..75a88a4 --- /dev/null +++ b/src/NLog.Targets.OpenTelemetryHttp/Properties/AssemblyInfo.cs @@ -0,0 +1,47 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +using System; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security; + +[assembly: AssemblyCulture("")] +[assembly: CLSCompliant(true)] +[assembly: ComVisible(false)] +[assembly: InternalsVisibleTo("NLog.Targets.OpenTelemetryHttp.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100ef8eab4fbdeb511eeb475e1659fe53f00ec1c1340700f1aa347bf3438455d71993b28b1efbed44c8d97a989e0cb6f01bcb5e78f0b055d311546f63de0a969e04cf04450f43834db9f909e566545a67e42822036860075a1576e90e1c43d43e023a24c22a427f85592ae56cac26f13b7ec2625cbc01f9490d60f16cfbb1bc34d9")] +[assembly: AllowPartiallyTrustedCallers] +#if !NET35 +[assembly: SecurityRules(SecurityRuleSet.Level1)] +#endif diff --git a/src/NLog.Targets.OpenTelemetryHttp/README.md b/src/NLog.Targets.OpenTelemetryHttp/README.md new file mode 100644 index 0000000..b7dbcfc --- /dev/null +++ b/src/NLog.Targets.OpenTelemetryHttp/README.md @@ -0,0 +1,180 @@ +# NLog.Targets.OpenTelemetry + +[![Version](https://badge.fury.io/nu/NLog.Targets.HttpOpenTelemetry.svg)](https://www.nuget.org/packages/NLog.Targets.HttpOpenTelemetry) +[![AppVeyor](https://img.shields.io/appveyor/ci/NLog/NLog-Targets-Network/master.svg)](https://ci.appveyor.com/project/NLog/NLog-Targets-Network/branch/master) + +NLog `OpenTelemetry` target for sending log messages to an OpenTelemetry OTLP endpoint using protobuf encoding over HTTP (OTLP/HTTP). + +If having trouble with output, then check [NLog InternalLogger](https://github.com/NLog/NLog/wiki/Internal-Logging) for clues. See also [Troubleshooting NLog](https://github.com/NLog/NLog/wiki/Logging-Troubleshooting). + +See also: + +* https://opentelemetry.io/docs/specs/otlp/#otlphttp +* https://opentelemetry.io/docs/specs/otel/protocol/exporter/ + +## Register Extension + +NLog will only recognize the type-alias `OpenTelemetry` when loading from an `NLog.config` file after registering the extension: + +```xml + + + +``` + +Alternative - register from code using the [fluent configuration API](https://github.com/NLog/NLog/wiki/Fluent-Configuration-API): + +```csharp +LogManager.Setup().SetupExtensions(ext => { + ext.RegisterTarget(); +}); +``` + +## Configuration Example + +Typical endpoint URL is `http://localhost:4318/v1/logs`. + +Supports standard OpenTelemetry environment variables as fallback defaults: +`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, +`OTEL_EXPORTER_OTLP_COMPRESSION`, `OTEL_EXPORTER_OTLP_LOGS_COMPRESSION`, `OTEL_EXPORTER_OTLP_TIMEOUT`, `OTEL_EXPORTER_OTLP_LOGS_TIMEOUT`, +`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`. + +```xml + + + + + + + +``` + +## Parameters + +## Parameters + +| Parameter | Default | Description | +| ------------------------ | -------------------------------| -----------------------------------------------------------------------------| +| `url` | OTEL_EXPORTER_OTLP_ENDPOINT | OTLP/HTTP endpoint URL. Automatically append `/v1/logs` when missing | +| `serviceName` | `${appdomain:format=Friendly}` | OpenTelemetry `service.name` resource attribute. | +| `scopeName` | `NLog` | OpenTelemetry instrumentation scope name. | +| `layout` | `${message}` | Layout used for the OTLP log body (message text). | +| `traceId` | `${activity:property=TraceId}` | Layout used to populate the OTLP LogRecord TraceId field. | +| `spanId` | `${activity:property=SpanId}` | Layout used to populate the OTLP LogRecord SpanId field. | +| `compress` | `None` | Optional payload compression. Supports `None`, `GZip`, and `GZipFast`. | +| `batchSize` | `200` | Maximum number of log events processed in a batch before transmission. | +| `includeEventProperties` | `true` | Includes NLog event properties as OpenTelemetry log attributes. | +| `sendTimeoutSeconds` | 30 | HTTP request timeout. | +| `maxPayloadSizeBytes` | 40960 | Maximum payload size before automatic request chunking occurs. | +| `sslCertificateFile` | | Client certificate file used for mutual TLS authentication. | +| `sslCertificatePassword` | | Password for the client certificate. | +| `proxyUrl` | | Proxy server URL. | +| `proxyUser` | | Proxy authentication username. | +| `proxyPassword` | | Proxy authentication password. | +| `batchSize` | `200` | Maximum number of log events to send in a single HTTP payload. | +| `taskDelayMilliseconds` | `50` | Delay before processing queued log events. Higher value can improve batching | +| `taskTimeoutSeconds` | `150` | Maximum execution time in seconds before cancellation of HTTP request. | +| `retryCount` | `3` | Number of retry attempts for failed write operations. | +| `retryDelayMilliseconds` | `2500` | Initial delay before retry after failed request. Delay doubles for each retry. | +| `queueLimit` | `10000` | Maximum number of pending requests allowed in the internal queue. | +| `overflowAction` | `Discard` | Action taken when the internal request queue reaches its limit. | + + +## Resource Attributes + +Additional OpenTelemetry resource attributes can be configured using `resourceAttribute` entries: + +```xml + + + + + + +``` + +These attributes are attached to every exported log record. + +## OpenTelemetry Environment Variables + +The target automatically resolves standard OpenTelemetry environment variables when equivalent target properties have not been explicitly configured. + +### Endpoint + +Order of precedence: + +1. `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` +2. `OTEL_EXPORTER_OTLP_ENDPOINT` + +When `OTEL_EXPORTER_OTLP_ENDPOINT` is used, `/v1/logs` is automatically appended. + +### Headers + +Order of precedence: + +1. `OTEL_EXPORTER_OTLP_LOGS_HEADERS` +2. `OTEL_EXPORTER_OTLP_HEADERS` + +Example: + +```text +OTEL_EXPORTER_OTLP_HEADERS=api-key=my-secret-key +``` + +### Compression + +Order of precedence: + +1. `OTEL_EXPORTER_OTLP_LOGS_COMPRESSION` +2. `OTEL_EXPORTER_OTLP_COMPRESSION` + +Supported values: + +* `gzip` +* `none` + +### Timeout + +Order of precedence: + +1. `OTEL_EXPORTER_OTLP_LOGS_TIMEOUT` +2. `OTEL_EXPORTER_OTLP_TIMEOUT` + +Timeout values are specified in milliseconds. + +### Service Name + +`OTEL_SERVICE_NAME` + +Overrides the `service.name` value from `OTEL_RESOURCE_ATTRIBUTES`. + +### Resource Attributes + +`OTEL_RESOURCE_ATTRIBUTES` + +Example: + +```text +OTEL_RESOURCE_ATTRIBUTES=service.namespace=Backend,deployment.environment=Production +``` + +The special attribute: + +```text +service.name +``` + +is automatically mapped to the `serviceName` property. + +### Client Certificate + +Order of precedence: + +1. `OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE` +2. `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE` + +Used to configure mutual TLS authentication. diff --git a/tests/NLog.Targets.HttpClient.Tests/HttpClientTargetTests.cs b/tests/NLog.Targets.HttpClient.Tests/HttpClientTargetTests.cs new file mode 100644 index 0000000..cd00bb6 --- /dev/null +++ b/tests/NLog.Targets.HttpClient.Tests/HttpClientTargetTests.cs @@ -0,0 +1,510 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets.HttpClient.Tests +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.IO.Compression; + using System.Net; + using System.Net.Sockets; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using NLog.Config; + using NLog.Targets; + using Xunit; + + public class HttpClientTargetTests + { + public HttpClientTargetTests() + { + LogManager.ThrowExceptions = true; + } + + [Fact] + public void PostSingleMessage_HappyPath() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${logger}|${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello world"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.Equal("POST", requests[0].Method); + Assert.Equal("/logs", requests[0].Path); + Assert.Equal("TestLogger|hello world", requests[0].Body); + } + } + +#if !NETFRAMEWORK + [Fact] + public void GetRequest_SendsCorrectMethod() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + HttpMethod = "Get", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.Equal("GET", requests[0].Method); + } + } +#endif + + [Fact] + public void BatchMessages_JoinedWithNewline() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + TaskDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("msg1"); + logger.Info("msg2"); + logger.Info("msg3"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + var allBodies = string.Concat(requests.ConvertAll(r => r.Body)); + Assert.Contains("msg1", allBodies); + Assert.Contains("msg2", allBodies); + Assert.Contains("msg3", allBodies); + } + } + + [Fact] + public void BatchAsJsonArray_WrapsMessagesInArray() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + BatchAsJsonArray = true, + BatchSize = 200, + TaskDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("msg1"); + logger.Info("msg2"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + Assert.Equal("[msg1, msg2]", requests[0].Body); + } + } + + [Fact] + public void ContentType_CustomValue_IsReflectedInHeader() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + ContentType = "text/plain", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.True(requests[0].Headers.TryGetValue("Content-Type", out var ctValue)); + Assert.Contains("text/plain", ctValue); + } + } + + [Fact] + public void GZipCompression_CompressesPayloadAndSetsHeader() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + Compress = HttpCompressionType.GZip, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("compressed message"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.True(requests[0].Headers.TryGetValue("Content-Encoding", out var encoding)); + Assert.Equal("gzip", encoding); + var decompressed = DecompressGzip(requests[0].BodyBytes); + Assert.Equal("compressed message", decompressed); + } + } + + [Fact] + public void CustomHeaders_AreSentInRequest() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + }; + target.Headers.Add(new TargetPropertyWithContext { Name = "Authorization", Layout = "Bearer mytoken123" }); + target.Headers.Add(new TargetPropertyWithContext { Name = "X-Custom-Header", Layout = "custom-value" }); + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.True(requests[0].Headers.TryGetValue("X-Custom-Header", out var customHeader)); + Assert.Equal("custom-value", customHeader); + Assert.True(requests[0].Headers.TryGetValue("Authorization", out var authorizationHeader)); + Assert.Equal("Bearer mytoken123", authorizationHeader); + } + } + + [Fact] + public void MaxPayloadSizeBytes_SplitsLargeBatchIntoMultipleRequests() + { + using (var server = new SimpleHttpServer()) + { + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + MaxPayloadSizeBytes = 5, // Forces split: each ~4-byte message overflows with separator + RetryCount = 0, + TaskDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("msg1"); + logger.Info("msg2"); + logger.Info("msg3"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(2); + Assert.True(requests.Count >= 2, $"Expected at least 2 batched requests, got {requests.Count}"); + var allBodies = string.Concat(requests.ConvertAll(r => r.Body)); + Assert.Contains("msg1", allBodies); + Assert.Contains("msg2", allBodies); + Assert.Contains("msg3", allBodies); + } + } + + [Fact] + public void ServerError5xx_TriggersRetry() + { + using (var server = new SimpleHttpServer()) + { + server.ResponseStatusCode = 500; + + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + RetryCount = 1, + RetryDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + logFactory.ThrowExceptions = false; + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello"); + logFactory.Flush(); + } + + // 1 initial attempt + 1 retry = 2 total requests + var requests = server.WaitForRequests(2); + Assert.Equal(2, requests.Count); + } + } + + [Fact] + public void Server4xx_DoesNotRetry() + { + using (var server = new SimpleHttpServer()) + { + server.ResponseStatusCode = 400; + + var target = new HttpClientTarget + { + Url = $"http://127.0.0.1:{server.Port}/logs", + Layout = "${message}", + RetryCount = 3, + RetryDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + logFactory.ThrowExceptions = false; + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello"); + logFactory.Flush(); + } + + // 400 is not retried, so only 1 request + var requests = server.WaitForRequests(1); + Assert.Single(requests); + } + } + + private static LogFactory BuildLogFactory(HttpClientTarget target) + { + var logFactory = new LogFactory(); + var config = new LoggingConfiguration(logFactory); + config.AddRuleForAllLevels(target); + logFactory.Configuration = config; + return logFactory; + } + + private static string DecompressGzip(byte[] compressed) + { + using (var input = new MemoryStream(compressed)) + using (var gzip = new GZipStream(input, CompressionMode.Decompress)) + using (var reader = new StreamReader(gzip, Encoding.UTF8)) + { + return reader.ReadToEnd(); + } + } + + private sealed class SimpleHttpServer : IDisposable + { + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + private readonly List _requests = new List(); + private readonly object _lock = new object(); + private readonly SemaphoreSlim _requestSignal = new SemaphoreSlim(0); + + public int ResponseStatusCode { get; set; } = 200; + + public int Port => ((IPEndPoint)_listener.LocalEndpoint).Port; + + public SimpleHttpServer() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Task.Run(AcceptLoopAsync, _cts.Token); + } + + public List WaitForRequests(int count, int timeoutMs = 15000) + { + if (timeoutMs > 1 && Debugger.IsAttached) + timeoutMs = 120000; // Allow more time when debugging + + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + lock (_lock) + { + if (_requests.Count >= count) + return new List(_requests); + } + _requestSignal.Wait(50); + } + lock (_lock) + return new List(_requests); + } + + private async Task AcceptLoopAsync() + { + while (!_cts.IsCancellationRequested) + { + try + { + var client = await _listener.AcceptTcpClientAsync().ConfigureAwait(false); + _ = Task.Run(() => HandleClientAsync(client), _cts.Token); + } + catch + { + break; + } + } + } + + private async Task HandleClientAsync(TcpClient client) + { + using (client) + { + var stream = client.GetStream(); + var request = await ReadHttpRequestAsync(stream, _cts.Token).ConfigureAwait(false); + lock (_lock) + _requests.Add(request); + _requestSignal.Release(); + + var statusLine = ResponseStatusCode == 200 ? "200 OK" : $"{ResponseStatusCode} Error"; + var response = $"HTTP/1.1 {statusLine}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + var responseBytes = Encoding.ASCII.GetBytes(response); + await stream.WriteAsync(responseBytes, 0, responseBytes.Length, _cts.Token).ConfigureAwait(false); + } + } + + private static async Task ReadHttpRequestAsync(NetworkStream stream, CancellationToken cancellationToken) + { + // Read headers byte by byte until the end-of-headers marker (\r\n\r\n) + var headerBytes = new List(512); + while (true) + { + var b = stream.ReadByte(); + if (b == -1) break; + headerBytes.Add((byte)b); + var n = headerBytes.Count; + if (n >= 4 + && headerBytes[n - 4] == '\r' + && headerBytes[n - 3] == '\n' + && headerBytes[n - 2] == '\r' + && headerBytes[n - 1] == '\n') + { + break; + } + } + + var headerText = Encoding.ASCII.GetString(headerBytes.ToArray()); + var lines = headerText.Split(new[] { "\r\n" }, StringSplitOptions.None); + + var requestParts = lines.Length > 0 ? lines[0].Split(' ') : new string[0]; + var method = requestParts.Length > 0 ? requestParts[0] : string.Empty; + var path = requestParts.Length > 1 ? requestParts[1] : string.Empty; + + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 1; i < lines.Length; i++) + { + var colonIdx = lines[i].IndexOf(':'); + if (colonIdx > 0) + headers[lines[i].Substring(0, colonIdx).Trim()] = lines[i].Substring(colonIdx + 1).Trim(); + } + + // Read the body according to Content-Length + var bodyBytes = new byte[0]; + if (headers.TryGetValue("Content-Length", out var contentLengthStr) + && int.TryParse(contentLengthStr, out var contentLength) + && contentLength > 0) + { + bodyBytes = new byte[contentLength]; + int bytesRead = 0; + while (bytesRead < contentLength) + { + var read = await stream.ReadAsync(bodyBytes, bytesRead, contentLength - bytesRead, cancellationToken).ConfigureAwait(false); + if (read == 0) break; + bytesRead += read; + } + } + + return new CapturedRequest + { + Method = method, + Path = path, + Headers = headers, + BodyBytes = bodyBytes, + Body = Encoding.UTF8.GetString(bodyBytes), + }; + } + + public void Dispose() + { + _cts.Cancel(); + _listener.Stop(); + _requestSignal.Dispose(); + _cts.Dispose(); + } + } + + private sealed class CapturedRequest + { + public string Method { get; set; } = string.Empty; + public string Path { get; set; } = string.Empty; + public Dictionary Headers { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public byte[] BodyBytes { get; set; } = new byte[0]; + public string Body { get; set; } = string.Empty; + } + } +} diff --git a/tests/NLog.Targets.HttpClient.Tests/NLog.Targets.HttpClient.Tests.csproj b/tests/NLog.Targets.HttpClient.Tests/NLog.Targets.HttpClient.Tests.csproj new file mode 100644 index 0000000..d1fed0e --- /dev/null +++ b/tests/NLog.Targets.HttpClient.Tests/NLog.Targets.HttpClient.Tests.csproj @@ -0,0 +1,36 @@ + + + + 18.0 + net471 + net471;net8.0 + net471;net10.0 + + false + + NLogTests.snk + false + true + true + + Full + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/tests/NLog.Targets.HttpClient.Tests/NLogTests.snk b/tests/NLog.Targets.HttpClient.Tests/NLogTests.snk new file mode 100644 index 0000000..f168e4d Binary files /dev/null and b/tests/NLog.Targets.HttpClient.Tests/NLogTests.snk differ diff --git a/tests/NLog.Targets.Network.Tests/NLog.Targets.Network.Tests.csproj b/tests/NLog.Targets.Network.Tests/NLog.Targets.Network.Tests.csproj index 060d21a..9131de5 100644 --- a/tests/NLog.Targets.Network.Tests/NLog.Targets.Network.Tests.csproj +++ b/tests/NLog.Targets.Network.Tests/NLog.Targets.Network.Tests.csproj @@ -1,10 +1,11 @@  - 17.0 - net462 - net462;net8.0 - + 18.0 + net462;net471 + net462;net471;net8.0 + net462;net471;net10.0 + false NLogTests.snk diff --git a/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLog.Targets.OpenTelemetryHttp.Tests.csproj b/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLog.Targets.OpenTelemetryHttp.Tests.csproj new file mode 100644 index 0000000..566038c --- /dev/null +++ b/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLog.Targets.OpenTelemetryHttp.Tests.csproj @@ -0,0 +1,36 @@ + + + + 18.0 + net471 + net471;net8.0 + net471;net10.0 + + false + + NLogTests.snk + false + true + true + + Full + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLogTests.snk b/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLogTests.snk new file mode 100644 index 0000000..f168e4d Binary files /dev/null and b/tests/NLog.Targets.OpenTelemetryHttp.Tests/NLogTests.snk differ diff --git a/tests/NLog.Targets.OpenTelemetryHttp.Tests/OpenTelemetryHttpTests.cs b/tests/NLog.Targets.OpenTelemetryHttp.Tests/OpenTelemetryHttpTests.cs new file mode 100644 index 0000000..220e678 --- /dev/null +++ b/tests/NLog.Targets.OpenTelemetryHttp.Tests/OpenTelemetryHttpTests.cs @@ -0,0 +1,1307 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets.HttpOTLP.Tests +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.IO.Compression; + using System.Net; + using System.Net.Sockets; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using NLog.Config; + using Xunit; + + public class OpenTelemetryHttpTests + { + public OpenTelemetryHttpTests() + { + LogManager.ThrowExceptions = true; + } + + [Fact] + public void PostSingleMessage_ProtobufContentType() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ServiceName = "TestService", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello otlp"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.Equal("POST", requests[0].Method); + Assert.Equal("/v1/logs", requests[0].Path); + Assert.True(requests[0].Headers.TryGetValue("Content-Type", out var contentType)); + Assert.Contains("application/x-protobuf", contentType); + Assert.True(requests[0].BodyBytes.Length > 0); + } + } + + [Fact] + public void PostSingleMessage_ContainsLogMessageInProtobuf() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ServiceName = "TestService", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("hello otlp world"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Protobuf stores strings as raw UTF-8, so the message should appear in the byte stream + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("hello otlp world", bodyText); + Assert.Contains("TestService", bodyText); + Assert.Contains("TestLogger", bodyText); + } + } + + [Fact] + public void PostSingleMessage_ContainsSeverityText() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Warn("warning message"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("Warn", bodyText); + Assert.Contains("warning message", bodyText); + } + } + + [Fact] + public void PostSingleMessage_ContainsScopeName() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ScopeName = "MyInstrumentationScope", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("scoped message"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("MyInstrumentationScope", bodyText); + } + } + + [Fact] + public void PostSingleMessage_IncludesEventProperties() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = true, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "with props"); + logEvent.Properties["CustomKey"] = "CustomValue"; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("CustomKey", bodyText); + Assert.Contains("CustomValue", bodyText); + } + } + + [Fact] + public void PostSingleMessage_TypedEventProperties_EncodedAsCorrectAnyValueTypes() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = true, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "typed props"); + logEvent.Properties["BoolProp"] = true; + logEvent.Properties["IntProp"] = 42; + logEvent.Properties["DoubleProp"] = 3.14; + logEvent.Properties["StringProp"] = "hello"; + logEvent.Properties["EnumProp"] = TraceEventType.Resume; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord attributes + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // Collect all attribute fields (field 6 = KeyValue in LogRecord) + var attributes = logRecord.FindAll(f => f.FieldNumber == 6); + + // Parse each KeyValue to extract key name and AnyValue wire type + var typedAttributes = new Dictionary(); + for (int i = 0; i < attributes.Count; i++) + { + var kvFields = ReadProtobufFields(attributes[i].Data); + var keyField = kvFields.Find(f => f.FieldNumber == 1); + var valueField = kvFields.Find(f => f.FieldNumber == 2); + var keyName = Encoding.UTF8.GetString(keyField.Data); + typedAttributes[keyName] = valueField; + } + + // BoolProp: AnyValue field 2 (bool_value), varint wire type 0 + var boolAnyValue = ReadProtobufFields(typedAttributes["BoolProp"].Data); + Assert.Equal(2, boolAnyValue[0].FieldNumber); // bool_value field + Assert.Equal(0, boolAnyValue[0].WireType); // varint + + // IntProp: AnyValue field 3 (int_value), varint wire type 0 + var intAnyValue = ReadProtobufFields(typedAttributes["IntProp"].Data); + Assert.Equal(3, intAnyValue[0].FieldNumber); // int_value field + Assert.Equal(0, intAnyValue[0].WireType); // varint + + // DoubleProp: AnyValue field 4 (double_value), fixed64 wire type 1 + var doubleAnyValue = ReadProtobufFields(typedAttributes["DoubleProp"].Data); + Assert.Equal(4, doubleAnyValue[0].FieldNumber); // double_value field + Assert.Equal(1, doubleAnyValue[0].WireType); // fixed64 + + // StringProp: AnyValue field 1 (string_value), length-delimited wire type 2 + var stringAnyValue = ReadProtobufFields(typedAttributes["StringProp"].Data); + Assert.Equal(1, stringAnyValue[0].FieldNumber); // string_value field + Assert.Equal(2, stringAnyValue[0].WireType); // length-delimited + Assert.Equal("hello", Encoding.UTF8.GetString(stringAnyValue[0].Data)); + + // EnumProp: AnyValue field 1 (string_value), length-delimited wire type 2 + var enumAnyValue = ReadProtobufFields(typedAttributes["EnumProp"].Data); + Assert.Equal(1, enumAnyValue[0].FieldNumber); // string_value field + Assert.Equal(2, enumAnyValue[0].WireType); // length-delimited + Assert.Equal("Resume", Encoding.UTF8.GetString(enumAnyValue[0].Data)); + } + } + + [Fact] + public void PostSingleMessage_SpecialDoubleValues_EncodedAsFixed64() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = true, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "special doubles"); + logEvent.Properties["NaNProp"] = double.NaN; + logEvent.Properties["PosInfProp"] = double.PositiveInfinity; + logEvent.Properties["NegInfProp"] = double.NegativeInfinity; + logEvent.Properties["FloatNaNProp"] = float.NaN; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord attributes + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // Collect all attribute fields + var attributes = logRecord.FindAll(f => f.FieldNumber == 6); + var typedAttributes = new Dictionary(); + for (int i = 0; i < attributes.Count; i++) + { + var kvFields = ReadProtobufFields(attributes[i].Data); + var keyField = kvFields.Find(f => f.FieldNumber == 1); + var valueField = kvFields.Find(f => f.FieldNumber == 2); + var keyName = Encoding.UTF8.GetString(keyField.Data); + typedAttributes[keyName] = valueField; + } + + // All special doubles should be encoded as double_value (field 4, fixed64 wire type 1) + var nanAnyValue = ReadProtobufFields(typedAttributes["NaNProp"].Data); + Assert.Equal(4, nanAnyValue[0].FieldNumber); + Assert.Equal(1, nanAnyValue[0].WireType); + Assert.Equal(8, nanAnyValue[0].Data.Length); + Assert.True(double.IsNaN(BitConverter.ToDouble(nanAnyValue[0].Data, 0))); + + var posInfAnyValue = ReadProtobufFields(typedAttributes["PosInfProp"].Data); + Assert.Equal(4, posInfAnyValue[0].FieldNumber); + Assert.Equal(1, posInfAnyValue[0].WireType); + Assert.True(double.IsPositiveInfinity(BitConverter.ToDouble(posInfAnyValue[0].Data, 0))); + + var negInfAnyValue = ReadProtobufFields(typedAttributes["NegInfProp"].Data); + Assert.Equal(4, negInfAnyValue[0].FieldNumber); + Assert.Equal(1, negInfAnyValue[0].WireType); + Assert.True(double.IsNegativeInfinity(BitConverter.ToDouble(negInfAnyValue[0].Data, 0))); + + // float.NaN should also be encoded as double_value (TypeCode.Single → ToDouble) + var floatNanAnyValue = ReadProtobufFields(typedAttributes["FloatNaNProp"].Data); + Assert.Equal(4, floatNanAnyValue[0].FieldNumber); + Assert.Equal(1, floatNanAnyValue[0].WireType); + Assert.True(double.IsNaN(BitConverter.ToDouble(floatNanAnyValue[0].Data, 0))); + } + } + + [Fact] + public void PostSingleMessage_DictionaryProperty_EncodedAsKvListAnyValue() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = true, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "dict prop"); + logEvent.Properties["MapProp"] = new Dictionary + { + ["nestedKey"] = "nestedValue", + ["nestedInt"] = 42, + }; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord attributes + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // Find the MapProp attribute + var attributes = logRecord.FindAll(f => f.FieldNumber == 6); + ProtobufField mapAnyValue = default; + for (int i = 0; i < attributes.Count; i++) + { + var kvFields = ReadProtobufFields(attributes[i].Data); + var keyField = kvFields.Find(f => f.FieldNumber == 1); + if (Encoding.UTF8.GetString(keyField.Data) == "MapProp") + { + mapAnyValue = kvFields.Find(f => f.FieldNumber == 2); + break; + } + } + Assert.True(mapAnyValue.Data.Length > 0, "MapProp attribute should be present"); + + // AnyValue field 6 = kvlist_value (wire type 2, length-delimited) + var anyValueFields = ReadProtobufFields(mapAnyValue.Data); + var kvListField = anyValueFields.Find(f => f.FieldNumber == 6); + Assert.Equal(2, kvListField.WireType); + + // KeyValueList { repeated KeyValue values = 1 } + var kvListEntries = ReadProtobufFields(kvListField.Data); + Assert.Equal(2, kvListEntries.Count); + + // Parse nested entries + var nestedEntries = new Dictionary(); + for (int i = 0; i < kvListEntries.Count; i++) + { + var entryFields = ReadProtobufFields(kvListEntries[i].Data); + var entryKey = Encoding.UTF8.GetString(entryFields.Find(f => f.FieldNumber == 1).Data); + nestedEntries[entryKey] = entryFields.Find(f => f.FieldNumber == 2); + } + + // nestedKey should be string_value (AnyValue field 1) + var nestedKeyAnyValue = ReadProtobufFields(nestedEntries["nestedKey"].Data); + Assert.Equal(1, nestedKeyAnyValue[0].FieldNumber); + Assert.Equal("nestedValue", Encoding.UTF8.GetString(nestedKeyAnyValue[0].Data)); + + // nestedInt should be int_value (AnyValue field 3) + var nestedIntAnyValue = ReadProtobufFields(nestedEntries["nestedInt"].Data); + Assert.Equal(3, nestedIntAnyValue[0].FieldNumber); + Assert.Equal(0, nestedIntAnyValue[0].WireType); + } + } + + [Fact] + public void PostSingleMessage_ListProperty_EncodedAsArrayAnyValue() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = true, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "list prop"); + logEvent.Properties["ListProp"] = new List { "alpha", 99, true }; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord attributes + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // Find the ListProp attribute + var attributes = logRecord.FindAll(f => f.FieldNumber == 6); + ProtobufField listAnyValue = default; + for (int i = 0; i < attributes.Count; i++) + { + var kvFields = ReadProtobufFields(attributes[i].Data); + var keyField = kvFields.Find(f => f.FieldNumber == 1); + if (Encoding.UTF8.GetString(keyField.Data) == "ListProp") + { + listAnyValue = kvFields.Find(f => f.FieldNumber == 2); + break; + } + } + Assert.True(listAnyValue.Data.Length > 0, "ListProp attribute should be present"); + + // AnyValue field 5 = array_value (wire type 2, length-delimited) + var anyValueFields = ReadProtobufFields(listAnyValue.Data); + var arrayField = anyValueFields.Find(f => f.FieldNumber == 5); + Assert.Equal(2, arrayField.WireType); + + // ArrayValue { repeated AnyValue values = 1 } + var arrayEntries = ReadProtobufFields(arrayField.Data); + Assert.Equal(3, arrayEntries.Count); + + // Element 0: "alpha" → string_value (AnyValue field 1) + var elem0 = ReadProtobufFields(arrayEntries[0].Data); + Assert.Equal(1, elem0[0].FieldNumber); + Assert.Equal("alpha", Encoding.UTF8.GetString(elem0[0].Data)); + + // Element 1: 99 → int_value (AnyValue field 3) + var elem1 = ReadProtobufFields(arrayEntries[1].Data); + Assert.Equal(3, elem1[0].FieldNumber); + Assert.Equal(0, elem1[0].WireType); + + // Element 2: true → bool_value (AnyValue field 2) + var elem2 = ReadProtobufFields(arrayEntries[2].Data); + Assert.Equal(2, elem2[0].FieldNumber); + Assert.Equal(0, elem2[0].WireType); + } + } + + [Fact] + public void PostSingleMessage_ExcludesEventPropertiesWhenDisabled() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + IncludeEventProperties = false, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + var logEvent = new LogEventInfo(LogLevel.Info, "TestLogger", "no props"); + logEvent.Properties["SecretKey"] = "SecretValue"; + logger.Log(logEvent); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.DoesNotContain("SecretKey", bodyText); + Assert.DoesNotContain("SecretValue", bodyText); + } + } + + [Fact] + public void PostSingleMessage_IncludesExceptionAttributes() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Error(new InvalidOperationException("test exception"), "error occurred"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("exception.type", bodyText); + Assert.Contains("InvalidOperationException", bodyText); + Assert.Contains("exception.message", bodyText); + Assert.Contains("test exception", bodyText); + } + } + + [Fact] + public void PostSingleMessage_IncludesResourceAttributes() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ServiceName = "MySvc", + }; + target.ResourceAttributes.Add(new TargetPropertyWithContext { Name = "deployment.environment", Layout = "staging" }); + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("resource test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("MySvc", bodyText); + Assert.Contains("deployment.environment", bodyText); + Assert.Contains("staging", bodyText); + } + } + + [Fact] + public void BatchMessages_SentAsSingleRequest() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + BatchSize = 200, + TaskDelayMilliseconds = 10, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("batch1"); + logger.Info("batch2"); + logger.Info("batch3"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + + // All messages should be in the protobuf payload(s) + var allBodyText = new StringBuilder(); + foreach (var req in requests) + allBodyText.Append(Encoding.UTF8.GetString(req.BodyBytes)); + + var combined = allBodyText.ToString(); + Assert.Contains("batch1", combined); + Assert.Contains("batch2", combined); + Assert.Contains("batch3", combined); + } + } + + [Fact] + public void GZipCompression_CompressesPayloadAndSetsHeader() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + Compress = HttpCompressionType.GZip, + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("compressed otlp message"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + Assert.True(requests[0].Headers.TryGetValue("Content-Encoding", out var encoding)); + Assert.Equal("gzip", encoding); + + var decompressed = DecompressGzip(requests[0].BodyBytes); + Assert.Contains("compressed otlp message", decompressed); + } + } + + [Fact] + public void ProtobufPayload_HasValidStructure() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ServiceName = "StructTest", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("structure test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Parse the top-level ExportLogsServiceRequest + var topFields = ReadProtobufFields(requests[0].BodyBytes); + Assert.True(topFields.Count >= 1, "ExportLogsServiceRequest should have at least 1 field (resource_logs)"); + + // Field 1 = ResourceLogs (wire type 2 = length-delimited) + var resourceLogsField = topFields.Find(f => f.FieldNumber == 1); + Assert.Equal(2, resourceLogsField.WireType); + + // Parse ResourceLogs + var rlFields = ReadProtobufFields(resourceLogsField.Data); + Assert.True(rlFields.Count >= 2, "ResourceLogs should have resource and scope_logs"); + + // Field 1 = Resource, Field 2 = ScopeLogs + var resourceField = rlFields.Find(f => f.FieldNumber == 1); + Assert.Equal(2, resourceField.WireType); + var scopeLogsField = rlFields.Find(f => f.FieldNumber == 2); + Assert.Equal(2, scopeLogsField.WireType); + + // Parse ScopeLogs and verify it has log_records (field 2) + var slFields = ReadProtobufFields(scopeLogsField.Data); + var logRecordField = slFields.Find(f => f.FieldNumber == 2); + Assert.NotNull(logRecordField.Data); + Assert.True(logRecordField.Data.Length > 0, "LogRecord should have content"); + + // Parse LogRecord and verify timestamp (field 1, fixed64) exists + var lrFields = ReadProtobufFields(logRecordField.Data); + var timestampField = lrFields.Find(f => f.FieldNumber == 1); + Assert.Equal(1, timestampField.WireType); // fixed64 + Assert.Equal(8, timestampField.Data.Length); + } + } + + [Fact] + public void TraceIdAndSpanId_EncodedAsBytesInLogRecord() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + TraceId = NLog.Layouts.Layout.FromMethod(l => System.Diagnostics.ActivityTraceId.CreateFromString("0af7651916cd43dd8448eb211c80319c".AsSpan())), + SpanId = NLog.Layouts.Layout.FromMethod(l => System.Diagnostics.ActivitySpanId.CreateFromString("b7ad6b7169203331".AsSpan())), + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("trace context test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // trace_id = field 9, bytes (wire type 2), 16 bytes + var traceIdField = logRecord.Find(f => f.FieldNumber == 9); + Assert.Equal(2, traceIdField.WireType); + Assert.Equal(16, traceIdField.Data.Length); + Assert.Equal("0af7651916cd43dd8448eb211c80319c", BitConverter.ToString(traceIdField.Data).Replace("-", "").ToLowerInvariant()); + + // span_id = field 10, bytes (wire type 2), 8 bytes + var spanIdField = logRecord.Find(f => f.FieldNumber == 10); + Assert.Equal(2, spanIdField.WireType); + Assert.Equal(8, spanIdField.Data.Length); + Assert.Equal("b7ad6b7169203331", BitConverter.ToString(spanIdField.Data).Replace("-", "").ToLowerInvariant()); + } + } + + [Fact] + public void TraceIdAndSpanId_OmittedWhenEmpty() + { + using (var server = new SimpleHttpServer()) + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + TraceId = "", + SpanId = "", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("no trace context"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.Single(requests); + + // Navigate to LogRecord + var topFields = ReadProtobufFields(requests[0].BodyBytes); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var scopeLogs = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 2).Data); + var logRecord = ReadProtobufFields(scopeLogs.Find(f => f.FieldNumber == 2).Data); + + // No trace_id (field 9) or span_id (field 10) should be present + Assert.DoesNotContain(logRecord, f => f.FieldNumber == 9); + Assert.DoesNotContain(logRecord, f => f.FieldNumber == 10); + } + } + + [Fact] + public void OtlpEndpointEnvVar_UsedAsUrlWhenNotExplicitlySet() + { + using (var server = new SimpleHttpServer()) + { + var envEndpoint = $"http://127.0.0.1:{server.Port}"; + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", envEndpoint); + try + { + var target = new OpenTelemetryHttpTarget + { + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("env endpoint test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + Assert.Equal("/v1/logs", requests[0].Path); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", null); + } + } + } + + [Fact] + public void OtlpLogsEndpointEnvVar_TakesPrecedenceOverGenericEndpoint() + { + using (var server = new SimpleHttpServer()) + { + var logsEndpoint = $"http://127.0.0.1:{server.Port}/custom/logs"; + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:9999"); + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", logsEndpoint); + try + { + var target = new OpenTelemetryHttpTarget + { + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("logs endpoint test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + Assert.Equal("/custom/logs", requests[0].Path); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", null); + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", null); + } + } + } + + [Fact] + public void OtlpHeadersEnvVar_ParsedAndSentInRequest() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=Bearer token123,X-Env-Header=envvalue"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("headers env test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + Assert.True(requests[0].Headers.TryGetValue("X-Env-Header", out var envHeader)); + Assert.Equal("envvalue", envHeader); + Assert.True(requests[0].Headers.TryGetValue("Authorization", out var authHeader)); + Assert.Equal("Bearer token123", authHeader); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS", null); + } + } + } + + [Fact] + public void OtlpServiceNameEnvVar_UsedWhenNotExplicitlySet() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", "EnvServiceName"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("service name env test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("EnvServiceName", bodyText); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", null); + } + } + } + + [Fact] + public void OtlpResourceAttributesEnvVar_ParsedIntoResourceAndServiceName() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", "service.name=AttrService,deployment.environment=staging"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("resource attrs test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("AttrService", bodyText); + Assert.Contains("deployment.environment", bodyText); + Assert.Contains("staging", bodyText); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", null); + } + } + } + + [Fact] + public void OtlpServiceNameEnvVar_TakesPrecedenceOverResourceAttributes() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", "service.name=FromAttrs"); + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", "FromEnvVar"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("precedence test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("FromEnvVar", bodyText); + Assert.DoesNotContain("FromAttrs", bodyText); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", null); + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", null); + } + } + } + + [Fact] + public void OtlpCompressionEnvVar_EnablesGzip() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_COMPRESSION", "gzip"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + }; + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("compression env test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + Assert.True(requests[0].Headers.TryGetValue("Content-Encoding", out var encoding)); + Assert.Equal("gzip", encoding); + + var decompressed = DecompressGzip(requests[0].BodyBytes); + Assert.Contains("compression env test", decompressed); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_COMPRESSION", null); + } + } + } + + [Fact] + public void ExplicitConfig_TakesPrecedenceOverEnvVars() + { + using (var server = new SimpleHttpServer()) + { + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", "EnvService"); + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS", "X-Env=envvalue"); + try + { + var target = new OpenTelemetryHttpTarget + { + Url = $"http://127.0.0.1:{server.Port}/v1/logs", + Layout = "${message}", + ServiceName = "ExplicitService", + }; + target.Headers.Add(new TargetPropertyWithContext { Name = "X-Explicit", Layout = "explicitvalue" }); + + using (var logFactory = BuildLogFactory(target)) + { + var logger = logFactory.GetLogger("TestLogger"); + logger.Info("precedence test"); + logFactory.Flush(); + } + + var requests = server.WaitForRequests(1); + Assert.True(requests.Count >= 1); + + // Explicit ServiceName should be used, not env var + var bodyText = Encoding.UTF8.GetString(requests[0].BodyBytes); + Assert.Contains("ExplicitService", bodyText); + Assert.DoesNotContain("EnvService", bodyText); + + // Explicit headers should be used, env var headers should not be added + Assert.True(requests[0].Headers.TryGetValue("X-Explicit", out var explicitHeader)); + Assert.Equal("explicitvalue", explicitHeader); + Assert.False(requests[0].Headers.ContainsKey("X-Env")); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_SERVICE_NAME", null); + Environment.SetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS", null); + } + } + } + + private static LogFactory BuildLogFactory(OpenTelemetryHttpTarget target) + { + var logFactory = new LogFactory(); + var config = new LoggingConfiguration(logFactory); + config.AddRuleForAllLevels(target); + logFactory.Configuration = config; + return logFactory; + } + + private static string DecompressGzip(byte[] compressed) + { + using (var input = new MemoryStream(compressed)) + using (var gzip = new GZipStream(input, CompressionMode.Decompress)) + using (var output = new MemoryStream()) + { + gzip.CopyTo(output); + return Encoding.UTF8.GetString(output.ToArray()); + } + } + + #region Protobuf Reader Helpers + + private struct ProtobufField + { + public int FieldNumber; + public int WireType; + public byte[] Data; + } + + private static List ReadProtobufFields(byte[] data) + { + var fields = new List(); + int offset = 0; + while (offset < data.Length) + { + var tag = ReadVarint(data, ref offset); + var fieldNumber = (int)(tag >> 3); + var wireType = (int)(tag & 0x7); + + byte[] fieldData; + switch (wireType) + { + case 0: // varint + var start = offset; + ReadVarint(data, ref offset); + fieldData = new byte[offset - start]; + Array.Copy(data, start, fieldData, 0, fieldData.Length); + break; + case 1: // 64-bit (fixed64) + fieldData = new byte[8]; + Array.Copy(data, offset, fieldData, 0, 8); + offset += 8; + break; + case 2: // length-delimited + var length = (int)ReadVarint(data, ref offset); + fieldData = new byte[length]; + Array.Copy(data, offset, fieldData, 0, length); + offset += length; + break; + case 5: // 32-bit (fixed32) + fieldData = new byte[4]; + Array.Copy(data, offset, fieldData, 0, 4); + offset += 4; + break; + default: + throw new InvalidOperationException($"Unknown protobuf wire type: {wireType}"); + } + + fields.Add(new ProtobufField { FieldNumber = fieldNumber, WireType = wireType, Data = fieldData }); + } + return fields; + } + + private static ulong ReadVarint(byte[] data, ref int offset) + { + ulong value = 0; + int shift = 0; + while (offset < data.Length) + { + var b = data[offset++]; + value |= (ulong)(b & 0x7F) << shift; + if ((b & 0x80) == 0) + break; + shift += 7; + } + return value; + } + + #endregion + + private sealed class SimpleHttpServer : IDisposable + { + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new CancellationTokenSource(); + private readonly List _requests = new List(); + private readonly object _lock = new object(); + private readonly SemaphoreSlim _requestSignal = new SemaphoreSlim(0); + + public int ResponseStatusCode { get; set; } = 200; + + public int Port => ((IPEndPoint)_listener.LocalEndpoint).Port; + + public SimpleHttpServer() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Task.Run(AcceptLoopAsync, _cts.Token); + } + + public List WaitForRequests(int count, int timeoutMs = 15000) + { + if (timeoutMs > 1 && Debugger.IsAttached) + timeoutMs = 120000; + + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + lock (_lock) + { + if (_requests.Count >= count) + return new List(_requests); + } + _requestSignal.Wait(50); + } + lock (_lock) + return new List(_requests); + } + + private async Task AcceptLoopAsync() + { + while (!_cts.IsCancellationRequested) + { + try + { + var client = await _listener.AcceptTcpClientAsync().ConfigureAwait(false); + _ = Task.Run(() => HandleClientAsync(client), _cts.Token); + } + catch + { + break; + } + } + } + + private async Task HandleClientAsync(TcpClient client) + { + using (client) + { + var stream = client.GetStream(); + var request = await ReadHttpRequestAsync(stream, _cts.Token).ConfigureAwait(false); + lock (_lock) + _requests.Add(request); + _requestSignal.Release(); + + var statusLine = ResponseStatusCode == 200 ? "200 OK" : $"{ResponseStatusCode} Error"; + var response = $"HTTP/1.1 {statusLine}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + var responseBytes = Encoding.ASCII.GetBytes(response); + await stream.WriteAsync(responseBytes, 0, responseBytes.Length, _cts.Token).ConfigureAwait(false); + } + } + + private static async Task ReadHttpRequestAsync(NetworkStream stream, CancellationToken cancellationToken) + { + var headerBytes = new List(512); + while (true) + { + var b = stream.ReadByte(); + if (b == -1) break; + headerBytes.Add((byte)b); + var n = headerBytes.Count; + if (n >= 4 + && headerBytes[n - 4] == '\r' + && headerBytes[n - 3] == '\n' + && headerBytes[n - 2] == '\r' + && headerBytes[n - 1] == '\n') + { + break; + } + } + + var headerText = Encoding.ASCII.GetString(headerBytes.ToArray()); + var lines = headerText.Split(new[] { "\r\n" }, StringSplitOptions.None); + + var requestParts = lines.Length > 0 ? lines[0].Split(' ') : new string[0]; + var method = requestParts.Length > 0 ? requestParts[0] : string.Empty; + var path = requestParts.Length > 1 ? requestParts[1] : string.Empty; + + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 1; i < lines.Length; i++) + { + var colonIdx = lines[i].IndexOf(':'); + if (colonIdx > 0) + headers[lines[i].Substring(0, colonIdx).Trim()] = lines[i].Substring(colonIdx + 1).Trim(); + } + + var bodyBytes = new byte[0]; + if (headers.TryGetValue("Content-Length", out var contentLengthStr) + && int.TryParse(contentLengthStr, out var contentLength) + && contentLength > 0) + { + bodyBytes = new byte[contentLength]; + int bytesRead = 0; + while (bytesRead < contentLength) + { + var read = await stream.ReadAsync(bodyBytes, bytesRead, contentLength - bytesRead, cancellationToken).ConfigureAwait(false); + if (read == 0) break; + bytesRead += read; + } + } + + return new CapturedRequest + { + Method = method, + Path = path, + Headers = headers, + BodyBytes = bodyBytes, + Body = Encoding.UTF8.GetString(bodyBytes), + }; + } + + public void Dispose() + { + _cts.Cancel(); + _listener.Stop(); + _requestSignal.Dispose(); + _cts.Dispose(); + } + } + + private sealed class CapturedRequest + { + public string Method { get; set; } = string.Empty; + public string Path { get; set; } = string.Empty; + public Dictionary Headers { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + public byte[] BodyBytes { get; set; } = new byte[0]; + public string Body { get; set; } = string.Empty; + } + } +} diff --git a/tests/NLog.Targets.OpenTelemetryHttp.Tests/OtlpProtobufSerializerTests.cs b/tests/NLog.Targets.OpenTelemetryHttp.Tests/OtlpProtobufSerializerTests.cs new file mode 100644 index 0000000..d185e9a --- /dev/null +++ b/tests/NLog.Targets.OpenTelemetryHttp.Tests/OtlpProtobufSerializerTests.cs @@ -0,0 +1,861 @@ +// +// Copyright (c) 2004-2024 Jaroslaw Kowalski , Kim Christensen, Julian Verdurmen +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of Jaroslaw Kowalski nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// + +namespace NLog.Targets.HttpOTLP.Tests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using NLog.Internal; + using Xunit; + + /// + /// Unit tests for that validate protobuf output + /// against the OpenTelemetry OTLP logs.proto specification without requiring an HTTP server. + /// + /// + /// Proto field references: + /// ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1 } + /// ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2 } + /// Resource { repeated KeyValue attributes = 1 } + /// ScopeLogs { InstrumentationScope scope = 1; repeated LogRecord log_records = 2 } + /// InstrumentationScope { string name = 1 } + /// LogRecord { fixed64 time_unix_nano = 1; SeverityNumber severity_number = 2; string severity_text = 3; + /// AnyValue body = 5; repeated KeyValue attributes = 6; bytes trace_id = 9; bytes span_id = 10 } + /// AnyValue { string string_value = 1; bool bool_value = 2; int64 int_value = 3; double double_value = 4; + /// ArrayValue array_value = 5; KeyValueList kvlist_value = 6 } + /// KeyValue { string key = 1; AnyValue value = 2 } + /// ArrayValue { repeated AnyValue values = 1 } + /// KeyValueList { repeated KeyValue values = 1 } + /// + public class OtlpProtobufSerializerTests + { + #region Structure tests + + [Fact] + public void Serialize_ProducesValidExportLogsServiceRequest_TopLevelStructure() + { + var serializer = new OtlpProtobufSerializer(); + + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "hello")); + + // Top-level message: ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1 } + var topFields = ReadProtobufFields(output); + Assert.Single(topFields); + var resourceLogsField = topFields[0]; + Assert.Equal(1, resourceLogsField.FieldNumber); // resource_logs field + Assert.Equal(2, resourceLogsField.WireType); // length-delimited + + // ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2 } + var rlFields = ReadProtobufFields(resourceLogsField.Data); + Assert.Single(rlFields); + Assert.Equal(2, rlFields[0].FieldNumber); // scope_logs + Assert.Equal(2, rlFields[0].WireType); // length-delimited + } + + [Fact] + public void Serialize_ProducesValidScopeLogs_WithScopeAndLogRecord() + { + var serializer = new OtlpProtobufSerializer(); + + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "msg")); + + var scopeLogs = NavigateToScopeLogs(output); + var slFields = ReadProtobufFields(scopeLogs); + + // ScopeLogs { InstrumentationScope scope = 1; repeated LogRecord log_records = 2 } + Assert.True(slFields.Count >= 2); + Assert.Equal(1, slFields[0].FieldNumber); // scope + Assert.Equal(2, slFields[0].WireType); + Assert.Equal(2, slFields[1].FieldNumber); // log_records + Assert.Equal(2, slFields[1].WireType); + } + + [Fact] + public void Serialize_InstrumentationScope_ContainsScopeName() + { + var serializer = new OtlpProtobufSerializer(); + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "msg"), scopeName: "MyInstrumentation"); + + var scopeLogs = NavigateToScopeLogs(output); + var slFields = ReadProtobufFields(scopeLogs); + var scopeField = slFields.Find(f => f.FieldNumber == 1); + + // InstrumentationScope { string name = 1 } + var scopeFields = ReadProtobufFields(scopeField.Data); + var nameField = scopeFields.Find(f => f.FieldNumber == 1); + Assert.Equal(2, nameField.WireType); + Assert.Equal("MyInstrumentation", Encoding.UTF8.GetString(nameField.Data)); + } + + [Fact] + public void Serialize_EmptyScopeName_OmitsScopeField() + { + var serializer = new OtlpProtobufSerializer(); + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "msg"), scopeName: ""); + + var scopeLogs = NavigateToScopeLogs(output); + var slFields = ReadProtobufFields(scopeLogs); + + // Without scope name only log_records (field 2) should be present + Assert.DoesNotContain(slFields, f => f.FieldNumber == 1); + Assert.Contains(slFields, f => f.FieldNumber == 2); + } + + [Fact] + public void Serialize_MultipleEvents_ProducesMultipleLogRecords() + { + var serializer = new OtlpProtobufSerializer(); + + var logEvents = new List + { + LogEventInfo.Create(LogLevel.Info, "L", "first"), + LogEventInfo.Create(LogLevel.Warn, "L", "second"), + LogEventInfo.Create(LogLevel.Error, "L", "third"), + }; + + var output = SerializeAll(serializer, logEvents); + + var topFields = ReadProtobufFields(output); + Assert.Equal(3, topFields.FindAll(f => f.FieldNumber == 1).Count); + } + + #endregion + + #region LogRecord field tests + + [Fact] + public void Serialize_LogRecord_TimestampIsFixed64UnixNanoseconds() + { + // Use a known UTC timestamp + var knownTime = new DateTime(2024, 6, 15, 12, 0, 0, DateTimeKind.Utc); + var logEvent = new LogEventInfo(LogLevel.Info, "Logger", "msg") { TimeStamp = knownTime }; + + var serializer = new OtlpProtobufSerializer(); + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + var timestampField = lrFields.Find(f => f.FieldNumber == 1); + + // time_unix_nano = fixed64 (wire type 1), 8 bytes + Assert.Equal(1, timestampField.WireType); + Assert.Equal(8, timestampField.Data.Length); + + var encodedNano = BitConverter.ToUInt64(timestampField.Data, 0); + var expectedNano = OtlpProtobufSerializer.ToUnixNano(knownTime); + Assert.Equal(expectedNano, encodedNano); + } + + [Theory] + [InlineData("Trace", 1)] + [InlineData("Debug", 5)] + [InlineData("Info", 9)] + [InlineData("Warn", 13)] + [InlineData("Error", 17)] + [InlineData("Fatal", 21)] + public void Serialize_LogRecord_SeverityNumberMappedCorrectly(string levelName, int expectedSeverityNumber) + { + var level = LogLevel.FromString(levelName); + var logEvent = LogEventInfo.Create(level, "Logger", "msg"); + var serializer = new OtlpProtobufSerializer(); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + // severity_number = field 2, varint (wire type 0) + var severityNumberField = lrFields.Find(f => f.FieldNumber == 2); + Assert.Equal(0, severityNumberField.WireType); + var severityNumber = (int)ReadVarintFromBytes(severityNumberField.Data); + Assert.Equal(expectedSeverityNumber, severityNumber); + + // severity_text = field 3, length-delimited (wire type 2) + var severityTextField = lrFields.Find(f => f.FieldNumber == 3); + Assert.Equal(2, severityTextField.WireType); + Assert.Equal(levelName, Encoding.UTF8.GetString(severityTextField.Data)); + } + + [Fact] + public void Serialize_LogRecord_BodyEncodedAsAnyValueStringValue() + { + var serializer = new OtlpProtobufSerializer(); + var logEvent = LogEventInfo.Create(LogLevel.Info, "Logger", "hello world"); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + // body = field 5, AnyValue { string string_value = 1 } + var bodyField = lrFields.Find(f => f.FieldNumber == 5); + Assert.Equal(2, bodyField.WireType); + + var anyValueFields = ReadProtobufFields(bodyField.Data); + Assert.Equal(1, anyValueFields[0].FieldNumber); // string_value + Assert.Equal(2, anyValueFields[0].WireType); // length-delimited + Assert.Equal("hello world", Encoding.UTF8.GetString(anyValueFields[0].Data)); + } + + [Fact] + public void Serialize_LogRecord_EmptyBodyFieldOmitted() + { + var serializer = new OtlpProtobufSerializer(); + var logEvent = LogEventInfo.Create(LogLevel.Info, "Logger", ""); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + // body (field 5) should not be present when the rendered body is empty + Assert.DoesNotContain(lrFields, f => f.FieldNumber == 5); + } + + [Fact] + public void Serialize_LogRecord_LoggerNameEncodedAsAttribute() + { + var serializer = new OtlpProtobufSerializer(); + var logEvent = LogEventInfo.Create(LogLevel.Info, "MyApp.Service", "msg"); + + var output = Serialize(serializer, logEvent); + + var attributes = GetLogRecordAttributes(output); + Assert.True(attributes.TryGetValue("LoggerName", out var loggerNameValue)); + + var anyValue = ReadProtobufFields(loggerNameValue.Data); + Assert.Equal(1, anyValue[0].FieldNumber); // string_value + Assert.Equal("MyApp.Service", Encoding.UTF8.GetString(anyValue[0].Data)); + } + + [Fact] + public void Serialize_LogRecord_ExceptionAttributesPresent() + { + var serializer = new OtlpProtobufSerializer(); + var exception = new InvalidOperationException("something went wrong"); + var logEvent = LogEventInfo.Create(LogLevel.Error, "Logger", exception, null, "error occurred"); + + var output = Serialize(serializer, logEvent); + + var attributes = GetLogRecordAttributes(output); + + Assert.True(attributes.ContainsKey("exception.type")); + var typeAnyValue = ReadProtobufFields(attributes["exception.type"].Data); + Assert.Contains("InvalidOperationException", Encoding.UTF8.GetString(typeAnyValue[0].Data)); + + Assert.True(attributes.ContainsKey("exception.message")); + var msgAnyValue = ReadProtobufFields(attributes["exception.message"].Data); + Assert.Equal("something went wrong", Encoding.UTF8.GetString(msgAnyValue[0].Data)); + + Assert.True(attributes.ContainsKey("exception.stacktrace")); + var stackAnyValue = ReadProtobufFields(attributes["exception.stacktrace"].Data); + Assert.Contains("InvalidOperationException", Encoding.UTF8.GetString(stackAnyValue[0].Data)); + } + + [Fact] + public void Serialize_LogRecord_EventPropertiesIncludedWhenEnabled() + { + var serializer = new OtlpProtobufSerializer(); + var logEvent = new LogEventInfo(LogLevel.Info, "Logger", "msg"); + logEvent.Properties["MyKey"] = "MyValue"; + + var output = Serialize(serializer, logEvent); + + var attributes = GetLogRecordAttributes(output); + Assert.True(attributes.ContainsKey("MyKey")); + var anyValue = ReadProtobufFields(attributes["MyKey"].Data); + Assert.Equal("MyValue", Encoding.UTF8.GetString(anyValue[0].Data)); + } + + [Fact] + public void Serialize_LogRecord_TraceIdEncodedAs16Bytes() + { + const string traceIdHex = "0af7651916cd43dd8448eb211c80319c"; + var serializer = new OtlpProtobufSerializer + { + TraceId = NLog.Layouts.Layout.FromMethod(l => System.Diagnostics.ActivityTraceId.CreateFromString(traceIdHex.AsSpan())) + }; + var logEvent = LogEventInfo.Create(LogLevel.Info, "Logger", "msg"); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + // trace_id = field 9, bytes (wire type 2), exactly 16 bytes + var traceIdField = lrFields.Find(f => f.FieldNumber == 9); + Assert.Equal(2, traceIdField.WireType); + Assert.Equal(16, traceIdField.Data.Length); + Assert.Equal(traceIdHex, ToHex(traceIdField.Data)); + } + + [Fact] + public void Serialize_LogRecord_SpanIdEncodedAs8Bytes() + { + const string spanIdHex = "b7ad6b7169203331"; + var serializer = new OtlpProtobufSerializer + { + SpanId = NLog.Layouts.Layout.FromMethod(l => System.Diagnostics.ActivitySpanId.CreateFromString(spanIdHex.AsSpan())), + }; + var logEvent = LogEventInfo.Create(LogLevel.Info, "Logger", "msg"); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + // span_id = field 10, bytes (wire type 2), exactly 8 bytes + var spanIdField = lrFields.Find(f => f.FieldNumber == 10); + Assert.Equal(2, spanIdField.WireType); + Assert.Equal(8, spanIdField.Data.Length); + Assert.Equal(spanIdHex, ToHex(spanIdField.Data)); + } + + [Fact] + public void Serialize_LogRecord_TraceIdAndSpanId_OmittedWhenEmpty() + { + var serializer = new OtlpProtobufSerializer + { + TraceId = NLog.Layouts.Layout.FromMethod(l => default(System.Diagnostics.ActivityTraceId?)), + SpanId = NLog.Layouts.Layout.FromMethod(l => default(System.Diagnostics.ActivitySpanId?)), + }; + var logEvent = LogEventInfo.Create(LogLevel.Info, "Logger", "msg"); + + var output = Serialize(serializer, logEvent); + + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + + Assert.DoesNotContain(lrFields, f => f.FieldNumber == 9); + Assert.DoesNotContain(lrFields, f => f.FieldNumber == 10); + } + + #endregion + + #region Resource tests + + [Fact] + public void Serialize_Resource_AdditionalResourceAttributesPresent() + { + var serializer = new OtlpProtobufSerializer(); + var resourceAttributes = new List + { + new TargetPropertyWithContext { Name = "deployment.environment", Layout = "production" }, + new TargetPropertyWithContext { Name = "host.name", Layout = "server01" }, + }; + + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "msg"), resourceAttributes); + + var resourceAttrs = GetResourceAttributes(output); + Assert.True(resourceAttrs.ContainsKey("deployment.environment")); + var envAnyValue = ReadProtobufFields(resourceAttrs["deployment.environment"].Data); + Assert.Equal("production", Encoding.UTF8.GetString(envAnyValue[0].Data)); + + Assert.True(resourceAttrs.ContainsKey("host.name")); + var hostAnyValue = ReadProtobufFields(resourceAttrs["host.name"].Data); + Assert.Equal("server01", Encoding.UTF8.GetString(hostAnyValue[0].Data)); + } + + [Fact] + public void Serialize_Resource_AttributeWithEmptyValueIncludedWhenForced() + { + var serializer = new OtlpProtobufSerializer(); + var resourceAttributes = new List + { + new TargetPropertyWithContext + { + Name = "forced.attr", + Layout = "", + IncludeEmptyValue = true, + }, + }; + + var output = Serialize(serializer, LogEventInfo.Create(LogLevel.Info, "Logger", "msg"), resourceAttributes); + + var resourceAttrs = GetResourceAttributes(output); + Assert.True(resourceAttrs.ContainsKey("forced.attr")); + } + + #endregion + + #region AnyValue encoding tests + + [Fact] + public void AnyValue_Bool_True_EncodedAsVarintField2Value1() + { + var output = SerializeWithProperty("flag", true); + var anyValue = GetPropertyAnyValue(output, "flag"); + + // AnyValue { bool bool_value = 2 } → field 2, wire type 0 (varint), value 1 + Assert.Equal(2, anyValue.FieldNumber); + Assert.Equal(0, anyValue.WireType); + Assert.Equal(1UL, ReadVarintFromBytes(anyValue.Data)); + } + + [Fact] + public void AnyValue_Bool_False_EncodedAsVarintField2Value0() + { + var output = SerializeWithProperty("flag", false); + var anyValue = GetPropertyAnyValue(output, "flag"); + + Assert.Equal(2, anyValue.FieldNumber); + Assert.Equal(0, anyValue.WireType); + Assert.Equal(0UL, ReadVarintFromBytes(anyValue.Data)); + } + + [Fact] + public void AnyValue_Int32_EncodedAsVarintField3() + { + var output = SerializeWithProperty("count", 42); + var anyValue = GetPropertyAnyValue(output, "count"); + + // AnyValue { int64 int_value = 3 } → field 3, wire type 0 (varint) + Assert.Equal(3, anyValue.FieldNumber); + Assert.Equal(0, anyValue.WireType); + Assert.Equal(42UL, ReadVarintFromBytes(anyValue.Data)); + } + + [Fact] + public void AnyValue_NegativeInt_EncodedAsVarintField3WithZigzagTwosComplement() + { + var output = SerializeWithProperty("neg", -1); + var anyValue = GetPropertyAnyValue(output, "neg"); + + // Negative values use two's complement encoding (int64 → ulong cast) + Assert.Equal(3, anyValue.FieldNumber); + Assert.Equal(0, anyValue.WireType); + var raw = ReadVarintFromBytes(anyValue.Data); + Assert.Equal(-1L, unchecked((long)raw)); + } + + [Fact] + public void AnyValue_UInt64_EncodedAsVarintField3() + { + var output = SerializeWithProperty("big", ulong.MaxValue); + var anyValue = GetPropertyAnyValue(output, "big"); + + Assert.Equal(3, anyValue.FieldNumber); + Assert.Equal(0, anyValue.WireType); + Assert.Equal(ulong.MaxValue, ReadVarintFromBytes(anyValue.Data)); + } + + [Fact] + public void AnyValue_Double_EncodedAsFixed64Field4() + { + var output = SerializeWithProperty("ratio", 3.14); + var anyValue = GetPropertyAnyValue(output, "ratio"); + + // AnyValue { double double_value = 4 } → field 4, wire type 1 (fixed64) + Assert.Equal(4, anyValue.FieldNumber); + Assert.Equal(1, anyValue.WireType); + Assert.Equal(8, anyValue.Data.Length); + Assert.Equal(3.14, BitConverter.ToDouble(anyValue.Data, 0), precision: 10); + } + + [Fact] + public void AnyValue_Float_EncodedAsFixed64Field4() + { + var output = SerializeWithProperty("f", 1.5f); + var anyValue = GetPropertyAnyValue(output, "f"); + + Assert.Equal(4, anyValue.FieldNumber); + Assert.Equal(1, anyValue.WireType); + } + + [Fact] + public void AnyValue_String_EncodedAsLengthDelimitedField1() + { + var output = SerializeWithProperty("msg", "hello"); + var anyValue = GetPropertyAnyValue(output, "msg"); + + // AnyValue { string string_value = 1 } → field 1, wire type 2 (length-delimited) + Assert.Equal(1, anyValue.FieldNumber); + Assert.Equal(2, anyValue.WireType); + Assert.Equal("hello", Encoding.UTF8.GetString(anyValue.Data)); + } + + [Fact] + public void AnyValue_Enum_EncodedAsStringField1() + { + var output = SerializeWithProperty("level", System.Diagnostics.TraceEventType.Warning); + var anyValue = GetPropertyAnyValue(output, "level"); + + // Enums are serialized as their name string + Assert.Equal(1, anyValue.FieldNumber); + Assert.Equal(2, anyValue.WireType); + Assert.Equal("Warning", Encoding.UTF8.GetString(anyValue.Data)); + } + + [Fact] + public void AnyValue_DateTime_EncodedAsIso8601StringField1() + { + var dt = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc); + var output = SerializeWithProperty("ts", dt); + var anyValue = GetPropertyAnyValue(output, "ts"); + + Assert.Equal(1, anyValue.FieldNumber); + Assert.Equal(2, anyValue.WireType); + var str = Encoding.UTF8.GetString(anyValue.Data); + Assert.Contains("2024-01-15", str); + } + + [Fact] + public void AnyValue_DoubleNaN_EncodedAsFixed64Field4() + { + var output = SerializeWithProperty("nan", double.NaN); + var anyValue = GetPropertyAnyValue(output, "nan"); + + Assert.Equal(4, anyValue.FieldNumber); + Assert.Equal(1, anyValue.WireType); + Assert.Equal(8, anyValue.Data.Length); + Assert.True(double.IsNaN(BitConverter.ToDouble(anyValue.Data, 0))); + } + + [Fact] + public void AnyValue_DoublePositiveInfinity_EncodedAsFixed64Field4() + { + var output = SerializeWithProperty("inf", double.PositiveInfinity); + var anyValue = GetPropertyAnyValue(output, "inf"); + + Assert.Equal(4, anyValue.FieldNumber); + Assert.Equal(1, anyValue.WireType); + Assert.True(double.IsPositiveInfinity(BitConverter.ToDouble(anyValue.Data, 0))); + } + + [Fact] + public void AnyValue_List_EncodedAsArrayValueField5() + { + var list = new List { "alpha", 99, true }; + var output = SerializeWithProperty("items", list); + var anyValueWrapper = GetPropertyAnyValueWrapper(output, "items"); + + // AnyValue { ArrayValue array_value = 5 } → field 5, wire type 2 + var wrapperFields = ReadProtobufFields(anyValueWrapper.Data); + var arrayField = wrapperFields.Find(f => f.FieldNumber == 5); + Assert.Equal(2, arrayField.WireType); + + // ArrayValue { repeated AnyValue values = 1 } + var arrayEntries = ReadProtobufFields(arrayField.Data); + Assert.Equal(3, arrayEntries.Count); + Assert.All(arrayEntries, e => Assert.Equal(1, e.FieldNumber)); + + // "alpha" → string_value (field 1) + var elem0 = ReadProtobufFields(arrayEntries[0].Data); + Assert.Equal(1, elem0[0].FieldNumber); + Assert.Equal("alpha", Encoding.UTF8.GetString(elem0[0].Data)); + + // 99 → int_value (field 3) + var elem1 = ReadProtobufFields(arrayEntries[1].Data); + Assert.Equal(3, elem1[0].FieldNumber); + Assert.Equal(99UL, ReadVarintFromBytes(elem1[0].Data)); + + // true → bool_value (field 2) + var elem2 = ReadProtobufFields(arrayEntries[2].Data); + Assert.Equal(2, elem2[0].FieldNumber); + Assert.Equal(1UL, ReadVarintFromBytes(elem2[0].Data)); + } + + [Fact] + public void AnyValue_Dictionary_EncodedAsKvListValueField6() + { + var dict = new Dictionary + { + ["key1"] = "value1", + ["key2"] = 42, + }; + var output = SerializeWithProperty("map", dict); + var anyValueWrapper = GetPropertyAnyValueWrapper(output, "map"); + + // AnyValue { KeyValueList kvlist_value = 6 } → field 6, wire type 2 + var wrapperFields = ReadProtobufFields(anyValueWrapper.Data); + var kvListField = wrapperFields.Find(f => f.FieldNumber == 6); + Assert.Equal(2, kvListField.WireType); + + // KeyValueList { repeated KeyValue values = 1 } + var kvEntries = ReadProtobufFields(kvListField.Data); + Assert.Equal(2, kvEntries.Count); + + var parsed = new Dictionary(); + foreach (var entry in kvEntries) + { + var entryFields = ReadProtobufFields(entry.Data); + var key = Encoding.UTF8.GetString(entryFields.Find(f => f.FieldNumber == 1).Data); + parsed[key] = entryFields.Find(f => f.FieldNumber == 2); + } + + // key1 → string_value + var key1AnyValue = ReadProtobufFields(parsed["key1"].Data); + Assert.Equal(1, key1AnyValue[0].FieldNumber); + Assert.Equal("value1", Encoding.UTF8.GetString(key1AnyValue[0].Data)); + + // key2 → int_value + var key2AnyValue = ReadProtobufFields(parsed["key2"].Data); + Assert.Equal(3, key2AnyValue[0].FieldNumber); + Assert.Equal(42UL, ReadVarintFromBytes(key2AnyValue[0].Data)); + } + + [Fact] + public void AnyValue_NullValue_AttributeKeyPresentWithoutValueField() + { + var output = SerializeWithProperty("n", (object)null); + + // null is converted to empty string → WriteStringField skips it → AnyValue is omitted. + // The KeyValue attribute is still written (key = "n") but carries no value submessage. + var logRecord = NavigateToLogRecord(output); + var lrFields = ReadProtobufFields(logRecord); + var attributeFields = lrFields.FindAll(f => f.FieldNumber == 6); + + // Find the KeyValue whose key is "n" + var found = false; + foreach (var kv in attributeFields) + { + var kvFields = ReadProtobufFields(kv.Data); + var keyField = kvFields.Find(f => f.FieldNumber == 1); + if (keyField.Data != null && Encoding.UTF8.GetString(keyField.Data) == "n") + { + found = true; + // No value field (field 2) should be present + Assert.DoesNotContain(kvFields, f => f.FieldNumber == 2); + break; + } + } + Assert.True(found, "Attribute 'n' should be present in the log record"); + } + + #endregion + + #region Helper methods for building serializer output + + private static byte[] Serialize(OtlpProtobufSerializer serializer, LogEventInfo logEvent, string scopeName = "NLog") + { + var output = new MemoryStream(); + using (var builder = serializer.BeginRecord(output)) + { + var logProperties = logEvent.HasProperties ? logEvent.Properties.ToDictionary(d => d.Key.ToString(), d => d.Value) : null; + builder.AddScopeLogs(scopeName, logEvent, logEvent.FormattedMessage, logProperties); + } + return output.ToArray(); + } + + private static byte[] Serialize(OtlpProtobufSerializer serializer, LogEventInfo logEvent, IList resourceAttributes, string scopeName = "NLog") + { + var output = new MemoryStream(); + using (var builder = serializer.BeginRecord(output)) + { + for (int i = 0; i < resourceAttributes.Count; i++) + builder.AddResourceAttribute(resourceAttributes[i].Name, resourceAttributes[i].Layout?.Render(logEvent)); + + var logProperties = logEvent.HasProperties ? logEvent.Properties.ToDictionary(d => d.Key.ToString(), d => d.Value) : null; + builder.AddScopeLogs(scopeName, logEvent, logEvent.FormattedMessage, logProperties); + } + return output.ToArray(); + } + + private static byte[] SerializeAll(OtlpProtobufSerializer serializer, IList logEvents, string scopeName = "NLog") + { + var output = new MemoryStream(); + for (int i = 0; i < logEvents.Count; i++) + { + using (var builder = serializer.BeginRecord(output)) + { + var logProperties = logEvents[i].HasProperties ? logEvents[i].Properties.ToDictionary(d => d.Key.ToString(), d => d.Value) : null; + builder.AddScopeLogs(scopeName, logEvents[i], logEvents[i].FormattedMessage, logProperties); + } + } + return output.ToArray(); + } + + private static byte[] SerializeWithProperty(string key, object value) + { + var serializer = new OtlpProtobufSerializer(); + var logEvent = new LogEventInfo(LogLevel.Info, "Logger", "msg"); + logEvent.Properties[key] = value; + return Serialize(serializer, logEvent); + } + + #endregion + + #region Protobuf navigation helpers + + private static byte[] NavigateToScopeLogs(byte[] data) + { + var topFields = ReadProtobufFields(data); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + return resourceLogs.Find(f => f.FieldNumber == 2).Data; + } + + private static byte[] NavigateToLogRecord(byte[] data) + { + var scopeLogs = NavigateToScopeLogs(data); + var slFields = ReadProtobufFields(scopeLogs); + return slFields.Find(f => f.FieldNumber == 2).Data; + } + + private static Dictionary GetLogRecordAttributes(byte[] data) + { + var logRecord = NavigateToLogRecord(data); + var lrFields = ReadProtobufFields(logRecord); + var attributeFields = lrFields.FindAll(f => f.FieldNumber == 6); + return ParseKeyValueFields(attributeFields); + } + + private static Dictionary GetResourceAttributes(byte[] data) + { + var topFields = ReadProtobufFields(data); + var resourceLogs = ReadProtobufFields(topFields.Find(f => f.FieldNumber == 1).Data); + var resource = ReadProtobufFields(resourceLogs.Find(f => f.FieldNumber == 1).Data); + var attributeFields = resource.FindAll(f => f.FieldNumber == 1); + return ParseKeyValueFields(attributeFields); + } + + private static Dictionary ParseKeyValueFields(List kvFields) + { + var result = new Dictionary(); + foreach (var kv in kvFields) + { + var fields = ReadProtobufFields(kv.Data); + var keyField = fields.Find(f => f.FieldNumber == 1); + var valueField = fields.Find(f => f.FieldNumber == 2); + if (keyField.Data?.Length > 0) + result[Encoding.UTF8.GetString(keyField.Data)] = valueField; + } + return result; + } + + /// + /// Returns the decoded inner AnyValue field (e.g. string_value=1, int_value=3) for an event property. + /// + private static ProtobufField GetPropertyAnyValue(byte[] data, string propertyKey) + { + var wrapper = GetPropertyAnyValueWrapper(data, propertyKey); + var anyValueFields = ReadProtobufFields(wrapper.Data); + return anyValueFields[0]; + } + + /// + /// Returns the raw AnyValue submessage field (field 2 of the KeyValue) for an event property. + /// + private static ProtobufField GetPropertyAnyValueWrapper(byte[] data, string propertyKey) + { + var attrs = GetLogRecordAttributes(data); + Assert.True(attrs.TryGetValue(propertyKey, out var wrapper), $"Property '{propertyKey}' not found in log record attributes"); + return wrapper; + } + + private static string ToHex(byte[] bytes) + { + return BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant(); + } + + #endregion + + #region Protobuf reader helpers + + private struct ProtobufField + { + public int FieldNumber; + public int WireType; + public byte[] Data; + } + + private static List ReadProtobufFields(byte[] data) + { + var fields = new List(); + int offset = 0; + while (offset < data.Length) + { + var tag = ReadVarint(data, ref offset); + var fieldNumber = (int)(tag >> 3); + var wireType = (int)(tag & 0x7); + + byte[] fieldData; + switch (wireType) + { + case 0: // varint + var start = offset; + ReadVarint(data, ref offset); + fieldData = new byte[offset - start]; + Array.Copy(data, start, fieldData, 0, fieldData.Length); + break; + case 1: // 64-bit fixed + fieldData = new byte[8]; + Array.Copy(data, offset, fieldData, 0, 8); + offset += 8; + break; + case 2: // length-delimited + var length = (int)ReadVarint(data, ref offset); + fieldData = new byte[length]; + Array.Copy(data, offset, fieldData, 0, length); + offset += length; + break; + case 5: // 32-bit fixed + fieldData = new byte[4]; + Array.Copy(data, offset, fieldData, 0, 4); + offset += 4; + break; + default: + throw new InvalidOperationException($"Unknown protobuf wire type {wireType} at offset {offset}"); + } + + fields.Add(new ProtobufField { FieldNumber = fieldNumber, WireType = wireType, Data = fieldData }); + } + return fields; + } + + private static ulong ReadVarint(byte[] data, ref int offset) + { + ulong value = 0; + int shift = 0; + while (offset < data.Length) + { + var b = data[offset++]; + value |= (ulong)(b & 0x7F) << shift; + if ((b & 0x80) == 0) + break; + shift += 7; + } + return value; + } + + private static ulong ReadVarintFromBytes(byte[] data) + { + int offset = 0; + return ReadVarint(data, ref offset); + } + + #endregion + } +}