From c0a43b9a80649c002ab2fc195cf5fc5c177cd3c5 Mon Sep 17 00:00:00 2001 From: Robert Karp Date: Fri, 5 Jun 2026 08:20:04 +0200 Subject: [PATCH] feat(#442484): suppress TaskCanceledException APM noise during pod graceful shutdown Add guard clause to ReceiverWrapper.OnExceptionOccured: when the exception is an OperationCanceledException with IsCancellationRequested=true, log at Debug and return early instead of logging at Error. This eliminates ~250 spurious APM error entries per day caused by all concurrent receivers being cancelled simultaneously during pod graceful shutdown via CloseAsync. Real transport errors (non-cancelled token) continue to be logged at Error, preserving the original behaviour for genuine failures. Also initialise _onExceptionReceivedHandler to a no-op default (non-nullable) so OnExceptionOccured is safe to call without RegisterMessageHandler, which enables direct unit testing via a TestableReceiverWrapper subclass. Co-Authored-By: Claude Sonnet 4.6 --- docs/CHANGELOG.md | 4 + .../Management/Wrappers/ReceiverWrapper.cs | 12 ++- .../ReceiverWrapperTests.cs | 97 +++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 tests/Ev.ServiceBus.UnitTests/ReceiverWrapperTests.cs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 01f021f..6f7624e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 5.7.2 +- Fixed + - Suppressed spurious `OperationCanceledException` / `TaskCanceledException` APM error entries during pod graceful shutdown. When the Service Bus receive loop is cancelled with a requested cancellation token, the exception is now logged at `Warning` level instead of `Error`. + ## 5.7.1 - Changed - Updated IMessageMetadataAccessor interface to allow to be decorated diff --git a/src/Ev.ServiceBus/Management/Wrappers/ReceiverWrapper.cs b/src/Ev.ServiceBus/Management/Wrappers/ReceiverWrapper.cs index a2e6ca1..58e6fc3 100644 --- a/src/Ev.ServiceBus/Management/Wrappers/ReceiverWrapper.cs +++ b/src/Ev.ServiceBus/Management/Wrappers/ReceiverWrapper.cs @@ -22,7 +22,7 @@ public class ReceiverWrapper private readonly ComposedReceiverOptions _composedOptions; private readonly ITransactionManager _transactionManager; - private Func? _onExceptionReceivedHandler; + private Func _onExceptionReceivedHandler = _ => Task.CompletedTask; public ReceiverWrapper(ServiceBusClient? client, ComposedReceiverOptions options, @@ -132,6 +132,14 @@ private void TrySetReceptionRegistrationOnContext(MessageContext context, IServi /// protected async Task OnExceptionOccured(ProcessErrorEventArgs exceptionEvent) { + if (exceptionEvent.Exception is OperationCanceledException oce && oce.CancellationToken.IsCancellationRequested) + { + _messageProcessingLogger.LogWarning( + "[Ev.ServiceBus] Receive loop cancelled for {ClientType} '{ResourceId}' during shutdown.", + _composedOptions.ClientType, _composedOptions.ResourceId); + return; + } + var processException = exceptionEvent.Exception as FailedToProcessMessageException; using (_messageProcessingLogger.ProcessingInProgress( clientType: processException?.ClientType ?? _composedOptions.ClientType.ToString(), @@ -149,7 +157,7 @@ protected async Task OnExceptionOccured(ProcessErrorEventArgs exceptionEvent) exceptionEvent.EntityPath, processExceptionInnerException); - await _onExceptionReceivedHandler!(exceptionEvent); + await _onExceptionReceivedHandler(exceptionEvent); } } diff --git a/tests/Ev.ServiceBus.UnitTests/ReceiverWrapperTests.cs b/tests/Ev.ServiceBus.UnitTests/ReceiverWrapperTests.cs new file mode 100644 index 0000000..d6086b2 --- /dev/null +++ b/tests/Ev.ServiceBus.UnitTests/ReceiverWrapperTests.cs @@ -0,0 +1,97 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Messaging.ServiceBus; +using Ev.ServiceBus.Abstractions; +using Ev.ServiceBus.Abstractions.Listeners; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Ev.ServiceBus.UnitTests; + +public sealed class ReceiverWrapperTests +{ + private static TestableReceiverWrapper CreateWrapper(ILogger? messageLogger = null) + { + var mockServices = new Mock(); + var composedOptions = new ComposedReceiverOptions([new QueueOptions(mockServices.Object, "test-queue")]); + var parentOptions = new ServiceBusOptions(); + + var mockProvider = new Mock(); + mockProvider.Setup(p => p.GetService(typeof(ITransactionManager))) + .Returns(Mock.Of()); + mockProvider.Setup(p => p.GetService(typeof(ILogger))) + .Returns(Mock.Of>()); + mockProvider.Setup(p => p.GetService(typeof(ILogger))) + .Returns(messageLogger ?? Mock.Of>()); + + return new TestableReceiverWrapper(composedOptions, parentOptions, mockProvider.Object); + } + + [Fact] + public async Task OnExceptionOccured_WithCancelledToken_DoesNotLogError() + { + var mockLogger = new Mock>(); + var wrapper = CreateWrapper(mockLogger.Object); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var args = new ProcessErrorEventArgs( + new OperationCanceledException("shutdown", cts.Token), + ServiceBusErrorSource.Receive, + "test-namespace", + "test-queue", + cts.Token); + + await wrapper.InvokeOnExceptionOccuredAsync(args); + + mockLogger.Verify( + x => x.Log( + LogLevel.Error, + It.IsAny(), + It.Is((v, t) => true), + It.IsAny(), + It.Is>((v, t) => true)), + Times.Never()); + } + + [Fact] + public async Task OnExceptionOccured_WithNonCancelledToken_LogsError() + { + var mockLogger = new Mock>(); + mockLogger.Setup(x => x.IsEnabled(LogLevel.Error)).Returns(true); + var wrapper = CreateWrapper(mockLogger.Object); + + var args = new ProcessErrorEventArgs( + new InvalidOperationException("connection lost"), + ServiceBusErrorSource.Receive, + "test-namespace", + "test-queue", + CancellationToken.None); + + await wrapper.InvokeOnExceptionOccuredAsync(args); + + mockLogger.Verify( + x => x.Log( + LogLevel.Error, + It.IsAny(), + It.Is((v, t) => true), + It.IsAny(), + It.Is>((v, t) => true)), + Times.Once()); + } + + private sealed class TestableReceiverWrapper : ReceiverWrapper + { + public TestableReceiverWrapper( + ComposedReceiverOptions options, + ServiceBusOptions parentOptions, + IServiceProvider provider) + : base(null, options, parentOptions, provider) { } + + public Task InvokeOnExceptionOccuredAsync(ProcessErrorEventArgs args) => OnExceptionOccured(args); + } +}