This is an automated email from the ASF dual-hosted git repository.
ptupitsyn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new aebca24f0a IGNITE-20468 .NET: Fix test timeouts (#2626)
aebca24f0a is described below
commit aebca24f0a39aa195cea97e540419482f1753fb8
Author: Pavel Tupitsyn <ptupit...@apache.org>
AuthorDate: Tue Sep 26 17:40:31 2023 +0300
IGNITE-20468 .NET: Fix test timeouts (#2626)
Improve .NET tests reliability and observability:
* Fix `IgniteProxy` - use two threads for bi-directional communication (it
was fully sequential, causing heartbeat timeouts when some other operation
takes a few seconds)
* Fix `IgniteServerBase` dispose logic
* Add global 15 second timeout for all tests to avoid waiting forever
* Enable client-side logging in base test class
`TestExecuteColocated` is still flaky because of a known deadlock issue in
`SchemaRegistry`: https://issues.apache.org/jira/browse/IGNITE-17931
---
.../Apache.Ignite.Tests/Apache.Ignite.Tests.csproj | 7 ++
.../Apache.Ignite.Tests/IgniteClientExtensions.cs | 2 +-
.../dotnet/Apache.Ignite.Tests/IgniteProxy.cs | 90 ++++++++++++++++------
.../dotnet/Apache.Ignite.Tests/IgniteServerBase.cs | 37 +++++----
.../dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs | 17 +++-
.../dotnet/Apache.Ignite.Tests/ReconnectTests.cs | 3 +-
.../Apache.Ignite.Tests/Table/DataStreamerTests.cs | 2 +-
.../Table/RecordViewCustomMappingTest.cs | 2 +-
.../Table/RecordViewDefaultMappingTest.cs | 2 +-
9 files changed, 112 insertions(+), 50 deletions(-)
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Apache.Ignite.Tests.csproj
b/modules/platforms/dotnet/Apache.Ignite.Tests/Apache.Ignite.Tests.csproj
index 232c2508e4..0e776fe96c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/Apache.Ignite.Tests.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/Apache.Ignite.Tests.csproj
@@ -49,4 +49,11 @@
</None>
</ItemGroup>
+ <ItemGroup>
+ <AssemblyAttribute Include="NUnit.Framework.TimeoutAttribute">
+ <_Parameter1>15000</_Parameter1>
+ <_Parameter1_TypeName>System.Int32</_Parameter1_TypeName>
+ </AssemblyAttribute>
+ </ItemGroup>
+
</Project>
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteClientExtensions.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteClientExtensions.cs
index 1f32a30cc0..b50800411c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteClientExtensions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteClientExtensions.cs
@@ -28,7 +28,7 @@ public static class IgniteClientExtensions
/// <param name="client">Client.</param>
/// <param name="count">Connection count.</param>
/// <param name="timeoutMs">Timeout.</param>
- public static void WaitForConnections(this IIgniteClient client, int
count, int timeoutMs = 5000) =>
+ public static void WaitForConnections(this IIgniteClient client, int
count, int timeoutMs = 15_000) =>
TestUtils.WaitForCondition(
condition: () => client.GetConnections().Count == count,
timeoutMs: timeoutMs,
diff --git a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteProxy.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteProxy.cs
index 57449fb49e..979c455ef7 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteProxy.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteProxy.cs
@@ -24,6 +24,7 @@ using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Threading;
+using System.Threading.Tasks;
using Internal.Proto;
/// <summary>
@@ -55,47 +56,86 @@ public sealed class IgniteProxy : IgniteServerBase
protected override void Handle(Socket handler, CancellationToken
cancellationToken)
{
- int messageCount = 0;
-
- while (!cancellationToken.IsCancellationRequested)
+ // Handshake.
{
- if (messageCount == 0)
- {
- // Forward magic from client to server.
- using var magic = ReceiveBytes(handler, 4);
- _socket.Send(magic.AsMemory().Span);
- }
+ // Forward magic from client to server.
+ using var magic = ReceiveBytes(handler, 4);
+ _socket.Send(magic.AsMemory().Span);
- // Receive from client.
+ // Receive handshake from client.
var msgSize = ReceiveMessageSize(handler);
using var msg = ReceiveBytes(handler, msgSize);
- if (messageCount > 0)
- {
- _ops.Enqueue((ClientOp)msg.GetReader().ReadInt32());
- }
-
- // Forward to server.
+ // Forward handshake to server.
_socket.Send(BitConverter.GetBytes(IPAddress.HostToNetworkOrder(msgSize)));
_socket.Send(msg.AsMemory().Span);
- // Receive from server.
- if (messageCount == 0)
- {
- // Forward magic from server to client.
- using var serverMagic = ReceiveBytes(_socket, 4);
- handler.Send(serverMagic.AsMemory().Span);
- }
+ // Forward magic from server to client.
+ using var serverMagic = ReceiveBytes(_socket, 4);
+ handler.Send(serverMagic.AsMemory().Span);
+ // Receive handshake from server.
var serverMsgSize = ReceiveMessageSize(_socket);
using var serverMsg = ReceiveBytes(_socket, serverMsgSize);
// Forward to client.
handler.Send(BitConverter.GetBytes(IPAddress.HostToNetworkOrder(serverMsgSize)));
handler.Send(serverMsg.AsMemory().Span);
-
- messageCount++;
}
+
+ // Separate relay loops for each direction: don't block heartbeats
while some request is being processed.
+ // Client -> Server.
+ var clientToServerRelay = Task.Run(
+ () =>
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ // Receive from client.
+ var msgSize = ReceiveMessageSize(handler);
+ using var msg = ReceiveBytes(handler, msgSize);
+ _ops.Enqueue((ClientOp)msg.GetReader().ReadInt32());
+
+ // Forward to server.
+
_socket.Send(BitConverter.GetBytes(IPAddress.HostToNetworkOrder(msgSize)));
+ _socket.Send(msg.AsMemory().Span);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine($"Error in IgniteProxy Client ->
Server relay (lastOp = {_ops.Last()}: {e}");
+ throw;
+ }
+ }
+ },
+ cancellationToken);
+
+ // Server -> Client.
+ var serverToClientRelay = Task.Run(
+ () =>
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ // Receive from server.
+ var serverMsgSize = ReceiveMessageSize(_socket);
+ using var serverMsg = ReceiveBytes(_socket,
serverMsgSize);
+
+ // Forward to client.
+
handler.Send(BitConverter.GetBytes(IPAddress.HostToNetworkOrder(serverMsgSize)));
+ handler.Send(serverMsg.AsMemory().Span);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine($"Error in IgniteProxy Server ->
Client relay (lastOp = {_ops.Last()}: {e}");
+ throw;
+ }
+ }
+ },
+ cancellationToken);
+
+ Task.WhenAll(clientToServerRelay,
serverToClientRelay).Wait(millisecondsTimeout: 5000, cancellationToken);
}
protected override void Dispose(bool disposing)
diff --git a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteServerBase.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteServerBase.cs
index 961ba14280..f9efb1565a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteServerBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteServerBase.cs
@@ -70,24 +70,7 @@ public abstract class IgniteServerBase : IDisposable
public void Dispose()
{
- lock (_disposeSyncRoot)
- {
- if (_disposed)
- {
- return;
- }
-
- _cts.Cancel();
- _handler?.Dispose();
- _listener.Disconnect(false);
- _listener.Dispose();
- _cts.Dispose();
-
- Dispose(true);
- GC.SuppressFinalize(this);
-
- _disposed = true;
- }
+ Dispose(true);
}
internal static int ReceiveMessageSize(Socket handler)
@@ -125,7 +108,23 @@ public abstract class IgniteServerBase : IDisposable
{
if (disposing)
{
- // No-op.
+ lock (_disposeSyncRoot)
+ {
+ if (_disposed)
+ {
+ return;
+ }
+
+ _cts.Cancel();
+ _handler?.Dispose();
+ _listener.Disconnect(false);
+ _listener.Dispose();
+ _cts.Dispose();
+
+ GC.SuppressFinalize(this);
+
+ _disposed = true;
+ }
}
}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs
index 4de5693910..c5f46c5953 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs
@@ -29,6 +29,8 @@ namespace Apache.Ignite.Tests
/// <summary>
/// Base class for client tests.
+ /// <para />
+ /// NOTE: Timeout is set for the entire assembly in csproj file.
/// </summary>
public class IgniteTestsBase
{
@@ -124,13 +126,23 @@ namespace Apache.Ignite.Tests
_eventListener.Dispose();
}
+ [SetUp]
+ public void SetUp()
+ {
+ Console.WriteLine("SetUp: " +
TestContext.CurrentContext.Test.Name);
+ }
+
[TearDown]
public void TearDown()
{
+ Console.WriteLine("TearDown start: " +
TestContext.CurrentContext.Test.Name);
+
CheckPooledBufferLeak();
_disposables.ForEach(x => x.Dispose());
_disposables.Clear();
+
+ Console.WriteLine("TearDown end: " +
TestContext.CurrentContext.Test.Name);
}
internal static string
GetRequestTargetNodeName(IEnumerable<IgniteProxy> proxies, ClientOp op)
@@ -172,7 +184,10 @@ namespace Apache.Ignite.Tests
};
protected static IgniteClientConfiguration
GetConfig(IEnumerable<IgniteProxy> proxies) =>
- new(proxies.Select(x => x.Endpoint).ToArray());
+ new(proxies.Select(x => x.Endpoint).ToArray())
+ {
+ Logger = new ConsoleLogger { MinLevel = LogLevel.Trace }
+ };
protected List<IgniteProxy> GetProxies()
{
diff --git a/modules/platforms/dotnet/Apache.Ignite.Tests/ReconnectTests.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/ReconnectTests.cs
index 23b22e563f..695377ded0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/ReconnectTests.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/ReconnectTests.cs
@@ -86,7 +86,8 @@ public class ReconnectTests
var cfg = new IgniteClientConfiguration
{
HeartbeatInterval = TimeSpan.FromMilliseconds(100),
- ReconnectInterval = TimeSpan.FromMilliseconds(300)
+ ReconnectInterval = TimeSpan.FromMilliseconds(300),
+ Logger = new ConsoleLogger { MinLevel = LogLevel.Trace }
};
using var servers = FakeServerGroup.Create(10);
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/DataStreamerTests.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/DataStreamerTests.cs
index 5d8c0f1bf6..2ace822027 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/DataStreamerTests.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/DataStreamerTests.cs
@@ -37,7 +37,7 @@ public class DataStreamerTests : IgniteTestsBase
private const int Count = 100;
[SetUp]
- public async Task SetUp() =>
+ public async Task DeleteAll() =>
await TupleView.DeleteAllAsync(null, Enumerable.Range(0,
Count).Select(x => GetTuple(x)));
[Test]
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewCustomMappingTest.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewCustomMappingTest.cs
index 5f5a7bc4d6..5286e5a5ad 100644
---
a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewCustomMappingTest.cs
+++
b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewCustomMappingTest.cs
@@ -37,7 +37,7 @@ public class RecordViewCustomMappingTest : IgniteTestsBase
private const string Val = "val1";
[SetUp]
- public async Task SetUp()
+ public async Task InsertData()
{
await Table.RecordBinaryView.UpsertAsync(null, GetTuple(Key, Val));
}
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewDefaultMappingTest.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewDefaultMappingTest.cs
index 39eb7dd43a..a71c7a2fd4 100644
---
a/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewDefaultMappingTest.cs
+++
b/modules/platforms/dotnet/Apache.Ignite.Tests/Table/RecordViewDefaultMappingTest.cs
@@ -30,7 +30,7 @@ namespace Apache.Ignite.Tests.Table
public class RecordViewDefaultMappingTest : IgniteTestsBase
{
[SetUp]
- public async Task SetUp()
+ public async Task InsertData()
{
await Table.RecordBinaryView.UpsertAsync(null, GetTuple(1, "2"));
}
