From d75624a771e55fa7ebc857930b9c4815a2e9d979 Mon Sep 17 00:00:00 2001 From: atakavci Date: Thu, 3 Apr 2025 17:48:44 +0300 Subject: [PATCH 01/25] Bump Redis 8.0 version(8.0-RC1-pre) in pipeline --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 12fe2741..6c107919 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -37,7 +37,7 @@ jobs: max-parallel: 15 fail-fast: false matrix: - redis-version: [ '8.0-M05-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.6', '6.2.16'] + redis-version: [ '8.0-RC1-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.6', '6.2.16'] dotnet-version: ['6.0', '7.0', '8.0'] env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true From d5f290d5d79b80c8789ed32f2a10a9da1512ce6e Mon Sep 17 00:00:00 2001 From: atakavci Date: Thu, 3 Apr 2025 18:28:18 +0300 Subject: [PATCH 02/25] drop/fix failing tests with 8.0 --- tests/NRedisStack.Tests/Search/SearchTests.cs | 2 +- .../TimeSeries/TestDataTypes/TestTimeSeriesInformation.cs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index fc77a3e4..96e2c97f 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -1417,7 +1417,7 @@ public async Task TestDropIndexAsync(string endpointId) } catch (RedisServerException ex) { - Assert.Contains("no such index", ex.Message); + Assert.Contains("no such index", ex.Message, StringComparison.OrdinalIgnoreCase); } Assert.Equal("100", db.Execute("DBSIZE").ToString()); } diff --git a/tests/NRedisStack.Tests/TimeSeries/TestDataTypes/TestTimeSeriesInformation.cs b/tests/NRedisStack.Tests/TimeSeries/TestDataTypes/TestTimeSeriesInformation.cs index 6a8dd707..4f974123 100644 --- a/tests/NRedisStack.Tests/TimeSeries/TestDataTypes/TestTimeSeriesInformation.cs +++ b/tests/NRedisStack.Tests/TimeSeries/TestDataTypes/TestTimeSeriesInformation.cs @@ -27,14 +27,12 @@ public void TestInformationSync() TimeSeriesInformation info = ts.Info(key); TimeSeriesInformation infoDebug = ts.Info(key, debug: true); - Assert.Equal(4184, info.MemoryUsage); Assert.Equal(0, info.RetentionTime); Assert.Equal(1, info.ChunkCount); Assert.Null(info.DuplicatePolicy); Assert.Null(info.KeySelfName); Assert.Null(info.Chunks); - Assert.Equal(4184, infoDebug.MemoryUsage); Assert.Equal(0, infoDebug.RetentionTime); Assert.Equal(1, infoDebug.ChunkCount); Assert.Null(infoDebug.DuplicatePolicy); @@ -55,14 +53,12 @@ public async Task TestInformationAsync() TimeSeriesInformation info = await ts.InfoAsync(key); TimeSeriesInformation infoDebug = await ts.InfoAsync(key, debug: true); - Assert.Equal(4184, info.MemoryUsage); Assert.Equal(0, info.RetentionTime); Assert.Equal(1, info.ChunkCount); Assert.Null(info.DuplicatePolicy); Assert.Null(info.KeySelfName); Assert.Null(info.Chunks); - Assert.Equal(4184, infoDebug.MemoryUsage); Assert.Equal(0, infoDebug.RetentionTime); Assert.Equal(1, infoDebug.ChunkCount); Assert.Null(infoDebug.DuplicatePolicy); From 4a9b95f0bd54a870b81088df84e6a69158ab392a Mon Sep 17 00:00:00 2001 From: atakavci Date: Thu, 3 Apr 2025 18:40:57 +0300 Subject: [PATCH 03/25] fix failing test --- tests/NRedisStack.Tests/Search/SearchTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 96e2c97f..68a973c2 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -1384,7 +1384,7 @@ public void TestDropIndex(string endpointId) } catch (RedisServerException ex) { - Assert.Contains("no such index", ex.Message); + Assert.Contains("no such index", ex.Message, StringComparison.OrdinalIgnoreCase); } Assert.Equal("100", db.Execute("DBSIZE").ToString()); } From dbd0db03a8896281881dbd95f07fa96de01e899e Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 17:12:43 +0300 Subject: [PATCH 04/25] fix flaky test --- tests/NRedisStack.Tests/Search/SearchTests.cs | 63 +++++++++++++++---- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 68a973c2..86d31bc2 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -8,6 +8,8 @@ using System.Runtime.InteropServices; using NetTopologySuite.IO; using NetTopologySuite.Geometries; +using System.Linq.Expressions; +using System.Collections; namespace NRedisStack.Tests.Search; @@ -3416,6 +3418,13 @@ public void TestNumericLogicalOperatorsInDialect4(string endpointId) Assert.Equal(1, ft.Search(index, new Query("@version==123 @id==456").Dialect(4)).TotalResults); } + /// + /// this test is to check if the issue 352 is fixed + /// Load operation was failing because the document was not being dropped in search result due to this behaviour; + /// "If a relevant key expires while a query is running, an attempt to load the key's value will return a null array. + /// However, the key is still counted in the total number of results." + /// https://redis.io/docs/latest/commands/ft.search/#:~:text=If%20a%20relevant%20key%20expires,the%20total%20number%20of%20results. + /// [Fact] public void TestDocumentLoad_Issue352() { @@ -3423,6 +3432,13 @@ public void TestDocumentLoad_Issue352() Assert.Empty(d.GetProperties().ToList()); } + /// + /// this test is to check if the issue 352 is fixed + /// Load operation was failing because the document was not being dropped in search result due to this behaviour; + /// "If a relevant key expires while a query is running, an attempt to load the key's value will return a null array. + /// However, the key is still counted in the total number of results." + /// https://redis.io/docs/latest/commands/ft.search/#:~:text=If%20a%20relevant%20key%20expires,the%20total%20number%20of%20results. + /// [SkippableTheory] [MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))] public void TestDocumentLoadWithDB_Issue352(string endpointId) @@ -3435,33 +3451,56 @@ public void TestDocumentLoadWithDB_Issue352(string endpointId) Document droppedDocument = null; int numberOfAttempts = 0; + do { - db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); - - Assert.True(db.KeyExpire("student:1111", TimeSpan.FromMilliseconds(500))); + // try until succesfully create the key and set the TTL + bool ttlRefreshed = false; + do + { + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); + ttlRefreshed = db.KeyExpire("student:1111", TimeSpan.FromMilliseconds(500)); + } while (!ttlRefreshed); Boolean cancelled = false; - Task searchTask = Task.Run(() => + Action checker = () => { - for (int i = 0; i < 100000; i++) + for (int i = 0; i < 100000 && !cancelled; i++) { SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; - if (docs.Count == 0 || cancelled) + // check if doc is already dropped before search and load; + // if yes then its already late and we missed the window that + // doc would show up in search result with no fields + if (docs.Count == 0) { break; } - else if (docs[0].GetProperties().ToList().Count == 0) + // if we get a document with no fields then we know that the key + // expired while the query is running, and we are able to catch the state + // so we can break the loop + else if (docs[0].GetProperties().Count() == 0) { droppedDocument = docs[0]; + break; } } - }); - Task.WhenAny(searchTask, Task.Delay(1000)).GetAwaiter().GetResult(); - Assert.True(searchTask.IsCompleted); - Assert.Null(searchTask.Exception); + }; + + List tasks = new List(); + // try with 3 different tasks simultaneously to increase the chance of hitting it + for (int i = 0; i < 3; i++) + { + tasks.Add(Task.Run(checker)); + } + Task checkTask = Task.WhenAll(tasks); + Task.WhenAny(checkTask, Task.Delay(1500)).GetAwaiter().GetResult(); + Assert.True(checkTask.IsCompleted); + Assert.Null(checkTask.Exception); cancelled = true; - } while (droppedDocument == null && numberOfAttempts++ < 3); + } while (droppedDocument == null && numberOfAttempts++ < 5); + // we wont do an actual assert here since + // it is not guaranteed that window stays open wide enough to catch it. + // instead we attempt 5 times } } From 98a635ce35ef235ed93ace2ede9ac8ef40ad6728 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 18:01:09 +0300 Subject: [PATCH 05/25] attempt with await --- tests/NRedisStack.Tests/Search/SearchTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 86d31bc2..bd9bffca 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3441,7 +3441,7 @@ public void TestDocumentLoad_Issue352() /// [SkippableTheory] [MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))] - public void TestDocumentLoadWithDB_Issue352(string endpointId) + public async void TestDocumentLoadWithDB_Issue352(string endpointId) { IDatabase db = GetCleanDatabase(endpointId); var ft = db.FT(); @@ -3494,7 +3494,7 @@ public void TestDocumentLoadWithDB_Issue352(string endpointId) tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); - Task.WhenAny(checkTask, Task.Delay(1500)).GetAwaiter().GetResult(); + await Task.WhenAny(checkTask, Task.Delay(1500)); Assert.True(checkTask.IsCompleted); Assert.Null(checkTask.Exception); cancelled = true; From 0186b02cd8985a97181b5f15e1913debf27bb6cb Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 18:18:54 +0300 Subject: [PATCH 06/25] check with completed count --- tests/NRedisStack.Tests/Search/SearchTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index bd9bffca..94cb1ebd 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -8,9 +8,6 @@ using System.Runtime.InteropServices; using NetTopologySuite.IO; using NetTopologySuite.Geometries; -using System.Linq.Expressions; -using System.Collections; - namespace NRedisStack.Tests.Search; @@ -3456,6 +3453,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) { // try until succesfully create the key and set the TTL bool ttlRefreshed = false; + Int32 completed = 0; + do { db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); @@ -3474,6 +3473,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) // doc would show up in search result with no fields if (docs.Count == 0) { + Interlocked.Increment(ref completed); break; } // if we get a document with no fields then we know that the key @@ -3482,6 +3482,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) else if (docs[0].GetProperties().Count() == 0) { droppedDocument = docs[0]; + Interlocked.Increment(ref completed); break; } } @@ -3495,8 +3496,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1500)); - Assert.True(checkTask.IsCompleted); - Assert.Null(checkTask.Exception); + Assert.Equal(3, completed); cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since From 6da3b6974425a22488814dc42b0f881e6cdcd370 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 19:28:38 +0300 Subject: [PATCH 07/25] add started count --- tests/NRedisStack.Tests/Search/SearchTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 94cb1ebd..798c6aea 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3454,6 +3454,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) // try until succesfully create the key and set the TTL bool ttlRefreshed = false; Int32 completed = 0; + Int32 started = 0; do { @@ -3464,7 +3465,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Boolean cancelled = false; Action checker = () => { - for (int i = 0; i < 100000 && !cancelled; i++) + Interlocked.Increment(ref started); + for (int i = 0; i < 1000000 && !cancelled; i++) { SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; @@ -3496,6 +3498,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1500)); + Assert.Equal(3, started); Assert.Equal(3, completed); cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); From 2b8f6984482bac0b1b2787bf13d29e63766c597c Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:10:15 +0300 Subject: [PATCH 08/25] checking serverside issue --- tests/NRedisStack.Tests/Search/SearchTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 798c6aea..8f1bc76c 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3463,13 +3463,20 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } while (!ttlRefreshed); Boolean cancelled = false; + Int32 serverSideDiscrepency = 0; Action checker = () => { Interlocked.Increment(ref started); for (int i = 0; i < 1000000 && !cancelled; i++) { + bool exists = db.KeyExists("student:1111"); SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; + if (!exists && docs.Count != 0) + { + Interlocked.Increment(ref serverSideDiscrepency); + } + // check if doc is already dropped before search and load; // if yes then its already late and we missed the window that // doc would show up in search result with no fields @@ -3498,7 +3505,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1500)); - Assert.Equal(3, started); + Assert.Equal(0, serverSideDiscrepency); + Assert.Equal(3 + 2, started + 2); Assert.Equal(3, completed); cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); From b09d7dabbc8b9873c6b264a1ebd8fb635f7c3531 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:19:37 +0300 Subject: [PATCH 09/25] check id --- tests/NRedisStack.Tests/Search/SearchTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 8f1bc76c..719e93ac 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3464,6 +3464,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Boolean cancelled = false; Int32 serverSideDiscrepency = 0; + string id = null; Action checker = () => { Interlocked.Increment(ref started); @@ -3494,6 +3495,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Interlocked.Increment(ref completed); break; } + id = docs[0].Id == "student:1111" ? null : docs[0].Id; } }; @@ -3505,6 +3507,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1500)); + Assert.Equal(null, id); Assert.Equal(0, serverSideDiscrepency); Assert.Equal(3 + 2, started + 2); Assert.Equal(3, completed); From 6fe89a53b5c9cc7ae0d5cdae7848e115aae61076 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:34:00 +0300 Subject: [PATCH 10/25] fchange index content --- tests/NRedisStack.Tests/Search/SearchTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 719e93ac..e326e6be 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3443,7 +3443,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) IDatabase db = GetCleanDatabase(endpointId); var ft = db.FT(); - Schema sc = new Schema().AddTextField("first", 1.0).AddTextField("last", 1.0).AddNumericField("age"); + Schema sc = new Schema().AddTextField("firsty", 1.0).AddTextField("lasty", 1.0).AddNumericField("agey"); Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); Document droppedDocument = null; @@ -3458,8 +3458,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) do { - db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); - ttlRefreshed = db.KeyExpire("student:1111", TimeSpan.FromMilliseconds(500)); + db.HashSet("student:11112", new HashEntry[] { new("firsty", "Joe"), new("lasty", "Dod"), new("agey", 18) }); + ttlRefreshed = db.KeyExpire("student:11112", TimeSpan.FromMilliseconds(500)); } while (!ttlRefreshed); Boolean cancelled = false; @@ -3470,7 +3470,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Interlocked.Increment(ref started); for (int i = 0; i < 1000000 && !cancelled; i++) { - bool exists = db.KeyExists("student:1111"); + bool exists = db.KeyExists("student:11112"); SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; if (!exists && docs.Count != 0) @@ -3495,7 +3495,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Interlocked.Increment(ref completed); break; } - id = docs[0].Id == "student:1111" ? null : docs[0].Id; + id = docs[0].Id == "student:11112" ? null : docs[0].Id; } }; From 9c94ad01a640e1dc9fe555131da9f2923a23cabd Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:40:34 +0300 Subject: [PATCH 11/25] exceptions --- tests/NRedisStack.Tests/Search/SearchTests.cs | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index e326e6be..7bdc5001 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3455,6 +3455,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) bool ttlRefreshed = false; Int32 completed = 0; Int32 started = 0; + Int32 exception = 0; do { @@ -3468,34 +3469,41 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Action checker = () => { Interlocked.Increment(ref started); - for (int i = 0; i < 1000000 && !cancelled; i++) + try { - bool exists = db.KeyExists("student:11112"); - SearchResult result = ft.Search(index, new Query()); - List docs = result.Documents; - if (!exists && docs.Count != 0) + for (int i = 0; i < 1000000 && !cancelled; i++) { - Interlocked.Increment(ref serverSideDiscrepency); + bool exists = db.KeyExists("student:11112"); + SearchResult result = ft.Search(index, new Query()); + List docs = result.Documents; + if (!exists && docs.Count != 0) + { + Interlocked.Increment(ref serverSideDiscrepency); + } + + // check if doc is already dropped before search and load; + // if yes then its already late and we missed the window that + // doc would show up in search result with no fields + if (docs.Count == 0) + { + Interlocked.Increment(ref completed); + break; + } + // if we get a document with no fields then we know that the key + // expired while the query is running, and we are able to catch the state + // so we can break the loop + else if (docs[0].GetProperties().Count() == 0) + { + droppedDocument = docs[0]; + Interlocked.Increment(ref completed); + break; + } + id = docs[0].Id == "student:11112" ? null : docs[0].Id; } - - // check if doc is already dropped before search and load; - // if yes then its already late and we missed the window that - // doc would show up in search result with no fields - if (docs.Count == 0) - { - Interlocked.Increment(ref completed); - break; - } - // if we get a document with no fields then we know that the key - // expired while the query is running, and we are able to catch the state - // so we can break the loop - else if (docs[0].GetProperties().Count() == 0) - { - droppedDocument = docs[0]; - Interlocked.Increment(ref completed); - break; - } - id = docs[0].Id == "student:11112" ? null : docs[0].Id; + } + catch (Exception e) + { + Interlocked.Increment(ref exception); } }; @@ -3508,6 +3516,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1500)); Assert.Equal(null, id); + Assert.Equal(0, exception); Assert.Equal(0, serverSideDiscrepency); Assert.Equal(3 + 2, started + 2); Assert.Equal(3, completed); From 3baf8787d29396ae9a169480cc61ffc153a82795 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:48:20 +0300 Subject: [PATCH 12/25] check ttl --- tests/NRedisStack.Tests/Search/SearchTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 7bdc5001..1547313c 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3519,6 +3519,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Assert.Equal(0, exception); Assert.Equal(0, serverSideDiscrepency); Assert.Equal(3 + 2, started + 2); + Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); From 2bab62bfa272b0cddee21778231975d425a42b1f Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 22:59:40 +0300 Subject: [PATCH 13/25] limit window --- tests/NRedisStack.Tests/Search/SearchTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 1547313c..9f40d16b 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3514,11 +3514,11 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); - await Task.WhenAny(checkTask, Task.Delay(1500)); + await Task.WhenAny(checkTask, Task.Delay(1000)); Assert.Equal(null, id); Assert.Equal(0, exception); Assert.Equal(0, serverSideDiscrepency); - Assert.Equal(3 + 2, started + 2); + Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); cancelled = true; From 5376f5170b0f2910113f93eff9517cf2be0a02ff Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:08:37 +0300 Subject: [PATCH 14/25] validate server is not expiring --- tests/NRedisStack.Tests/Search/SearchTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 9f40d16b..478f9065 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3487,7 +3487,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) if (docs.Count == 0) { Interlocked.Increment(ref completed); - break; + // break; } // if we get a document with no fields then we know that the key // expired while the query is running, and we are able to catch the state @@ -3496,9 +3496,12 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) { droppedDocument = docs[0]; Interlocked.Increment(ref completed); - break; + // break; + } + else + { + id = docs[0].Id == "student:11112" ? null : docs[0].Id; } - id = docs[0].Id == "student:11112" ? null : docs[0].Id; } } catch (Exception e) @@ -3521,7 +3524,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); - cancelled = true; + // cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. From 11ef4ab96bab8f736153c94a1bf973f0b4afacbf Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:22:36 +0300 Subject: [PATCH 15/25] back to complete counts --- tests/NRedisStack.Tests/Search/SearchTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 478f9065..7130780f 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3487,7 +3487,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) if (docs.Count == 0) { Interlocked.Increment(ref completed); - // break; + break; } // if we get a document with no fields then we know that the key // expired while the query is running, and we are able to catch the state @@ -3496,7 +3496,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) { droppedDocument = docs[0]; Interlocked.Increment(ref completed); - // break; + break; } else { From ffb2aa1c8eb59c62e4ec33ae1910b5f2ba61e1b4 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:29:18 +0300 Subject: [PATCH 16/25] back to failing version --- tests/NRedisStack.Tests/Search/SearchTests.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 7130780f..571fb15e 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3471,15 +3471,15 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Interlocked.Increment(ref started); try { - for (int i = 0; i < 1000000 && !cancelled; i++) + for (int i = 0; i < 100000 && !cancelled; i++) { - bool exists = db.KeyExists("student:11112"); + // bool exists = db.KeyExists("student:11112"); SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; - if (!exists && docs.Count != 0) - { - Interlocked.Increment(ref serverSideDiscrepency); - } + // if (!exists && docs.Count != 0) + // { + // Interlocked.Increment(ref serverSideDiscrepency); + // } // check if doc is already dropped before search and load; // if yes then its already late and we missed the window that @@ -3500,7 +3500,10 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } else { - id = docs[0].Id == "student:11112" ? null : docs[0].Id; + if (docs[0].Id != "student:11112") + { + id = docs[0].Id; + } } } } @@ -3517,7 +3520,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); - await Task.WhenAny(checkTask, Task.Delay(1000)); + await Task.WhenAny(checkTask, Task.Delay(1500)); Assert.Equal(null, id); Assert.Equal(0, exception); Assert.Equal(0, serverSideDiscrepency); From 98319915f3369cccd5e2b5f755bd599094f433a5 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:35:11 +0300 Subject: [PATCH 17/25] play with time --- tests/NRedisStack.Tests/Search/SearchTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 571fb15e..c769d749 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3520,14 +3520,14 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); - await Task.WhenAny(checkTask, Task.Delay(1500)); + await Task.WhenAny(checkTask, Task.Delay(1000)); Assert.Equal(null, id); Assert.Equal(0, exception); Assert.Equal(0, serverSideDiscrepency); Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); - // cancelled = true; + cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. From c7dce115a7eb9bf9b5df2fc623c2ce88c65bcf81 Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:49:37 +0300 Subject: [PATCH 18/25] only success when result is empty --- tests/NRedisStack.Tests/Search/SearchTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index c769d749..fde3207c 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3495,8 +3495,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) else if (docs[0].GetProperties().Count() == 0) { droppedDocument = docs[0]; - Interlocked.Increment(ref completed); - break; + // Interlocked.Increment(ref completed); + // break; } else { @@ -3527,7 +3527,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); - cancelled = true; + // cancelled = true; + } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. From 875b110ac9734aa4c6f4debe81d4e68ee7b331fb Mon Sep 17 00:00:00 2001 From: atakavci Date: Tue, 8 Apr 2025 23:52:22 +0300 Subject: [PATCH 19/25] use cancelled --- tests/NRedisStack.Tests/Search/SearchTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index fde3207c..35fb5442 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3527,7 +3527,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); - // cancelled = true; + cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since From c78c0492931d2fb9651a75d57cd97b66f146d1d4 Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 00:02:26 +0300 Subject: [PATCH 20/25] try find fail --- tests/NRedisStack.Tests/Search/SearchTests.cs | 69 +++++-------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 35fb5442..f3d27b3c 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3448,15 +3448,11 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Document droppedDocument = null; int numberOfAttempts = 0; - do { // try until succesfully create the key and set the TTL bool ttlRefreshed = false; Int32 completed = 0; - Int32 started = 0; - Int32 exception = 0; - do { db.HashSet("student:11112", new HashEntry[] { new("firsty", "Joe"), new("lasty", "Dod"), new("agey", 18) }); @@ -3464,52 +3460,30 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } while (!ttlRefreshed); Boolean cancelled = false; - Int32 serverSideDiscrepency = 0; - string id = null; Action checker = () => { - Interlocked.Increment(ref started); - try + for (int i = 0; i < 1000000 && !cancelled; i++) { - for (int i = 0; i < 100000 && !cancelled; i++) + SearchResult result = ft.Search(index, new Query()); + List docs = result.Documents; + + // check if doc is already dropped before search and load; + // if yes then its already late and we missed the window that + // doc would show up in search result with no fields + if (docs.Count == 0) { - // bool exists = db.KeyExists("student:11112"); - SearchResult result = ft.Search(index, new Query()); - List docs = result.Documents; - // if (!exists && docs.Count != 0) - // { - // Interlocked.Increment(ref serverSideDiscrepency); - // } - - // check if doc is already dropped before search and load; - // if yes then its already late and we missed the window that - // doc would show up in search result with no fields - if (docs.Count == 0) - { - Interlocked.Increment(ref completed); - break; - } - // if we get a document with no fields then we know that the key - // expired while the query is running, and we are able to catch the state - // so we can break the loop - else if (docs[0].GetProperties().Count() == 0) - { - droppedDocument = docs[0]; - // Interlocked.Increment(ref completed); - // break; - } - else - { - if (docs[0].Id != "student:11112") - { - id = docs[0].Id; - } - } + Interlocked.Increment(ref completed); + break; + } + // if we get a document with no fields then we know that the key + // expired while the query is running, and we are able to catch the state + // so we can break the loop + else if (docs[0].GetProperties().Count() == 0) + { + droppedDocument = docs[0]; + Interlocked.Increment(ref completed); + break; } - } - catch (Exception e) - { - Interlocked.Increment(ref exception); } }; @@ -3521,14 +3495,9 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1000)); - Assert.Equal(null, id); - Assert.Equal(0, exception); - Assert.Equal(0, serverSideDiscrepency); - Assert.Equal(3, started); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); cancelled = true; - } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. From a2129a48df39d78b130bc55ea33fcec3776a712e Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 00:55:06 +0300 Subject: [PATCH 21/25] remove cancelled --- tests/NRedisStack.Tests/Search/SearchTests.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index f3d27b3c..d611ad1f 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3452,17 +3452,16 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) { // try until succesfully create the key and set the TTL bool ttlRefreshed = false; - Int32 completed = 0; do { db.HashSet("student:11112", new HashEntry[] { new("firsty", "Joe"), new("lasty", "Dod"), new("agey", 18) }); ttlRefreshed = db.KeyExpire("student:11112", TimeSpan.FromMilliseconds(500)); } while (!ttlRefreshed); - Boolean cancelled = false; + Int32 completed = 0; Action checker = () => { - for (int i = 0; i < 1000000 && !cancelled; i++) + for (int i = 0; i < 1000000; i++) { SearchResult result = ft.Search(index, new Query()); List docs = result.Documents; @@ -3489,15 +3488,11 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) List tasks = new List(); // try with 3 different tasks simultaneously to increase the chance of hitting it - for (int i = 0; i < 3; i++) - { - tasks.Add(Task.Run(checker)); - } + for (int i = 0; i < 3; i++) { tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1000)); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); - cancelled = true; } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. From 07c790838fb794102f156e06f28e21d38e101a54 Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 01:17:11 +0300 Subject: [PATCH 22/25] clean up --- tests/NRedisStack.Tests/Search/SearchTests.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index d611ad1f..bd154214 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using NetTopologySuite.IO; using NetTopologySuite.Geometries; +using System.Diagnostics; namespace NRedisStack.Tests.Search; @@ -3459,6 +3460,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) } while (!ttlRefreshed); Int32 completed = 0; + Stopwatch sw = new Stopwatch(); + sw.Start(); Action checker = () => { for (int i = 0; i < 1000000; i++) @@ -3475,13 +3478,13 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) break; } // if we get a document with no fields then we know that the key - // expired while the query is running, and we are able to catch the state - // so we can break the loop + // is going to be expired while the query is running, and we are able to catch the state + // but key itself might not be expired yet else if (docs[0].GetProperties().Count() == 0) { droppedDocument = docs[0]; - Interlocked.Increment(ref completed); - break; + // Interlocked.Increment(ref completed); + // break; } } }; @@ -3491,6 +3494,7 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) for (int i = 0; i < 3; i++) { tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1000)); + Assert.True(sw.ElapsedMilliseconds > 500); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); } while (droppedDocument == null && numberOfAttempts++ < 5); From 1b00717af86e200bb7583bb52327f17400988451 Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 01:24:28 +0300 Subject: [PATCH 23/25] more clenaup --- tests/NRedisStack.Tests/Search/SearchTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index bd154214..d7ce99cc 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3483,8 +3483,6 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) else if (docs[0].GetProperties().Count() == 0) { droppedDocument = docs[0]; - // Interlocked.Increment(ref completed); - // break; } } }; @@ -3494,7 +3492,6 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) for (int i = 0; i < 3; i++) { tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1000)); - Assert.True(sw.ElapsedMilliseconds > 500); Assert.Null(db.KeyTimeToLive("student:11112")); Assert.Equal(3, completed); } while (droppedDocument == null && numberOfAttempts++ < 5); From 65bb8b01ec9a1ebc44bfdf0f9676bdc61292e7f3 Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 01:31:38 +0300 Subject: [PATCH 24/25] replace ttl --- tests/NRedisStack.Tests/Search/SearchTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index d7ce99cc..88ba17da 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3492,7 +3492,8 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) for (int i = 0; i < 3; i++) { tasks.Add(Task.Run(checker)); } Task checkTask = Task.WhenAll(tasks); await Task.WhenAny(checkTask, Task.Delay(1000)); - Assert.Null(db.KeyTimeToLive("student:11112")); + var keyTtl = db.KeyTimeToLive("student:11112"); + Assert.Equal(0, keyTtl.HasValue ? keyTtl.Value.Milliseconds : 0); Assert.Equal(3, completed); } while (droppedDocument == null && numberOfAttempts++ < 5); // we wont do an actual assert here since From 7f1eb7d0cc42901887e2d4476ea7e370af17451a Mon Sep 17 00:00:00 2001 From: atakavci Date: Wed, 9 Apr 2025 01:42:21 +0300 Subject: [PATCH 25/25] add more comment --- tests/NRedisStack.Tests/Search/SearchTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 88ba17da..de24e572 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -3496,8 +3496,10 @@ public async void TestDocumentLoadWithDB_Issue352(string endpointId) Assert.Equal(0, keyTtl.HasValue ? keyTtl.Value.Milliseconds : 0); Assert.Equal(3, completed); } while (droppedDocument == null && numberOfAttempts++ < 5); - // we wont do an actual assert here since + // we won't do an actual assert here since // it is not guaranteed that window stays open wide enough to catch it. - // instead we attempt 5 times + // instead we attempt 5 times. + // Without fix for Issue352 , document load fails %100 with my local test runs,, and %100 success with fixed version. + // The results in pipeline might become complete different story.. } }