Added Redis 8.0 to test matrix (#3469)

* Added Redis 8.0 to test matrix

* Fixed test cases

* Added version annotation

* Changed FT.PROFILE response type

* Added version restrictions

* Updated file names, fixed tests assertions

* Removed unused API

Author: vladvildanov

.github/workflows/integration.yaml

@@ -74,7 +74,7 @@ jobs:
       max-parallel: 15
       fail-fast: false
       matrix:
-        redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}', '7.2.6', '6.2.16']
+        redis-version: ['8.0-M02', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.6', '6.2.16']
         python-version: ['3.8', '3.12']
         parser-backend: ['plain']
         event-loop: ['asyncio']

doctests/query_agg.py

@@ -6,7 +6,7 @@
 from redis.commands.search import Search
 from redis.commands.search.aggregation import AggregateRequest
 from redis.commands.search.field import NumericField, TagField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 import redis.commands.search.reducers as reducers
 
 r = redis.Redis(decode_responses=True)

doctests/query_combined.py

@@ -6,7 +6,7 @@
 import warnings
 from redis.commands.json.path import Path
 from redis.commands.search.field import NumericField, TagField, TextField, VectorField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
 from sentence_transformers import  SentenceTransformer
 

doctests/query_em.py

@@ -4,7 +4,7 @@
 import redis
 from redis.commands.json.path import Path
 from redis.commands.search.field import TextField, NumericField, TagField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import NumericFilter, Query
 
 r = redis.Redis(decode_responses=True)

doctests/query_ft.py

@@ -5,7 +5,7 @@
 import redis
 from redis.commands.json.path import Path
 from redis.commands.search.field import TextField, NumericField, TagField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import NumericFilter, Query
 
 r = redis.Redis(decode_responses=True)

doctests/query_geo.py

@@ -5,7 +5,7 @@
 import redis
 from redis.commands.json.path import Path
 from redis.commands.search.field import GeoField, GeoShapeField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
 
 r = redis.Redis(decode_responses=True)

doctests/query_range.py

@@ -5,7 +5,7 @@
 import redis
 from redis.commands.json.path import Path
 from redis.commands.search.field import TextField, NumericField, TagField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import NumericFilter, Query
 
 r = redis.Redis(decode_responses=True)

doctests/search_quickstart.py

@@ -10,7 +10,7 @@
 import redis.commands.search.reducers as reducers
 from redis.commands.json.path import Path
 from redis.commands.search.field import NumericField, TagField, TextField
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
 
 # HIDE_END

doctests/search_vss.py

@@ -20,7 +20,7 @@
     TextField,
     VectorField,
 )
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
 from sentence_transformers import SentenceTransformer
 

redis/commands/helpers.py

@@ -79,29 +79,6 @@ def parse_list_to_dict(response):
     return res
 
 
-def parse_to_dict(response):
-    if response is None:
-        return {}
-
-    res = {}
-    for det in response:
-        if not isinstance(det, list) or not det:
-            continue
-        if len(det) == 1:
-            res[det[0]] = True
-        elif isinstance(det[1], list):
-            res[det[0]] = parse_list_to_dict(det[1])
-        else:
-            try:  # try to set the attribute. may be provided without value
-                try:  # try to convert the value to float
-                    res[det[0]] = float(det[1])
-                except (TypeError, ValueError):
-                    res[det[0]] = det[1]
-            except IndexError:
-                pass
-    return res
-
-
 def random_string(length=10):
     """
     Returns a random N character long string.

redis/commands/search/commands.py

@@ -5,12 +5,13 @@
 from redis.client import NEVER_DECODE, Pipeline
 from redis.utils import deprecated_function
 
-from ..helpers import get_protocol_version, parse_to_dict
+from ..helpers import get_protocol_version
 from ._util import to_string
 from .aggregation import AggregateRequest, AggregateResult, Cursor
 from .document import Document
 from .field import Field
-from .indexDefinition import IndexDefinition
+from .index_definition import IndexDefinition
+from .profile_information import ProfileInformation
 from .query import Query
 from .result import Result
 from .suggestion import SuggestionParser
@@ -67,7 +68,7 @@ class SearchCommands:
 
     def _parse_results(self, cmd, res, **kwargs):
         if get_protocol_version(self.client) in ["3", 3]:
-            return res
+            return ProfileInformation(res) if cmd == "FT.PROFILE" else res
         else:
             return self._RESP2_MODULE_CALLBACKS[cmd](res, **kwargs)
 
@@ -101,7 +102,7 @@ def _parse_profile(self, res, **kwargs):
                 with_scores=query._with_scores,
             )
 
-        return result, parse_to_dict(res[1])
+        return result, ProfileInformation(res[1])
 
     def _parse_spellcheck(self, res, **kwargs):
         corrections = {}

redis/commands/search/indexDefinition.py renamed to redis/commands/search/index_definition.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+
+class ProfileInformation:
+    """
+    Wrapper around FT.PROFILE response
+    """
+
+    def __init__(self, info: Any) -> None:
+        self._info: Any = info
+
+    @property
+    def info(self) -> Any:
+        return self._info

redis/commands/search/profile_information.py

@@ -19,14 +19,16 @@
     TextField,
     VectorField,
 )
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import GeoFilter, NumericFilter, Query
 from redis.commands.search.result import Result
 from redis.commands.search.suggestion import Suggestion
 from tests.conftest import (
     is_resp2_connection,
     skip_if_redis_enterprise,
     skip_if_resp_version,
+    skip_if_server_version_gte,
+    skip_if_server_version_lt,
     skip_ifmodversion_lt,
 )
 
@@ -1111,6 +1113,7 @@ async def test_get(decoded_r: redis.Redis):
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 @skip_ifmodversion_lt("2.2.0", "search")
+@skip_if_server_version_gte("7.9.0")
 async def test_config(decoded_r: redis.Redis):
     assert await decoded_r.ft().config_set("TIMEOUT", "100")
     with pytest.raises(redis.ResponseError):
@@ -1121,6 +1124,19 @@ async def test_config(decoded_r: redis.Redis):
     assert "100" == res["TIMEOUT"]
 
 
+@pytest.mark.redismod
+@pytest.mark.onlynoncluster
+@skip_if_server_version_lt("7.9.0")
+async def test_config_with_removed_ftconfig(decoded_r: redis.Redis):
+    assert await decoded_r.config_set("timeout", "100")
+    with pytest.raises(redis.ResponseError):
+        await decoded_r.config_set("timeout", "null")
+    res = await decoded_r.config_get("*")
+    assert "100" == res["timeout"]
+    res = await decoded_r.config_get("timeout")
+    assert "100" == res["timeout"]
+
+
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 async def test_aggregations_groupby(decoded_r: redis.Redis):

tests/test_asyncio/test_search.py

@@ -1823,6 +1823,7 @@ def try_delete_libs(self, r, *lib_names):
 
     @pytest.mark.onlynoncluster
     @skip_if_server_version_lt("7.1.140")
+    @skip_if_server_version_gte("7.9.0")
     def test_tfunction_load_delete(self, stack_r):
         self.try_delete_libs(stack_r, "lib1")
         lib_code = self.generate_lib_code("lib1")
@@ -1831,6 +1832,7 @@ def test_tfunction_load_delete(self, stack_r):
 
     @pytest.mark.onlynoncluster
     @skip_if_server_version_lt("7.1.140")
+    @skip_if_server_version_gte("7.9.0")
     def test_tfunction_list(self, stack_r):
         self.try_delete_libs(stack_r, "lib1", "lib2", "lib3")
         assert stack_r.tfunction_load(self.generate_lib_code("lib1"))
@@ -1861,6 +1863,7 @@ def test_tfunction_list(self, stack_r):
 
     @pytest.mark.onlynoncluster
     @skip_if_server_version_lt("7.1.140")
+    @skip_if_server_version_gte("7.9.0")
     def test_tfcall(self, stack_r):
         self.try_delete_libs(stack_r, "lib1")
         assert stack_r.tfunction_load(self.generate_lib_code("lib1"))
@@ -4329,6 +4332,7 @@ def test_xgroup_create_mkstream(self, r):
         assert r.xinfo_groups(stream) == expected
 
     @skip_if_server_version_lt("7.0.0")
+    @skip_if_server_version_gte("7.9.0")
     def test_xgroup_create_entriesread(self, r: redis.Redis):
         stream = "stream"
         group = "group"
@@ -4350,6 +4354,28 @@ def test_xgroup_create_entriesread(self, r: redis.Redis):
         ]
         assert r.xinfo_groups(stream) == expected
 
+    @skip_if_server_version_lt("7.9.0")
+    def test_xgroup_create_entriesread_with_fixed_lag_field(self, r: redis.Redis):
+        stream = "stream"
+        group = "group"
+        r.xadd(stream, {"foo": "bar"})
+
+        # no group is setup yet, no info to obtain
+        assert r.xinfo_groups(stream) == []
+
+        assert r.xgroup_create(stream, group, 0, entries_read=7)
+        expected = [
+            {
+                "name": group.encode(),
+                "consumers": 0,
+                "pending": 0,
+                "last-delivered-id": b"0-0",
+                "entries-read": 7,
+                "lag": 1,
+            }
+        ]
+        assert r.xinfo_groups(stream) == expected
+
     @skip_if_server_version_lt("5.0.0")
     def test_xgroup_delconsumer(self, r):
         stream = "stream"

tests/test_commands.py

@@ -4,7 +4,6 @@
     delist,
     list_or_args,
     nativestr,
-    parse_to_dict,
     parse_to_list,
     quote_string,
     random_string,
@@ -26,40 +25,6 @@ def test_parse_to_list():
     assert parse_to_list(r) == ["hello", "my name", 45, 555.55, "is simon!", None]
 
 
-def test_parse_to_dict():
-    assert parse_to_dict(None) == {}
-    r = [
-        ["Some number", "1.0345"],
-        ["Some string", "hello"],
-        [
-            "Child iterators",
-            [
-                "Time",
-                "0.2089",
-                "Counter",
-                3,
-                "Child iterators",
-                ["Type", "bar", "Time", "0.0729", "Counter", 3],
-                ["Type", "barbar", "Time", "0.058", "Counter", 3],
-                ["Type", "barbarbar", "Time", "0.0234", "Counter", 3],
-            ],
-        ],
-    ]
-    assert parse_to_dict(r) == {
-        "Child iterators": {
-            "Child iterators": [
-                {"Counter": 3.0, "Time": 0.0729, "Type": "bar"},
-                {"Counter": 3.0, "Time": 0.058, "Type": "barbar"},
-                {"Counter": 3.0, "Time": 0.0234, "Type": "barbarbar"},
-            ],
-            "Counter": 3.0,
-            "Time": 0.2089,
-        },
-        "Some number": 1.0345,
-        "Some string": "hello",
-    }
-
-
 def test_nativestr():
     assert nativestr("teststr") == "teststr"
     assert nativestr(b"teststr") == "teststr"

tests/test_helpers.py

@@ -20,7 +20,7 @@
     TextField,
     VectorField,
 )
-from redis.commands.search.indexDefinition import IndexDefinition, IndexType
+from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import GeoFilter, NumericFilter, Query
 from redis.commands.search.result import Result
 from redis.commands.search.suggestion import Suggestion
@@ -30,6 +30,7 @@
     is_resp2_connection,
     skip_if_redis_enterprise,
     skip_if_resp_version,
+    skip_if_server_version_gte,
     skip_if_server_version_lt,
     skip_ifmodversion_lt,
 )
@@ -1007,6 +1008,7 @@ def test_get(client):
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 @skip_ifmodversion_lt("2.2.0", "search")
+@skip_if_server_version_gte("7.9.0")
 def test_config(client):
     assert client.ft().config_set("TIMEOUT", "100")
     with pytest.raises(redis.ResponseError):
@@ -1017,6 +1019,19 @@ def test_config(client):
     assert "100" == res["TIMEOUT"]
 
 
+@pytest.mark.redismod
+@pytest.mark.onlynoncluster
+@skip_if_server_version_lt("7.9.0")
+def test_config_with_removed_ftconfig(client):
+    assert client.config_set("timeout", "100")
+    with pytest.raises(redis.ResponseError):
+        client.config_set("timeout", "null")
+    res = client.config_get("*")
+    assert "100" == res["timeout"]
+    res = client.config_get("timeout")
+    assert "100" == res["timeout"]
+
+
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 def test_aggregations_groupby(client):
@@ -1571,6 +1586,7 @@ def test_index_definition(client):
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 @skip_if_redis_enterprise()
+@skip_if_server_version_gte("7.9.0")
 def test_expire(client):
     client.ft().create_index((TextField("txt", sortable=True),), temporary=4)
     ttl = client.execute_command("ft.debug", "TTL", "idx")
@@ -2025,6 +2041,8 @@ def test_json_with_jsonpath(client):
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
 @skip_if_redis_enterprise()
+@skip_if_server_version_gte("7.9.0")
+@skip_if_server_version_lt("6.3.0")
 def test_profile(client):
     client.ft().create_index((TextField("t"),))
     client.ft().client.hset("1", "t", "hello")
@@ -2034,10 +2052,9 @@ def test_profile(client):
     q = Query("hello|world").no_content()
     if is_resp2_connection(client):
         res, det = client.ft().profile(q)
-        assert det["Iterators profile"]["Counter"] == 2.0
-        assert len(det["Iterators profile"]["Child iterators"]) == 2
-        assert det["Iterators profile"]["Type"] == "UNION"
-        assert det["Parsing time"] < 0.5
+        det = det.info
+
+        assert isinstance(det, list)
         assert len(res.docs) == 2  # check also the search result
 
         # check using AggregateRequest
@@ -2047,15 +2064,14 @@ def test_profile(client):
             .apply(prefix="startswith(@t, 'hel')")
         )
         res, det = client.ft().profile(req)
-        assert det["Iterators profile"]["Counter"] == 2
-        assert det["Iterators profile"]["Type"] == "WILDCARD"
-        assert isinstance(det["Parsing time"], float)
+        det = det.info
+        assert isinstance(det, list)
         assert len(res.rows) == 2  # check also the search result
     else:
         res = client.ft().profile(q)
-        assert res["profile"]["Iterators profile"][0]["Counter"] == 2.0
-        assert res["profile"]["Iterators profile"][0]["Type"] == "UNION"
-        assert res["profile"]["Parsing time"] < 0.5
+        res = res.info
+
+        assert isinstance(res, dict)
         assert len(res["results"]) == 2  # check also the search result
 
         # check using AggregateRequest
@@ -2065,14 +2081,97 @@ def test_profile(client):
             .apply(prefix="startswith(@t, 'hel')")
         )
         res = client.ft().profile(req)
-        assert res["profile"]["Iterators profile"][0]["Counter"] == 2
-        assert res["profile"]["Iterators profile"][0]["Type"] == "WILDCARD"
-        assert isinstance(res["profile"]["Parsing time"], float)
+        res = res.info
+
+        assert isinstance(res, dict)
         assert len(res["results"]) == 2  # check also the search result
 
 
 @pytest.mark.redismod
 @pytest.mark.onlynoncluster
+@skip_if_redis_enterprise()
+@skip_if_server_version_lt("7.9.0")
+def test_profile_with_coordinator(client):
+    client.ft().create_index((TextField("t"),))
+    client.ft().client.hset("1", "t", "hello")
+    client.ft().client.hset("2", "t", "world")
+
+    # check using Query
+    q = Query("hello|world").no_content()
+    if is_resp2_connection(client):
+        res, det = client.ft().profile(q)
+        det = det.info
+
+        assert isinstance(det, list)
+        assert len(res.docs) == 2  # check also the search result
+
+        # check using AggregateRequest
+        req = (
+            aggregations.AggregateRequest("*")
+            .load("t")
+            .apply(prefix="startswith(@t, 'hel')")
+        )
+        res, det = client.ft().profile(req)
+        det = det.info
+
+        assert isinstance(det, list)
+        assert det[0] == "Shards"
+        assert det[2] == "Coordinator"
+        assert len(res.rows) == 2  # check also the search result
+    else:
+        res = client.ft().profile(q)
+        res = res.info
+
+        assert isinstance(res, dict)
+        assert len(res["Results"]["results"]) == 2  # check also the search result
+
+        # check using AggregateRequest
+        req = (
+            aggregations.AggregateRequest("*")
+            .load("t")
+            .apply(prefix="startswith(@t, 'hel')")
+        )
+        res = client.ft().profile(req)
+        res = res.info
+
+        assert isinstance(res, dict)
+        assert len(res["Results"]["results"]) == 2  # check also the search result
+
+
+@pytest.mark.redismod
+@pytest.mark.onlynoncluster
+@skip_if_redis_enterprise()
+@skip_if_server_version_gte("6.3.0")
+def test_profile_with_no_warnings(client):
+    client.ft().create_index((TextField("t"),))
+    client.ft().client.hset("1", "t", "hello")
+    client.ft().client.hset("2", "t", "world")
+
+    # check using Query
+    q = Query("hello|world").no_content()
+    res, det = client.ft().profile(q)
+    det = det.info
+
+    assert isinstance(det, list)
+    assert len(res.docs) == 2  # check also the search result
+
+    # check using AggregateRequest
+    req = (
+        aggregations.AggregateRequest("*")
+        .load("t")
+        .apply(prefix="startswith(@t, 'hel')")
+    )
+    res, det = client.ft().profile(req)
+    det = det.info
+
+    assert isinstance(det, list)
+    assert len(res.rows) == 2  # check also the search result
+
+
+@pytest.mark.redismod
+@pytest.mark.onlynoncluster
+@skip_if_server_version_gte("7.9.0")
+@skip_if_server_version_lt("6.3.0")
 def test_profile_limited(client):
     client.ft().create_index((TextField("t"),))
     client.ft().client.hset("1", "t", "hello")
@@ -2083,18 +2182,14 @@ def test_profile_limited(client):
     q = Query("%hell% hel*")
     if is_resp2_connection(client):
         res, det = client.ft().profile(q, limited=True)
-        assert (
-            det["Iterators profile"]["Child iterators"][0]["Child iterators"]
-            == "The number of iterators in the union is 3"
-        )
-        assert (
-            det["Iterators profile"]["Child iterators"][1]["Child iterators"]
-            == "The number of iterators in the union is 4"
-        )
-        assert det["Iterators profile"]["Type"] == "INTERSECT"
+        det = det.info
+        assert det[4][1][7][9] == "The number of iterators in the union is 3"
+        assert det[4][1][8][9] == "The number of iterators in the union is 4"
+        assert det[4][1][1] == "INTERSECT"
         assert len(res.docs) == 3  # check also the search result
     else:
         res = client.ft().profile(q, limited=True)
+        res = res.info
         iterators_profile = res["profile"]["Iterators profile"]
         assert (
             iterators_profile[0]["Child iterators"][0]["Child iterators"]
@@ -2110,6 +2205,8 @@ def test_profile_limited(client):
 
 @pytest.mark.redismod
 @skip_ifmodversion_lt("2.4.3", "search")
+@skip_if_server_version_gte("7.9.0")
+@skip_if_server_version_lt("6.3.0")
 def test_profile_query_params(client):
     client.ft().create_index(
         (
@@ -2125,13 +2222,15 @@ def test_profile_query_params(client):
     q = Query(query).return_field("__v_score").sort_by("__v_score", True)
     if is_resp2_connection(client):
         res, det = client.ft().profile(q, query_params={"vec": "aaaaaaaa"})
-        assert det["Iterators profile"]["Counter"] == 2.0
-        assert det["Iterators profile"]["Type"] == "VECTOR"
+        det = det.info
+        assert det[4][1][5] == 2.0
+        assert det[4][1][1] == "VECTOR"
         assert res.total == 2
         assert "a" == res.docs[0].id
         assert "0" == res.docs[0].__getattribute__("__v_score")
     else:
         res = client.ft().profile(q, query_params={"vec": "aaaaaaaa"})
+        res = res.info
         assert res["profile"]["Iterators profile"][0]["Counter"] == 2
         assert res["profile"]["Iterators profile"][0]["Type"] == "VECTOR"
         assert res["total_results"] == 2