feat(commands): handle multiple indices per Document

* Index creation is now done through the Document.init() to allow naming through a given suffix
* Index rebuild makes no sense anymore

Author: David-Guillot

django_opensearch_dsl/management/commands/opensearch.py

@@ -5,6 +5,7 @@
 import sys
 from argparse import ArgumentParser
 from collections import defaultdict
+from datetime import datetime
 from typing import Any, Callable
 
 import opensearchpy
@@ -17,6 +18,7 @@
 from django_opensearch_dsl.registries import registry
 from ..enums import OpensearchAction
 from ..types import parse
+from ...documents import Document
 
 
 class Command(BaseCommand):
@@ -54,19 +56,40 @@ def __list_index(self, **options):  # noqa pragma: no cover
         indices = registry.get_indices()
         result = defaultdict(list)
         for index in indices:
-            module = index._doc_types[0].__module__.split(".")[-2]  # noqa
+            document = index._doc_types[0]
+            module = document.__module__.split(".")[-2]  # noqa
             exists = index.exists()
             checkbox = f"[{'X' if exists else ' '}]"
-            count = f" ({index.search().count()} documents)" if exists else ""
-            result[module].append(f"{checkbox} {index._name}{count}")
+            document_indices = document.get_all_indices()
+            if document_indices:
+                details = ". Following indices exist:"
+                for document_index in document_indices:
+                    is_active = ""
+                    if document_index.exists_alias(name=index._name):
+                        is_active = self.style.SUCCESS("Active")
+                    count = f" ({document_index.search().count()} documents)"
+                    details += f"\n    - {document_index._name} {is_active}{count}"
+            elif exists:
+                details = f" ({index.search().count()} documents)"
+            else:
+                details = ""
+            result[module].append(f"{checkbox} {index._name}{details}")
         for app, indices in result.items():
             self.stdout.write(self.style.MIGRATE_LABEL(app))
             self.stdout.write("\n".join(indices))
 
-    def _manage_index(self, action, indices, force, verbosity, ignore_error, **options):  # noqa
+    def _manage_index(self, action, indices, suffix, force, verbosity, ignore_error, **options):  # noqa
         """Manage the creation and deletion of indices."""
         action = OpensearchAction(action)
-        known = registry.get_indices()
+
+        suffix = suffix or datetime.now().strftime("%Y%m%d%H%M%S%f")
+
+        if action == OpensearchAction.CREATE:
+            known = registry.get_indices()
+        else:
+            known = []
+            for document in [i._doc_types[0] for i in registry.get_indices()]:
+                known.extend(document.get_all_indices())
 
         # Filter indices
         if indices:
@@ -84,9 +107,15 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio
 
         # Display expected action
         if verbosity or not force:
+            if not indices:
+                self.stdout.write("Nothing to do, exiting")
+                exit(0)
             self.stdout.write(f"The following indices will be {action.past}:")
             for index in indices:
-                self.stdout.write(f"\t- {index._name}.")  # noqa
+                index_name = index._name  # noqa
+                if action == OpensearchAction.CREATE:
+                    index_name += f"{Document.VERSION_NAME_SEPARATOR}{suffix}"
+                self.stdout.write(f"\t- {index_name}.")  # noqa
             self.stdout.write("")
 
         # Ask for confirmation to continue
@@ -101,23 +130,21 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio
 
         pp = action.present_participle.title()
         for index in indices:
+            index_name = index._name  # noqa
+            if action == OpensearchAction.CREATE:
+                index_name += f"{Document.VERSION_NAME_SEPARATOR}{suffix}"
+
             if verbosity:
                 self.stdout.write(
-                    f"{pp} index '{index._name}'...\r",
+                    f"{pp} index '{index_name}'...\r",
                     ending="",
                 )  # noqa
                 self.stdout.flush()
             try:
                 if action == OpensearchAction.CREATE:
-                    index.create()
+                    index._doc_types[0].init(suffix=suffix)
                 elif action == OpensearchAction.DELETE:
                     index.delete()
-                else:
-                    try:
-                        index.delete()
-                    except opensearchpy.exceptions.NotFoundError:
-                        pass
-                    index.create()
             except opensearchpy.exceptions.NotFoundError:
                 if verbosity or not ignore_error:
                     self.stderr.write(f"{pp} index '{index._name}'...{self.style.ERROR('Error (not found)')}")  # noqa
@@ -127,17 +154,17 @@ def _manage_index(self, action, indices, force, verbosity, ignore_error, **optio
             except opensearchpy.exceptions.RequestError:
                 if verbosity or not ignore_error:
                     self.stderr.write(
-                        f"{pp} index '{index._name}'... {self.style.ERROR('Error (already exists)')}"
+                        f"{pp} index '{index._name}'... {self.style.ERROR('Error')}"
                     )  # noqa
                 if not ignore_error:
                     self.stderr.write("exiting...")
                     exit(1)
             else:
                 if verbosity:
-                    self.stdout.write(f"{pp} index '{index._name}'... {self.style.SUCCESS('OK')}")  # noqa
+                    self.stdout.write(f"{pp} index '{index_name}'... {self.style.SUCCESS('OK')}")  # noqa
 
     def _manage_document(
-        self, action, indices, force, filters, excludes, verbosity, parallel, count, refresh, missing, **options
+        self, action, indices, index_suffix, force, filters, excludes, verbosity, parallel, count, refresh, missing, **options
     ):  # noqa
         """Manage the creation and deletion of indices."""
         action = OpensearchAction(action)
@@ -206,7 +233,9 @@ def _manage_document(
             document = index._doc_types[0]()  # noqa
             qs = document.get_indexing_queryset(stdout=self.stdout._out, verbose=verbosity, action=action, **kwargs)
             success, errors = document.update(
-                qs, parallel=parallel, refresh=refresh, action=action, raise_on_error=False
+                qs, action,
+                parallel=parallel, index_suffix=index_suffix,
+                refresh=refresh, raise_on_error=False
             )
 
             success_str = self.style.SUCCESS(success) if success else success
@@ -241,23 +270,23 @@ def add_arguments(self, parser):
         )
         subparser.set_defaults(func=self.__list_index)
 
-        # 'manage' subcommand
+        # 'index' subcommand
         subparser = subparsers.add_parser(
             "index",
-            help="Manage the creation an deletion of indices.",
-            description="Manage the creation an deletion of indices.",
+            help="Manage the deletion of indices.",
+            description="Manage the deletion of indices.",
         )
         subparser.set_defaults(func=self._manage_index)
         subparser.add_argument(
             "action",
             type=str,
-            help="Whether you want to create, delete or rebuild the indices.",
+            help="Whether you want to create or delete the indices.",
             choices=[
                 OpensearchAction.CREATE.value,
                 OpensearchAction.DELETE.value,
-                OpensearchAction.REBUILD.value,
             ],
         )
+        subparser.add_argument("--suffix", type=str, default=None, help="A suffix for the index name to create/delete (if you don't provide one, a timestamp will be used for creation).")
         subparser.add_argument("--force", action="store_true", default=False, help="Do not ask for confirmation.")
         subparser.add_argument("--ignore-error", action="store_true", default=False, help="Do not stop on error.")
         subparser.add_argument(
@@ -279,7 +308,7 @@ def add_arguments(self, parser):
         subparser.add_argument(
             "action",
             type=str,
-            help="Whether you want to create, delete or rebuild the indices.",
+            help="Whether you want to index, delete or update documents in indices.",
             choices=[
                 OpensearchAction.INDEX.value,
                 OpensearchAction.DELETE.value,
@@ -316,6 +345,7 @@ def add_arguments(self, parser):
             ),
         )
         subparser.add_argument("--force", action="store_true", default=False, help="Do not ask for confirmation.")
+        subparser.add_argument("--index-suffix", type=str, default=None, help="The suffix for the index name (if you don't provide one, the current index will be used).")
         subparser.add_argument(
             "-i", "--indices", type=str, nargs="*", help="Only update documents on the given indices."
         )

django_opensearch_dsl/management/enums.py

@@ -7,10 +7,8 @@ class OpensearchAction(str, Enum):
     INDEX = ("index", "indexing", "indexed")
     UPDATE = ("update", "updating", "updated")
     CREATE = ("create", "creating", "created")
-    REBUILD = ("rebuild", "rebuilding", "rebuilded")
     LIST = ("list", "listing", "listed")
     DELETE = ("delete", "deleting", "deleted")
-    MANAGE = ("manage", "managing", "managed")
 
     def __new__(cls, value: str, present_participle: str, past: str):  # noqa: D102
         obj = str.__new__(cls, value)

tests/tests/management/test_document.py

@@ -3,11 +3,11 @@
 import time
 
 from django.test import TestCase
+from opensearch_dsl.connections import get_connection
 
 from django_dummy_app.commands import call_command
 from django_dummy_app.documents import CountryDocument, ContinentDocument, EventDocument
 from django_dummy_app.models import Country, Event, Continent
-from django_opensearch_dsl.registries import registry
 
 
 class DocumentTestCase(TestCase):
@@ -20,9 +20,7 @@ def setUpClass(cls):
         cls.call_command = functools.partial(call_command, stdout=devnull, stderr=devnull)
 
     def setUp(self) -> None:
-        indices = registry.get_indices()
-        for i in indices:
-            i.delete(ignore_unavailable=True)
+        get_connection().indices.delete('_all')
 
     def test_unknown_index(self):
         with self.assertRaises(SystemExit):

tests/tests/management/test_index.py

@@ -2,6 +2,7 @@
 import os
 
 from django.test import SimpleTestCase
+from opensearch_dsl.connections import get_connection
 
 from django_dummy_app.commands import call_command
 from django_dummy_app.documents import CountryDocument, ContinentDocument, EventDocument
@@ -16,15 +17,13 @@ def setUpClass(cls):
         cls.call_command = functools.partial(call_command, stdout=devnull, stderr=devnull)
 
     def setUp(self) -> None:
-        indices = registry.get_indices()
-        for i in indices:
-            i.delete(ignore_unavailable=True)
+        get_connection().indices.delete('_all')
 
     def test_index_creation_all(self):
         indices = registry.get_indices()
 
         self.assertFalse(any(map(lambda i: i.exists(), indices)))
-        self.call_command("opensearch", "index", "create", force=True)
+        self.call_command("opensearch", "index", "create", suffix="v1", force=True)
         self.assertTrue(all(map(lambda i: i.exists(), indices)))
 
     def test_index_creation_one(self):
@@ -60,38 +59,30 @@ def test_index_creation_two(self):
         self.assertTrue(country_document._index.exists())
         self.assertTrue(event_document._index.exists())
 
-    def test_index_creation_error(self):
-        country_document = CountryDocument()
-        self.call_command("opensearch", "index", "create", country_document.Index.name, force=True)
-
-        self.call_command("opensearch", "index", "create", country_document.Index.name, force=True, ignore_error=True)
-        with self.assertRaises(SystemExit):
-            self.call_command("opensearch", "index", "create", country_document.Index.name, force=True)
-
     def test_index_deletion_all(self):
-        self.call_command("opensearch", "index", "create", force=True)
+        self.call_command("opensearch", "index", "create", suffix="v1", force=True)
         indices = registry.get_indices()
 
         self.assertTrue(all(map(lambda i: i.exists(), indices)))
         self.call_command("opensearch", "index", "delete", force=True)
         self.assertFalse(any(map(lambda i: i.exists(), indices)))
 
     def test_index_deletion_one(self):
-        self.call_command("opensearch", "index", "create", force=True)
+        self.call_command("opensearch", "index", "create", suffix="v1", force=True)
         continent_document = ContinentDocument()
         country_document = CountryDocument()
         event_document = EventDocument()
 
         self.assertTrue(continent_document._index.exists())
         self.assertTrue(country_document._index.exists())
         self.assertTrue(event_document._index.exists())
-        self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True)
+        self.call_command("opensearch", "index", "delete", "country--v1", force=True)
         self.assertTrue(continent_document._index.exists())
         self.assertFalse(country_document._index.exists())
         self.assertTrue(event_document._index.exists())
 
     def test_index_deletion_two(self):
-        self.call_command("opensearch", "index", "create", force=True)
+        self.call_command("opensearch", "index", "create", suffix="v1", force=True)
         continent_document = ContinentDocument()
         country_document = CountryDocument()
         event_document = EventDocument()
@@ -103,8 +94,8 @@ def test_index_deletion_two(self):
             "opensearch",
             "index",
             "delete",
-            country_document.Index.name,
-            event_document.Index.name,
+            "country--v1",
+            "event--v1",
             force=True,
         )
         self.assertTrue(continent_document._index.exists())
@@ -113,38 +104,11 @@ def test_index_deletion_two(self):
 
     def test_index_deletion_error(self):
         country_document = CountryDocument()
+        country_document.init(suffix="v1")
 
-        self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True, ignore_error=True)
+        self.call_command("opensearch", "index", "delete", "country--v1", force=True, ignore_error=True)
         with self.assertRaises(SystemExit):
-            self.call_command("opensearch", "index", "delete", country_document.Index.name, force=True)
-
-    def test_index_rebuild_two(self):
-        continent_document = ContinentDocument()
-        country_document = CountryDocument()
-        event_document = EventDocument()
-        self.call_command(
-            "opensearch",
-            "index",
-            "create",
-            continent_document.Index.name,
-            event_document.Index.name,
-            force=True,
-        )
-
-        self.assertTrue(continent_document._index.exists())
-        self.assertFalse(country_document._index.exists())
-        self.assertTrue(event_document._index.exists())
-        self.call_command(
-            "opensearch",
-            "index",
-            "rebuild",
-            country_document.Index.name,
-            event_document.Index.name,
-            force=True,
-        )
-        self.assertTrue(continent_document._index.exists())
-        self.assertTrue(country_document._index.exists())
-        self.assertTrue(event_document._index.exists())
+            self.call_command("opensearch", "index", "delete", "country--v1", force=True)
 
     def test_unknown_index(self):
         with self.assertRaises(SystemExit):