Added LevelDB-based attribute container store

log2timeline · Jan 23, 2024 · 100d9f6 · 100d9f6
1 parent 180e7db
commit 100d9f6
Show file tree

Hide file tree

Showing 9 changed files with 74 additions and 13 deletions.
diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml
@@ -18,7 +18,7 @@ jobs:
     - name: Install dependencies
       run: |
         dnf copr -y enable @gift/dev
-        dnf install -y @development-tools python3 python3-devel python3-pyyaml python3-setuptools
+        dnf install -y @development-tools python3 python3-devel python3-leveldb python3-pyyaml python3-setuptools
     - name: Run tests
       env:
         LANG: C.utf8
@@ -58,7 +58,7 @@ jobs:
       run: |
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential python3 python3-dev python3-distutils python3-pip python3-setuptools python3-wheel python3-yaml
+        apt-get install -y build-essential python3 python3-dev python3-distutils python3-leveldb python3-pip python3-setuptools python3-wheel python3-yaml
     - name: Run tests
       env:
         LANG: en_US.UTF-8

diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml
@@ -36,7 +36,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools python3-yaml
+        apt-get install -y build-essential git  python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-leveldb python3-pip python3-setuptools python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox

diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml
@@ -46,7 +46,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools python3-yaml
+        apt-get install -y build-essential git  python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-leveldb python3-pip python3-setuptools python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox
@@ -82,7 +82,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools python3-yaml
+        apt-get install -y build-essential git  python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-leveldb python3-pip python3-setuptools python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox
@@ -120,7 +120,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools python3-yaml
+        apt-get install -y build-essential git  python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-leveldb python3-pip python3-setuptools python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox

diff --git a/acstore/leveldb_store.py b/acstore/leveldb_store.py
@@ -33,6 +33,25 @@ def __init__(self):
     self.format_version = self._FORMAT_VERSION
     self.serialization_format = 'json'
 
+  def _GetNumberOfAttributeContainerKeys(self, container_type):
+    """Retrieves the number of attribute container keys.
+
+    Args:
+      container_type (str): attribute container type.
+
+    Returns:
+      int: the number of keys of a specified attribute container type.
+    """
+    first_key = f'{container_type:s}.1'.encode('utf8')
+
+    try:
+      # Check if the first key exists otherwise RangeIter will return all keys.
+      self._leveldb_database.Get(first_key)
+    except KeyError:
+      return 0
+
+    return sum(1 for _ in self._leveldb_database.RangeIter(key_from=first_key))
+
   def _RaiseIfNotReadable(self):
     """Raises if the attribute container store is not readable.
 
@@ -243,3 +262,11 @@ def Open(self, path=None, **unused_kwargs):  # pylint: disable=arguments-differ
     self._is_open = True
 
     # TODO: read metadata.
+
+    # Initialize next_sequence_number based on the file contents so that
+    # AttributeContainerIdentifier points to the correct attribute container.
+    for container_type in self._containers_manager.GetContainerTypes():
+      next_sequence_number = self._GetNumberOfAttributeContainerKeys(
+          container_type)
+      self._SetAttributeContainerNextSequenceNumber(
+          container_type, next_sequence_number)
diff --git a/config/appveyor/install.ps1 b/config/appveyor/install.ps1
@@ -1,6 +1,6 @@
 # Script to set up tests on AppVeyor Windows.
 
-$Dependencies = "PyYAML"
+$Dependencies = "PyYAML leveldb"
 
 If ($Dependencies.Length -gt 0)
 {

diff --git a/config/dpkg/control b/config/dpkg/control
@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/acstore
 
 Package: python3-acstore
 Architecture: all
-Depends: python3-yaml (>= 3.10), ${misc:Depends}
+Depends: python3-leveldb (>= 0.20), python3-yaml (>= 3.10), ${misc:Depends}
 Description: Python 3 module of ACStore
  ACStore, or Attribute Container Storage, provides a stand-alone
  implementation to read and write attribute container storage files.
diff --git a/dependencies.ini b/dependencies.ini
@@ -1,3 +1,9 @@
+[leveldb]
+dpkg_name: python3-leveldb
+is_optional: true
+minimum_version: 0.20
+rpm_name: python3-leveldb
+
 [yaml]
 dpkg_name: python3-yaml
 l2tbinaries_name: PyYAML

diff --git a/requirements.txt b/requirements.txt
@@ -1 +1,2 @@
 PyYAML >= 3.10
+leveldb >= 0.20
diff --git a/tests/leveldb_store.py b/tests/leveldb_store.py
@@ -32,6 +32,33 @@ def tearDown(self):
     containers_manager.AttributeContainersManager.DeregisterAttributeContainer(
         test_lib.TestAttributeContainer)
 
+  def testGetNumberOfAttributeContainerKeys(self):
+    """Tests the _GetNumberOfAttributeContainerKeys function."""
+    attribute_container = test_lib.TestAttributeContainer()
+
+    with test_lib.TempDirectory() as temp_directory:
+      test_path = os.path.join(temp_directory, 'acstore.leveldb')
+      test_store = leveldb_store.LevelDBAttributeContainerStore()
+      test_store.Open(path=test_path, read_only=False)
+
+      try:
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
+            attribute_container.CONTAINER_TYPE)
+        self.assertEqual(number_of_containers, 0)
+
+        test_store.AddAttributeContainer(attribute_container)
+
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
+            attribute_container.CONTAINER_TYPE)
+        self.assertEqual(number_of_containers, 1)
+
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
+            'bogus')
+        self.assertEqual(number_of_containers, 0)
+
+      finally:
+        test_store.Close()
+
   def testRaiseIfNotReadable(self):
     """Tests the _RaiseIfNotReadable function."""
     test_store = leveldb_store.LevelDBAttributeContainerStore()
@@ -56,19 +83,19 @@ def testWriteExistingAttributeContainer(self):
       test_store.Open(path=test_path, read_only=False)
 
       try:
-        number_of_containers = test_store.GetNumberOfAttributeContainers(
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
             attribute_container.CONTAINER_TYPE)
         self.assertEqual(number_of_containers, 0)
 
         test_store._WriteNewAttributeContainer(attribute_container)
 
-        number_of_containers = test_store.GetNumberOfAttributeContainers(
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
             attribute_container.CONTAINER_TYPE)
         self.assertEqual(number_of_containers, 1)
 
         test_store._WriteExistingAttributeContainer(attribute_container)
 
-        number_of_containers = test_store.GetNumberOfAttributeContainers(
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
             attribute_container.CONTAINER_TYPE)
         self.assertEqual(number_of_containers, 1)
 
@@ -85,13 +112,13 @@ def testWriteNewAttributeContainer(self):
       test_store.Open(path=test_path)
 
       try:
-        number_of_containers = test_store.GetNumberOfAttributeContainers(
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
             attribute_container.CONTAINER_TYPE)
         self.assertEqual(number_of_containers, 0)
 
         test_store._WriteNewAttributeContainer(attribute_container)
 
-        number_of_containers = test_store.GetNumberOfAttributeContainers(
+        number_of_containers = test_store._GetNumberOfAttributeContainerKeys(
             attribute_container.CONTAINER_TYPE)
         self.assertEqual(number_of_containers, 1)