Remove unused methods from ArchiveService and unused ArchiveField (#793)

Author: Swatinem

services/archive.py

@@ -1,13 +1,10 @@
-import json
 import logging
 from base64 import b16encode
 from enum import Enum
 from hashlib import md5
 from uuid import uuid4
 
-from django.conf import settings
 from django.utils import timezone
-from shared.utils.ReportEncoder import ReportEncoder
 
 from services.storage import StorageService
 from utils.config import get_config
@@ -17,11 +14,7 @@
 
 class MinioEndpoints(Enum):
     chunks = "{version}/repos/{repo_hash}/commits/{commitid}/chunks.txt"
-    json_data = "{version}/repos/{repo_hash}/commits/{commitid}/json_data/{table}/{field}/{external_id}.json"
-    json_data_no_commit = (
-        "{version}/repos/{repo_hash}/json_data/{table}/{field}/{external_id}.json"
-    )
-    raw = "v4/raw/{date}/{repo_hash}/{commit_sha}/{reportid}.txt"
+
     raw_with_upload_id = (
         "v4/raw/{date}/{repo_hash}/{commit_sha}/{reportid}/{uploadid}.txt"
     )
@@ -31,7 +24,6 @@ class MinioEndpoints(Enum):
     static_analysis_single_file = (
         "{version}/repos/{repo_hash}/static_analysis/files/{location}"
     )
-
     test_results = "test_results/v1/raw/{date}/{repo_hash}/{commit_sha}/{uploadid}.txt"
 
     def get_path(self, **kwaargs):
@@ -71,21 +63,6 @@ def __init__(self, repository, ttl=None):
         self.storage = StorageService()
         self.storage_hash = self.get_archive_hash(repository)
 
-    """
-    Accessor for underlying StorageService. You typically shouldn't need
-    this for anything.
-    """
-
-    def storage_client(self):
-        return self.storage
-
-    """
-    Getter. Returns true if the current configuration is enterprise.
-    """
-
-    def is_enterprise(self):
-        return settings.IS_ENTERPRISE
-
     """
     Generates a hash key from repo specific information.
     Provides slight obfuscation of data in minio storage
@@ -109,133 +86,23 @@ def get_archive_hash(cls, repository):
         _hash.update(val)
         return b16encode(_hash.digest()).decode()
 
-    def write_json_data_to_storage(
-        self,
-        commit_id,
-        table: str,
-        field: str,
-        external_id: str,
-        data: dict,
-        *,
-        encoder=ReportEncoder,
-    ):
-        if commit_id is None:
-            # Some classes don't have a commit associated with them
-            # For example Pull belongs to multiple commits.
-            path = MinioEndpoints.json_data_no_commit.get_path(
-                version="v4",
-                repo_hash=self.storage_hash,
-                table=table,
-                field=field,
-                external_id=external_id,
-            )
-        else:
-            path = MinioEndpoints.json_data.get_path(
-                version="v4",
-                repo_hash=self.storage_hash,
-                commitid=commit_id,
-                table=table,
-                field=field,
-                external_id=external_id,
-            )
-        stringified_data = json.dumps(data, cls=encoder)
-        self.write_file(path, stringified_data)
-        return path
-
-    """
-    Writes a generic file to the archive -- it's typically recommended to
-    not use this in lieu of the convenience methods write_raw_upload and
-    write_chunks
-    """
-
-    def write_file(self, path, data, reduced_redundancy=False, gzipped=False):
-        self.storage.write_file(
-            self.root,
-            path,
-            data,
-            reduced_redundancy=reduced_redundancy,
-            gzipped=gzipped,
-        )
-
-    """
-    Convenience write method, writes a raw upload to a destination.
-    Returns the path it writes.
-    """
-
-    def write_raw_upload(self, commit_sha, report_id, data, gzipped=False):
-        # create a custom report path for a raw upload.
-        # write the file.
-        path = "/".join(
-            (
-                "v4/raw",
-                timezone.now().strftime("%Y-%m-%d"),
-                self.storage_hash,
-                commit_sha,
-                "%s.txt" % report_id,
-            )
-        )
-
-        self.write_file(path, data, gzipped=gzipped)
-
-        return path
-
-    """
-    Convenience method to write a chunks.txt file to storage.
-    """
-
-    def write_chunks(self, commit_sha, data):
-        path = MinioEndpoints.chunks.get_path(
-            version="v4", repo_hash=self.storage_hash, commitid=commit_sha
-        )
-
-        self.write_file(path, data)
-        return path
-
-    """
-    Generic method to read a file from the archive
-    """
-
     def read_file(self, path):
+        """
+        Generic method to read a file from the archive
+        """
         contents = self.storage.read_file(self.root, path)
         return contents.decode()
 
-    """
-    Generic method to delete a file from the archive.
-    """
-
-    def delete_file(self, path):
-        self.storage.delete_file(self.root, path)
-
-    """
-    Deletes an entire repository's contents
-    """
-
-    def delete_repo_files(self):
-        path = "v4/repos/{}".format(self.storage_hash)
-        objects = self.storage.list_folder_contents(self.root, path)
-        for obj in objects:
-            self.storage.delete_file(self.root, obj.object_name)
-
-    """
-    Convenience method to read a chunks file from the archive.
-    """
-
     def read_chunks(self, commit_sha):
+        """
+        Convenience method to read a chunks file from the archive.
+        """
         path = MinioEndpoints.chunks.get_path(
             version="v4", repo_hash=self.storage_hash, commitid=commit_sha
         )
         log.info("Downloading chunks from path %s for commit %s", path, commit_sha)
         return self.read_file(path)
 
-    """
-    Delete a chunk file from the archive
-    """
-
-    def delete_chunk_from_archive(self, commit_sha):
-        path = "v4/repos/{}/commits/{}/chunks.txt".format(self.storage_hash, commit_sha)
-
-        self.delete_file(path)
-
     def create_presigned_put(self, path):
         return self.storage.create_presigned_put(self.root, path, self.ttl)
 

services/tests/test_archive.py

@@ -1,9 +1,7 @@
-import json
 from pathlib import Path
 from unittest.mock import patch
 
 from django.test import TestCase
-from shared.storage import MinioStorageService
 
 from core.tests.factories import RepositoryFactory
 from services.archive import ArchiveService
@@ -18,85 +16,3 @@ def test_create_raw_upload_presigned_put(self, create_presigned_put_mock):
         repo = RepositoryFactory.create()
         service = ArchiveService(repo)
         assert service.create_raw_upload_presigned_put("ABCD") == "presigned url"
-
-
-class TestWriteData(object):
-    def test_write_report_details_to_storage(self, mocker, db):
-        repo = RepositoryFactory()
-        mock_write_file = mocker.patch.object(MinioStorageService, "write_file")
-
-        data = [
-            {
-                "filename": "file_1.go",
-                "file_index": 0,
-                "file_totals": [0, 8, 5, 3, 0, "62.50000", 0, 0, 0, 0, 10, 2, 0],
-                "diff_totals": None,
-            },
-            {
-                "filename": "file_2.py",
-                "file_index": 1,
-                "file_totals": [0, 2, 1, 0, 1, "50.00000", 1, 0, 0, 0, 0, 0, 0],
-                "diff_totals": None,
-            },
-        ]
-        archive_service = ArchiveService(repository=repo)
-        commitid = "some-commit-sha"
-        external_id = "some-uuid4-id"
-        path = archive_service.write_json_data_to_storage(
-            commit_id=commitid,
-            table="reports_reportsdetails",
-            field="files_array",
-            external_id=external_id,
-            data=data,
-        )
-        assert (
-            path
-            == f"v4/repos/{archive_service.storage_hash}/commits/{commitid}/json_data/reports_reportsdetails/files_array/{external_id}.json"
-        )
-        mock_write_file.assert_called_with(
-            archive_service.root,
-            path,
-            json.dumps(data),
-            gzipped=False,
-            reduced_redundancy=False,
-        )
-
-    def test_write_report_details_to_storage_no_commitid(self, mocker, db):
-        repo = RepositoryFactory()
-        mock_write_file = mocker.patch.object(MinioStorageService, "write_file")
-
-        data = [
-            {
-                "filename": "file_1.go",
-                "file_index": 0,
-                "file_totals": [0, 8, 5, 3, 0, "62.50000", 0, 0, 0, 0, 10, 2, 0],
-                "diff_totals": None,
-            },
-            {
-                "filename": "file_2.py",
-                "file_index": 1,
-                "file_totals": [0, 2, 1, 0, 1, "50.00000", 1, 0, 0, 0, 0, 0, 0],
-                "diff_totals": None,
-            },
-        ]
-        archive_service = ArchiveService(repository=repo)
-        commitid = None
-        external_id = "some-uuid4-id"
-        path = archive_service.write_json_data_to_storage(
-            commit_id=commitid,
-            table="reports_reportsdetails",
-            field="files_array",
-            external_id=external_id,
-            data=data,
-        )
-        assert (
-            path
-            == f"v4/repos/{archive_service.storage_hash}/json_data/reports_reportsdetails/files_array/{external_id}.json"
-        )
-        mock_write_file.assert_called_with(
-            archive_service.root,
-            path,
-            json.dumps(data),
-            gzipped=False,
-            reduced_redundancy=False,
-        )