Add a TODOs sheet containing on REQUIRES_REVIEW resources in XLSX #1524 (#1527)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -35,6 +35,9 @@ v34.9.4 (unreleased)
   at once from a directory containing input files.
   https://github.com/aboutcode-org/scancode.io/issues/1437
 
+- Add a "TODOS" sheet containing on REQUIRES_REVIEW resources in XLSX.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
 v34.9.3 (2024-12-31)
 --------------------
 

scanpipe/api/views.py

@@ -232,15 +232,17 @@ def packages(self, request, *args, **kwargs):
     @action(detail=True, filterset_class=None)
     def dependencies(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = project.discovereddependencies.all()
+        queryset = project.discovereddependencies.prefetch_for_serializer()
         return self.get_filtered_response(
             request, queryset, DependencyFilterSet, DiscoveredDependencySerializer
         )
 
     @action(detail=True, filterset_class=None)
     def relations(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = project.codebaserelations.all()
+        queryset = project.codebaserelations.select_related(
+            "from_resource", "to_resource"
+        )
         return self.get_filtered_response(
             request, queryset, RelationFilterSet, CodebaseRelationSerializer
         )

scanpipe/filters.py

@@ -423,8 +423,10 @@ def __init__(self, data=None, *args, **kwargs):
         if not data or data.get("is_archived", "") == "":
             self.queryset = self.queryset.filter(is_archived=False)
 
-        active_count = Project.objects.filter(is_archived=False).count()
-        archived_count = Project.objects.filter(is_archived=True).count()
+        counts = Project.objects.get_active_archived_counts()
+        active_count = counts["active_count"]
+        archived_count = counts["archived_count"]
+
         self.filters["is_archived"].extra["widget"] = BulmaLinkWidget(
             choices=[
                 ("", f'<i class="fa-solid fa-seedling"></i> {active_count} Active'),

scanpipe/forms.py

@@ -272,6 +272,23 @@ class ProjectOutputDownloadForm(forms.Form):
     )
 
 
+class ProjectReportForm(forms.Form):
+    model_name = forms.ChoiceField(
+        label="Choose the object type to include in the XLSX file",
+        choices=[
+            ("discoveredpackage", "Packages"),
+            ("discovereddependency", "Dependencies"),
+            ("codebaseresource", "Resources"),
+            ("codebaserelation", "Relations"),
+            ("projectmessage", "Messages"),
+            ("todos", "TODOs"),
+        ],
+        required=True,
+        initial="discoveredpackage",
+        widget=forms.RadioSelect,
+    )
+
+
 class ListTextarea(forms.CharField):
     """
     A Django form field that displays as a textarea and converts each line of input

scanpipe/models.py

@@ -44,13 +44,15 @@
 from django.core.validators import EMPTY_VALUES
 from django.db import models
 from django.db import transaction
+from django.db.models import Case
 from django.db.models import Count
 from django.db.models import IntegerField
 from django.db.models import OuterRef
 from django.db.models import Prefetch
 from django.db.models import Q
 from django.db.models import Subquery
 from django.db.models import TextField
+from django.db.models import When
 from django.db.models.functions import Cast
 from django.db.models.functions import Lower
 from django.dispatch import receiver
@@ -518,6 +520,16 @@ def with_counts(self, *fields):
 
         return self.annotate(**annotations)
 
+    def get_active_archived_counts(self):
+        return self.aggregate(
+            active_count=Count(
+                Case(When(is_archived=False, then=1), output_field=IntegerField())
+            ),
+            archived_count=Count(
+                Case(When(is_archived=True, then=1), output_field=IntegerField())
+            ),
+        )
+
 
 class UUIDTaggedItem(GenericUUIDTaggedItemBase, TaggedItemBase):
     class Meta:
@@ -2913,7 +2925,9 @@ def create_from_data(cls, project, resource_data):
         cleaned_data = {
             field_name: value
             for field_name, value in resource_data.items()
-            if field_name in cls.model_fields() and value not in EMPTY_VALUES
+            if field_name in cls.model_fields()
+            and value not in EMPTY_VALUES
+            and field_name != "project"
         }
 
         return cls.objects.create(project=project, **cleaned_data)
@@ -3451,7 +3465,9 @@ def create_from_data(cls, project, package_data):
         cleaned_data = {
             field_name: value
             for field_name, value in package_data.items()
-            if field_name in cls.model_fields() and value not in EMPTY_VALUES
+            if field_name in cls.model_fields()
+            and value not in EMPTY_VALUES
+            and field_name != "project"
         }
 
         discovered_package = cls(project=project, **cleaned_data)
@@ -3912,7 +3928,9 @@ def create_from_data(
         cleaned_data = {
             field_name: value
             for field_name, value in dependency_data.items()
-            if field_name in cls.model_fields() and value not in EMPTY_VALUES
+            if field_name in cls.model_fields()
+            and value not in EMPTY_VALUES
+            and field_name != "project"
         }
 
         return cls.objects.create(

scanpipe/pipes/output.py

@@ -52,7 +52,13 @@
 
 from scancodeio import SCAN_NOTICE
 from scancodeio import __version__ as scancodeio_version
+from scanpipe.models import CodebaseRelation
+from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
+from scanpipe.models import DiscoveredPackage
+from scanpipe.models import ProjectMessage
 from scanpipe.pipes import docker
+from scanpipe.pipes import flag
 from scanpipe.pipes import spdx
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -67,17 +73,15 @@ def get_queryset(project, model_name):
     """Return a consistent QuerySet for all supported outputs (json, xlsx, csv, ...)"""
     querysets = {
         "discoveredpackage": (
-            project.discoveredpackages.all().order_by(
+            DiscoveredPackage.objects.order_by(
                 "type",
                 "namespace",
                 "name",
                 "version",
             )
         ),
         "discovereddependency": (
-            project.discovereddependencies.all()
-            .prefetch_for_serializer()
-            .order_by(
+            DiscoveredDependency.objects.prefetch_for_serializer().order_by(
                 "type",
                 "namespace",
                 "name",
@@ -86,14 +90,20 @@ def get_queryset(project, model_name):
             )
         ),
         "codebaseresource": (
-            project.codebaseresources.without_symlinks().prefetch_for_serializer()
+            CodebaseResource.objects.without_symlinks().prefetch_for_serializer()
         ),
         "codebaserelation": (
-            project.codebaserelations.select_related("from_resource", "to_resource")
+            CodebaseRelation.objects.select_related("from_resource", "to_resource")
         ),
-        "projectmessage": project.projectmessages.all(),
+        "projectmessage": ProjectMessage.objects.all(),
+        "todos": CodebaseResource.objects.files().status(flag.REQUIRES_REVIEW),
     }
-    return querysets.get(model_name)
+
+    queryset = querysets.get(model_name)
+    if project:
+        queryset = queryset.filter(project=project)
+
+    return queryset
 
 
 def queryset_to_csv_file(queryset, fieldnames, output_file):
@@ -295,7 +305,11 @@ def to_json(project):
 
 
 def queryset_to_xlsx_worksheet(
-    queryset, workbook, exclude_fields=None, extra_fields=None
+    queryset,
+    workbook,
+    exclude_fields=None,
+    extra_fields=None,
+    worksheet_name=None,
 ):
     """
     Add a new worksheet to the ``workbook`` ``xlsxwriter.Workbook`` using the
@@ -310,7 +324,7 @@ def queryset_to_xlsx_worksheet(
 
     model_class = queryset.model
     model_name = model_class._meta.model_name
-    worksheet_name = model_name_to_worksheet_name.get(model_name)
+    worksheet_name = worksheet_name or model_name_to_worksheet_name.get(model_name)
 
     fields = get_serializer_fields(model_class)
     exclude_fields = exclude_fields or []
@@ -346,8 +360,12 @@ def _add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
 
     for row_index, record in enumerate(rows, start=1):
         row_errors = []
+        record_is_dict = isinstance(record, dict)
         for col_index, field in enumerate(fields):
-            value = getattr(record, field)
+            if record_is_dict:
+                value = record.get(field)
+            else:
+                value = getattr(record, field)
 
             if not value:
                 continue
@@ -481,6 +499,12 @@ def to_xlsx(project):
         if layers_data := docker.get_layers_data(project):
             _add_xlsx_worksheet(workbook, "LAYERS", layers_data, docker.layer_fields)
 
+        todos_queryset = get_queryset(project, "todos")
+        if todos_queryset:
+            queryset_to_xlsx_worksheet(
+                todos_queryset, workbook, exclude_fields, worksheet_name="TODOS"
+            )
+
     return output_file
 
 

scanpipe/templates/scanpipe/modals/projects_report_modal.html

@@ -7,9 +7,9 @@
     </header>
     <form action="{% url 'project_action' %}" method="post" id="report-projects-form">{% csrf_token %}
       <section class="modal-card-body">
-        <div class="notification is-info has-text-weight-semibold">
-          All the packages for the selected projects will be included in an XLSX report.
-        </div>
+        <ul class="mb-3">
+          {{ report_form.as_ul }}
+        </ul>
       </section>
       <input type="hidden" name="action" value="report">
       <footer class="modal-card-foot is-flex is-justify-content-space-between">

scanpipe/tests/pipes/test_d2d.py

@@ -1530,6 +1530,7 @@ def test_scanpipe_pipes_d2d_map_go_paths(self):
             ).count(),
         )
 
+    @skipIf(sys.platform == "darwin", "Test is failing on macOS")
     def test_scanpipe_pipes_d2d_map_rust_paths(self):
         input_dir = self.project1.input_path
         input_resources = [

scanpipe/tests/pipes/test_output.py

@@ -33,6 +33,7 @@
 from django.core.management import call_command
 from django.test import TestCase
 
+import openpyxl
 import xlsxwriter
 from licensedcode.cache import get_licensing
 from lxml import etree
@@ -42,10 +43,12 @@
 from scanpipe.models import CodebaseResource
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
+from scanpipe.pipes import flag
 from scanpipe.pipes import output
 from scanpipe.tests import FIXTURES_REGEN
 from scanpipe.tests import make_dependency
 from scanpipe.tests import make_package
+from scanpipe.tests import make_resource_file
 from scanpipe.tests import mocked_now
 from scanpipe.tests import package_data1
 
@@ -210,16 +213,30 @@ def test_scanpipe_pipes_outputs_to_xlsx(self):
             model="Model",
             details={},
         )
+        make_resource_file(
+            project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW
+        )
 
         output_file = output.to_xlsx(project=project)
         self.assertIn(output_file.name, project.output_root)
 
         # Make sure the output can be generated even if the work_directory was wiped
         shutil.rmtree(project.work_directory)
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(10):
             output_file = output.to_xlsx(project=project)
         self.assertIn(output_file.name, project.output_root)
 
+        workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True)
+        expected_sheet_names = [
+            "PACKAGES",
+            "DEPENDENCIES",
+            "RESOURCES",
+            "RELATIONS",
+            "MESSAGES",
+            "TODOS",
+        ]
+        self.assertEqual(expected_sheet_names, workbook.get_sheet_names())
+
     def test_scanpipe_pipes_outputs_vulnerability_as_cyclonedx(self):
         component_bom_ref = "pkg:pypi/[email protected]"
         data = self.data / "cyclonedx/django-4.0.10-vulnerability.json"

scanpipe/tests/test_views.py

@@ -192,6 +192,16 @@ def test_scanpipe_views_project_actions_view(self):
         )
         self.assertContains(response, expected, html=True)
 
+    def test_scanpipe_views_project_action_report_view(self):
+        url = reverse("project_action")
+        data = {
+            "action": "report",
+            "selected_ids": f"{self.project1.uuid}",
+            "model_name": "todos",
+        }
+        response = self.client.post(url, data=data, follow=True)
+        self.assertEqual("report.xlsx", response.filename)
+
     def test_scanpipe_views_project_details_is_archived(self):
         url = self.project1.get_absolute_url()
         expected1 = "WARNING: This project is archived and read-only."
@@ -701,7 +711,7 @@ def test_scanpipe_views_project_views(self):
         project2.labels.add("label3", "label4")
 
         url = reverse("project_list")
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(7):
             self.client.get(url)
 
         with self.assertNumQueries(13):