From 769149f54410ff924168588b64f9da6afb9a6985 Mon Sep 17 00:00:00 2001
From: Leane Schlundt <lhecht@techfak.de>
Date: Tue, 13 Feb 2024 09:22:19 +0100
Subject: [PATCH] Chunk annotations and annotation labels

Use chunking to prevent exceeding
the database insert limit during volume cloning.

Fixes #777.
---
 app/Jobs/CloneImagesOrVideos.php | 36 +++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/app/Jobs/CloneImagesOrVideos.php b/app/Jobs/CloneImagesOrVideos.php
index bf58c1e5f..dec6e9740 100644
--- a/app/Jobs/CloneImagesOrVideos.php
+++ b/app/Jobs/CloneImagesOrVideos.php
@@ -17,6 +17,7 @@
 use Biigle\VideoAnnotationLabel;
 use Biigle\VideoLabel;
 use Biigle\Volume;
+use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Http\Request;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
@@ -253,6 +254,7 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
         }
 
         $chunkSize = 100;
+        $databaseRecordLimit = 64999;
         $newImageIds = $copy->images()->orderBy('id')->pluck('id');
         $volume->images()
             ->with([
@@ -267,9 +269,9 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
                 $newImageIds,
                 $chunkSize,
                 $usedAnnotationIds,
-                $selectedLabelIds
+                $databaseRecordLimit,
             ) {
-                $insertData = [];
+                $insertData = new Collection();
                 $chunkNewImageIds = [];
                 // Consider all previous image chunks when calculating the start of the index.
                 $baseImageIndex = ($page - 1) * $chunkSize;
@@ -281,16 +283,17 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
                         $original = $annotation->getRawOriginal();
                         $original['image_id'] = $newImageId;
                         unset($original['id']);
-                        $insertData[] = $original;
+                        $insertData->push($original);
 
                     }
                 }
-                ImageAnnotation::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => ImageAnnotation::insert($chunk->toArray()));
+
                 // Get the IDs of all newly inserted annotations. Ordering is essential.
                 $newAnnotationIds = ImageAnnotation::whereIn('image_id', $chunkNewImageIds)
                     ->orderBy('id')
                     ->pluck('id');
-                $insertData = [];
+                $insertData = new Collection();
                 foreach ($chunk as $image) {
                     foreach ($image->annotations as $annotation) {
                         $newAnnotationId = $newAnnotationIds->shift();
@@ -298,11 +301,12 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
                             $original = $annotationLabel->getRawOriginal();
                             $original['annotation_id'] = $newAnnotationId;
                             unset($original['id']);
-                            $insertData[] = $original;
+                            $insertData->push($original);
                         }
                     }
                 }
-                ImageAnnotationLabel::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => ImageAnnotationLabel::insert($chunk->toArray()));
+
             });
     }
 
@@ -410,6 +414,7 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
         }
 
         $chunkSize = 100;
+        $databaseRecordLimit = 64999;
         $newVideoIds = $copy->videos()->orderBy('id')->pluck('id');
         $volume->videos()
             ->with([
@@ -424,9 +429,9 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
                 $newVideoIds,
                 $chunkSize,
                 $usedAnnotationIds,
-                $selectedLabelIds
+                $databaseRecordLimit,
             ) {
-                $insertData = [];
+                $insertData = new Collection();
                 $chunkNewVideoIds = [];
                 // Consider all previous video chunks when calculating the start of the index.
                 $baseVideoIndex = ($page - 1) * $chunkSize;
@@ -438,16 +443,18 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
                         $original = $annotation->getRawOriginal();
                         $original['video_id'] = $newVideoId;
                         unset($original['id']);
-                        $insertData[] = $original;
+                        $insertData->push($original);
 
                     }
                 }
-                VideoAnnotation::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => VideoAnnotation::insert($chunk->toArray()));
+
+                
                 // Get the IDs of all newly inserted annotations. Ordering is essential.
                 $newAnnotationIds = VideoAnnotation::whereIn('video_id', $chunkNewVideoIds)
                     ->orderBy('id')
                     ->pluck('id');
-                $insertData = [];
+                $insertData = new Collection();
                 foreach ($chunk as $video) {
                     foreach ($video->annotations as $annotation) {
                         $newAnnotationId = $newAnnotationIds->shift();
@@ -455,11 +462,12 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
                             $original = $annotationLabel->getRawOriginal();
                             $original['annotation_id'] = $newAnnotationId;
                             unset($original['id']);
-                            $insertData[] = $original;
+                            $insertData->push($original);
                         }
                     }
                 }
-                VideoAnnotationLabel::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => VideoAnnotationLabel::insert($chunk->toArray()));
+
             });
     }