Chunk annotations and annotation labels

Use chunking to prevent exceeding the database insert limit during volume cloning. Fixes #777.
biigle · Feb 13, 2024 · 769149f · 769149f
1 parent f547b9d
commit 769149f
Showing 1 changed file with 22 additions and 14 deletions.
diff --git a/app/Jobs/CloneImagesOrVideos.php b/app/Jobs/CloneImagesOrVideos.php
@@ -17,6 +17,7 @@
 use Biigle\VideoAnnotationLabel;
 use Biigle\VideoLabel;
 use Biigle\Volume;
+use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Http\Request;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
@@ -253,6 +254,7 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
         }
 
         $chunkSize = 100;
+        $databaseRecordLimit = 64999;
         $newImageIds = $copy->images()->orderBy('id')->pluck('id');
         $volume->images()
             ->with([
@@ -267,9 +269,9 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
                 $newImageIds,
                 $chunkSize,
                 $usedAnnotationIds,
-                $selectedLabelIds
+                $databaseRecordLimit,
             ) {
-                $insertData = [];
+                $insertData = new Collection();
                 $chunkNewImageIds = [];
                 // Consider all previous image chunks when calculating the start of the index.
                 $baseImageIndex = ($page - 1) * $chunkSize;
@@ -281,28 +283,30 @@ private function copyImageAnnotation($volume, $copy, $selectedFileIds, $selected
                         $original = $annotation->getRawOriginal();
                         $original['image_id'] = $newImageId;
                         unset($original['id']);
-                        $insertData[] = $original;
+                        $insertData->push($original);
 
                     }
                 }
-                ImageAnnotation::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => ImageAnnotation::insert($chunk->toArray()));
+
                 // Get the IDs of all newly inserted annotations. Ordering is essential.
                 $newAnnotationIds = ImageAnnotation::whereIn('image_id', $chunkNewImageIds)
                     ->orderBy('id')
                     ->pluck('id');
-                $insertData = [];
+                $insertData = new Collection();
                 foreach ($chunk as $image) {
                     foreach ($image->annotations as $annotation) {
                         $newAnnotationId = $newAnnotationIds->shift();
                         foreach ($annotation->labels as $annotationLabel) {
                             $original = $annotationLabel->getRawOriginal();
                             $original['annotation_id'] = $newAnnotationId;
                             unset($original['id']);
-                            $insertData[] = $original;
+                            $insertData->push($original);
                         }
                     }
                 }
-                ImageAnnotationLabel::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => ImageAnnotationLabel::insert($chunk->toArray()));
+
             });
     }
 
@@ -410,6 +414,7 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
         }
 
         $chunkSize = 100;
+        $databaseRecordLimit = 64999;
         $newVideoIds = $copy->videos()->orderBy('id')->pluck('id');
         $volume->videos()
             ->with([
@@ -424,9 +429,9 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
                 $newVideoIds,
                 $chunkSize,
                 $usedAnnotationIds,
-                $selectedLabelIds
+                $databaseRecordLimit,
             ) {
-                $insertData = [];
+                $insertData = new Collection();
                 $chunkNewVideoIds = [];
                 // Consider all previous video chunks when calculating the start of the index.
                 $baseVideoIndex = ($page - 1) * $chunkSize;
@@ -438,28 +443,31 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected
                         $original = $annotation->getRawOriginal();
                         $original['video_id'] = $newVideoId;
                         unset($original['id']);
-                        $insertData[] = $original;
+                        $insertData->push($original);
 
                     }
                 }
-                VideoAnnotation::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => VideoAnnotation::insert($chunk->toArray()));
+
+
                 // Get the IDs of all newly inserted annotations. Ordering is essential.
                 $newAnnotationIds = VideoAnnotation::whereIn('video_id', $chunkNewVideoIds)
                     ->orderBy('id')
                     ->pluck('id');
-                $insertData = [];
+                $insertData = new Collection();
                 foreach ($chunk as $video) {
                     foreach ($video->annotations as $annotation) {
                         $newAnnotationId = $newAnnotationIds->shift();
                         foreach ($annotation->labels as $annotationLabel) {
                             $original = $annotationLabel->getRawOriginal();
                             $original['annotation_id'] = $newAnnotationId;
                             unset($original['id']);
-                            $insertData[] = $original;
+                            $insertData->push($original);
                         }
                     }
                 }
-                VideoAnnotationLabel::insert($insertData);
+                $insertData->chunk($databaseRecordLimit)->each(fn ($chunk) => VideoAnnotationLabel::insert($chunk->toArray()));
+
             });
     }