diff --git a/src/Illuminate/Database/Concerns/BuildsQueries.php b/src/Illuminate/Database/Concerns/BuildsQueries.php index f78e13b2d500..68309ef93382 100644 --- a/src/Illuminate/Database/Concerns/BuildsQueries.php +++ b/src/Illuminate/Database/Concerns/BuildsQueries.php @@ -611,4 +611,120 @@ public function pipe($callback) { return $callback($this) ?? $this; } + + /** + * Chunk the results of the query based on available memory usage. + * + * @param int $maxMemoryUsage Maximum memory usage in megabytes + * @param callable(\Illuminate\Support\Collection, int): mixed $callback + * @return bool + */ + public function chunkByMemory($maxMemoryUsage, callable $callback) + { + $maxMemoryBytes = $maxMemoryUsage * 1024 * 1024; // Convert MB to bytes + $initialMemory = memory_get_usage(); + + // Start with a reasonable chunk size + $chunkSize = 1000; + $page = 1; + + do { + // Clone the query to avoid modifying the original + $clone = clone $this; + + // Get a chunk of results + $results = $clone->forPage($page, $chunkSize)->get(); + + $countResults = $results->count(); + + if ($countResults == 0) { + break; + } + + // Process the results + if ($callback($results, $page) === false) { + return false; + } + + // Check memory usage after processing + $currentMemory = memory_get_usage(); + $memoryDelta = $currentMemory - $initialMemory; + + // Adjust chunk size based on memory usage + if ($memoryDelta > $maxMemoryBytes) { + // If we're using too much memory, reduce chunk size + $chunkSize = max(10, (int) ($chunkSize * 0.75)); + } elseif ($memoryDelta < ($maxMemoryBytes * 0.5)) { + // If we're using less than half the allowed memory, increase chunk size + $chunkSize = min(10000, (int) ($chunkSize * 1.25)); + } + + // Force garbage collection to free memory + if (function_exists('gc_collect_cycles')) { + gc_collect_cycles(); + } + + unset($results); + + $page++; + } while ($countResults > 0); + + return true; + } + + /** + * Create a lazy collection from the query with automatic memory management. + * + * @param int $maxMemoryUsage Maximum memory usage in megabytes + * @return \Illuminate\Support\LazyCollection + */ + public function lazyByMemory($maxMemoryUsage = 100) + { + return new LazyCollection(function () use ($maxMemoryUsage) { + $page = 1; + + // Start with a reasonable chunk size + $chunkSize = 1000; + $maxMemoryBytes = $maxMemoryUsage * 1024 * 1024; // Convert MB to bytes + $initialMemory = memory_get_usage(); + + while (true) { + // Clone the query to avoid modifying the original + $clone = clone $this; + + // Get a chunk of results + $results = $clone->forPage($page, $chunkSize)->get(); + + if ($results->isEmpty()) { + break; + } + + foreach ($results as $item) { + yield $item; + } + + // Check memory usage after processing + $currentMemory = memory_get_usage(); + $memoryDelta = $currentMemory - $initialMemory; + + // Adjust chunk size based on memory usage + if ($memoryDelta > $maxMemoryBytes) { + // If we're using too much memory, reduce chunk size + $chunkSize = max(10, (int) ($chunkSize * 0.75)); + } elseif ($memoryDelta < ($maxMemoryBytes * 0.5)) { + // If we're using less than half the allowed memory, increase chunk size + $chunkSize = min(10000, (int) ($chunkSize * 1.25)); + } + + // Force garbage collection to free memory + if (function_exists('gc_collect_cycles')) { + gc_collect_cycles(); + } + + unset($results); + + $page++; + } + }); + } } diff --git a/src/Illuminate/Database/Eloquent/Builder.php b/src/Illuminate/Database/Eloquent/Builder.php index 8d29884faf0b..07d54975bd76 100755 --- a/src/Illuminate/Database/Eloquent/Builder.php +++ b/src/Illuminate/Database/Eloquent/Builder.php @@ -854,11 +854,80 @@ protected function eagerLoadRelation(array $models, $name, Closure $constraints) // using the relationship instance. Then we just return the finished arrays // of models which have been eagerly hydrated and are readied for return. return $relation->match( + $models, $relation->initRelation($models, $name), $relation->getEager(), $name ); } + /** + * Eagerly load the relationship on a set of models with chunking support. + * + * @param array $models + * @param string $name + * @param int $chunkSize + * @return array + */ + protected function eagerLoadRelationChunked(array $models, $name, $chunkSize = 500) + { + $relation = $this->getRelation($name); + + // Instead of calling protected method directly, use addEagerConstraints + $relation->addEagerConstraints($models); + + // Get the query builder from the relation + $query = $relation->getQuery(); + + // Execute the query with chunking + $relatedModels = collect(); + $query->chunk($chunkSize, function ($chunk) use (&$relatedModels) { + $relatedModels = $relatedModels->merge($chunk); + + // Clean up memory + if (function_exists('gc_collect_cycles')) { + gc_collect_cycles(); + } + }); + + // Match the related models to their parents using public API + return $relation->match( + $models, + $relation->initRelation($models, $name), + $relatedModels, + $name + ); + } + + /** + * Eager load relation with low memory footprint. + * + * @param array $models + * @param string $name + * @param \Closure $constraints + * @param bool $useChunking + * @param int $chunkSize + * @return array + */ + protected function eagerLoadLowMemory(array $models, $name, \Closure $constraints, $useChunking = true, $chunkSize = 500) + { + if ($useChunking && count($models) > $chunkSize) { + return $this->eagerLoadRelationChunked($models, $name, $chunkSize); + } + + $relation = $this->getRelation($name); + + $relation->addEagerConstraints($models); + + call_user_func($constraints, $relation); + + return $relation->match( + $models, + $relation->initRelation($models, $name), + $relation->getEager(), + $name + ); + } + /** * Get the relation instance for the given relation name. * @@ -2238,4 +2307,49 @@ public function __clone() $onCloneCallback($this); } } + + /** + * Eager load relations with a limit on each relation to reduce memory usage. + * + * @param mixed $relations The relations to eager load + * @param array $limits Array with relation names as keys and their limits as values + * @return $this + */ + public function withLimited($relations, array $limits = []) + { + if (is_string($relations)) { + $relations = func_get_args(); + // Remove the limits array if it exists + $relations = is_array(end($relations)) ? array_slice($relations, 0, -1) : $relations; + } + + $eagerLoad = $this->parseWithRelations($relations); + $limitedEagerLoad = []; + + foreach ($eagerLoad as $name => $constraints) { + $segments = explode('.', $name); + $baseRelation = array_shift($segments); + + // Store the original constraints + $limitedEagerLoad[$name] = $constraints; + + // Apply limit if specified + if (isset($limits[$name]) && is_numeric($limits[$name])) { + $limitValue = (int) $limits[$name]; + + $originalConstraint = $constraints; + $limitedEagerLoad[$name] = function ($builder) use ($originalConstraint, $limitValue) { + // Apply the original constraints first + if ($originalConstraint instanceof Closure) { + $originalConstraint($builder); + } + + // Then apply the limit + return $builder->limit($limitValue); + }; + } + } + + return $this->with($limitedEagerLoad); + } } diff --git a/src/Illuminate/Database/Eloquent/Model.php b/src/Illuminate/Database/Eloquent/Model.php index b4c1c449d17f..13320539fc7c 100644 --- a/src/Illuminate/Database/Eloquent/Model.php +++ b/src/Illuminate/Database/Eloquent/Model.php @@ -2459,4 +2459,89 @@ public function __wakeup() $this->initializeTraits(); } + + /** + * Optimize memory usage by unloading unused relationships and attributes. + * + * @param array|string|null $relations Relations to keep, others will be unloaded + * @param array|string|null $attributes Attributes to keep, others will be unloaded + * @return $this + */ + public function optimizeMemory($relations = null, $attributes = null) + { + // Keep only specific relations if requested + if ($relations !== null) { + $relations = is_array($relations) ? $relations : [$relations]; + $currentRelations = array_keys($this->relations); + + foreach ($currentRelations as $relation) { + if (! in_array($relation, $relations)) { + unset($this->relations[$relation]); + } + } + } + + // Keep only specific attributes if requested + if ($attributes !== null) { + $attributes = is_array($attributes) ? $attributes : [$attributes]; + // Always keep primary key + $attributes[] = $this->getKeyName(); + + $currentAttributes = array_keys($this->attributes); + foreach ($currentAttributes as $attribute) { + if (! in_array($attribute, $attributes)) { + unset($this->attributes[$attribute]); + } + } + } + + // Force garbage collection + if (function_exists('gc_collect_cycles')) { + gc_collect_cycles(); + } + + return $this; + } + + /** + * Create a collection of models with memory optimization after loading. + * + * @param array $models + * @param array|string|null $keepRelations Relations to keep, others will be unloaded + * @param array|string|null $keepAttributes Attributes to keep, others will be unloaded + * @return \Illuminate\Database\Eloquent\Collection + */ + public static function optimizedCollection(array $models, $keepRelations = null, $keepAttributes = null) + { + // Use the same pattern as the model's make method for consistency + $model = new static; + $collection = $model->newCollection($models); + + // Apply memory optimization to each model + return $collection->each(function ($model) use ($keepRelations, $keepAttributes) { + $model->optimizeMemory($keepRelations, $keepAttributes); + }); + } + + /** + * Cleanup model to free memory. + * + * @return $this + */ + public function cleanup() + { + $this->relations = []; + $this->hidden = []; + $this->visible = []; + $this->appends = []; + $this->touches = []; + $this->observables = []; + + // Force garbage collection + if (function_exists('gc_collect_cycles')) { + gc_collect_cycles(); + } + + return $this; + } } diff --git a/tests/Database/DatabaseEloquentBuilderMemoryOptimizationTest.php b/tests/Database/DatabaseEloquentBuilderMemoryOptimizationTest.php new file mode 100644 index 000000000000..bc977d2ecb7e --- /dev/null +++ b/tests/Database/DatabaseEloquentBuilderMemoryOptimizationTest.php @@ -0,0 +1,242 @@ +addConnection([ + 'driver' => 'sqlite', + 'database' => ':memory:', + ]); + + $db->bootEloquent(); + $db->setAsGlobal(); + + $this->createSchema(); + } + + protected function createSchema() + { + $this->schema()->create('users', function (Blueprint $table) { + $table->increments('id'); + $table->string('email'); + $table->string('name'); + $table->string('address')->nullable(); + $table->string('phone')->nullable(); + $table->timestamps(); + }); + + $this->schema()->create('posts', function (Blueprint $table) { + $table->increments('id'); + $table->integer('user_id'); + $table->string('title'); + $table->text('content'); + $table->timestamps(); + }); + + $this->schema()->create('comments', function (Blueprint $table) { + $table->increments('id'); + $table->integer('post_id'); + $table->string('body'); + $table->timestamps(); + }); + } + + /** + * Tear down the database schema. + * + * @return void + */ + protected function tearDown(): void + { + parent::tearDown(); + + $this->schema()->drop('users'); + $this->schema()->drop('posts'); + $this->schema()->drop('comments'); + } + + /** + * Get the schema builder instance. + * + * @return \Illuminate\Database\Schema\Builder + */ + protected function schema() + { + return DB::connection()->getSchemaBuilder(); + } + + public function testChunkByMemoryAutomaticallyAdjustsChunkSize() + { + // Create test data + for ($i = 1; $i <= 100; $i++) { + TestUserModel::create([ + 'email' => "user{$i}@example.com", + 'name' => "User {$i}", + 'address' => "Address {$i}", + 'phone' => "555-000-{$i}", + ]); + } + + $iterations = 0; + $totalProcessed = 0; + + $initialMemory = memory_get_usage(); + + TestUserModel::query()->chunkByMemory(10, function ($users, $page) use (&$iterations, &$totalProcessed) { + $iterations++; + $totalProcessed += count($users); + // Simulate some processing + $users->map(function ($user) { + return $user->name.' - '.$user->email; + })->implode(', '); + }); + + $this->assertEquals(100, $totalProcessed); + $this->assertLessThan(10, $iterations); // Should be optimized to fewer than 10 iterations + + // Verify memory usage is controlled + $finalMemory = memory_get_usage(); + $this->assertLessThan($initialMemory * 2, $finalMemory); + } + + public function testLazyByMemoryCreatesLazyCollection() + { + // Create test data + for ($i = 1; $i <= 100; $i++) { + TestUserModel::create([ + 'email' => "user{$i}@example.com", + 'name' => "User {$i}", + ]); + } + + $collection = TestUserModel::query()->lazyByMemory(10); + + $this->assertInstanceOf(LazyCollection::class, $collection); + $this->assertEquals(100, $collection->count()); + } + + public function testModelMemoryOptimization() + { + $user = TestUserModel::create([ + 'email' => 'test@example.com', + 'name' => 'Test User', + 'address' => '123 Test St', + 'phone' => '555-1234', + ]); + + // Create related posts + for ($i = 1; $i <= 5; $i++) { + $post = $user->posts()->create([ + 'title' => "Post {$i}", + 'content' => "Content {$i}", + ]); + + // Create comments for each post + for ($j = 1; $j <= 3; $j++) { + $post->comments()->create([ + 'body' => "Comment {$j} on Post {$i}", + ]); + } + } + + // Load the user with all relations + $userWithRelations = TestUserModel::with('posts.comments')->find($user->id); + + // Verify all relations are loaded + $this->assertTrue($userWithRelations->relationLoaded('posts')); + $this->assertTrue($userWithRelations->posts->first()->relationLoaded('comments')); + + // Get list of attributes before optimization + $attributesBeforeOptimize = array_keys($userWithRelations->getAttributes()); + $this->assertContains('address', $attributesBeforeOptimize); + $this->assertContains('phone', $attributesBeforeOptimize); + + // Optimize the user model memory (keep only specific attributes and relations) + $userWithRelations->optimizeMemory(['posts'], ['id', 'email', 'name']); + + // The posts relation should still be loaded + $this->assertTrue($userWithRelations->relationLoaded('posts')); + + // Check that only the specified attributes are kept + $attributesAfterOptimize = array_keys($userWithRelations->getAttributes()); + $this->assertContains('id', $attributesAfterOptimize); + $this->assertContains('email', $attributesAfterOptimize); + $this->assertContains('name', $attributesAfterOptimize); + $this->assertNotContains('address', $attributesAfterOptimize); + $this->assertNotContains('phone', $attributesAfterOptimize); + } + + public function testWithLimitedRelations() + { + // Create test data + $user = TestUserModel::create([ + 'email' => 'test@example.com', + 'name' => 'Test User', + ]); + + // Create a large number of posts + for ($i = 1; $i <= 20; $i++) { + $user->posts()->create([ + 'title' => "Post {$i}", + 'content' => "Content {$i}", + ]); + } + + // Test withLimited to load only a subset of posts + $userWithLimitedPosts = TestUserModel::withLimited('posts', ['posts' => 5])->find($user->id); + + $this->assertCount(5, $userWithLimitedPosts->posts); + } +} + +class TestUserModel extends Model +{ + protected $table = 'users'; + protected $fillable = ['email', 'name', 'address', 'phone']; + + public function posts() + { + return $this->hasMany(TestPostModel::class, 'user_id'); + } +} + +class TestPostModel extends Model +{ + protected $table = 'posts'; + protected $fillable = ['user_id', 'title', 'content']; + + public function user() + { + return $this->belongsTo(TestUserModel::class, 'user_id'); + } + + public function comments() + { + return $this->hasMany(TestCommentModel::class, 'post_id'); + } +} + +class TestCommentModel extends Model +{ + protected $table = 'comments'; + protected $fillable = ['post_id', 'body']; + + public function post() + { + return $this->belongsTo(TestPostModel::class, 'post_id'); + } +}