Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: exclude system collection in getAllCollection #3321

Closed
wants to merge 4 commits into from
+227 −40
Closed

feat: exclude system collection in getAllCollection #3321

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
09c864f
feat: exclude system collection in getAllCollection
masterbater Mar 12, 2025
f2e6358
feat: exclude system collections in getTable and getTableListing
masterbater Mar 24, 2025
02d4146
feat: getTable skip if collection type is view
masterbater Mar 24, 2025
837ace4
feat: use 5.2 Builder as basis
masterbater Mar 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 153 additions & 40 deletions src/Schema/Builder.php
View file
Original file line number Diff line number Diff line change
@@ -5,27 +5,41 @@
namespace MongoDB\Laravel\Schema;

use Closure;
use MongoDB\Collection;
use MongoDB\Driver\Exception\ServerException;
use MongoDB\Laravel\Connection;
use MongoDB\Model\CollectionInfo;
use MongoDB\Model\IndexInfo;

use function array_column;
use function array_fill_keys;
use function array_filter;
use function array_keys;
use function array_map;
use function array_merge;
use function array_values;
use function assert;
use function count;
use function current;
use function implode;
use function in_array;
use function is_array;
use function is_string;
use function iterator_to_array;
use function sort;
use function sprintf;
use function str_ends_with;
use function substr;
use function usort;

/** @property Connection $connection */
class Builder extends \Illuminate\Database\Schema\Builder
{
/**
* Check if column exists in the collection schema.
*
* @param string $table
* @param string $column
* @param string $table
* @param string $column
*/
public function hasColumn($table, $column): bool
{
@@ -35,11 +49,21 @@ public function hasColumn($table, $column): bool
/**
* Check if columns exists in the collection schema.
*
* @param string $table
* @param string[] $columns
* @param string $table
* @param string[] $columns
*/
public function hasColumns($table, array $columns): bool
{
// The field "id" (alias of "_id") always exists in MongoDB documents
$columns = array_filter($columns, fn (string $column): bool => ! in_array($column, ['_id', 'id'], true));

// Any subfield named "*.id" is an alias of "*._id"
$columns = array_map(fn (string $column): string => str_ends_with($column, '.id') ? substr($column, 0, -3).'._id' : $column, $columns);

if ($columns === []) {
return true;
}

$collection = $this->connection->table($table);

return $collection
@@ -51,13 +75,12 @@ public function hasColumns($table, array $columns): bool
/**
* Determine if the given collection exists.
*
* @param string $name
*
* @param string $name
* @return bool
*/
public function hasCollection($name)
{
$db = $this->connection->getMongoDB();
$db = $this->connection->getDatabase();

$collections = iterator_to_array($db->listCollections([
'filter' => ['name' => $name],
@@ -66,13 +89,13 @@ public function hasCollection($name)
return count($collections) !== 0;
}

/** @inheritdoc */
/** {@inheritdoc} */
public function hasTable($table)
{
return $this->hasCollection($table);
}

/** @inheritdoc */
/** {@inheritdoc} */
public function table($table, Closure $callback)
{
$blueprint = $this->createBlueprint($table);
@@ -82,7 +105,7 @@ public function table($table, Closure $callback)
}
}

/** @inheritdoc */
/** {@inheritdoc} */
public function create($table, ?Closure $callback = null, array $options = [])
{
$blueprint = $this->createBlueprint($table);
@@ -94,61 +117,98 @@ public function create($table, ?Closure $callback = null, array $options = [])
}
}

/** @inheritdoc */
/** {@inheritdoc} */
public function dropIfExists($table)
{
if ($this->hasCollection($table)) {
$this->drop($table);
}
}

/** @inheritdoc */
/** {@inheritdoc} */
public function drop($table)
{
$blueprint = $this->createBlueprint($table);

$blueprint->drop();
}

/** @inheritdoc */
/** {@inheritdoc} */
public function dropAllTables()
{
foreach ($this->getAllCollections() as $collection) {
$this->drop($collection);
}
}

public function getTables()
/**
* @param string|null $schema Database name
*/
public function getTables($schema = null)
{
$db = $this->connection->getMongoDB();
$db = $this->connection->getDatabase($schema);
$collections = [];

foreach ($db->listCollectionNames() as $collectionName) {
$stats = $db->selectCollection($collectionName)->aggregate([
['$collStats' => ['storageStats' => ['scale' => 1]]],
['$project' => ['storageStats.totalSize' => 1]],
])->toArray();
foreach ($db->listCollections() as $collectionInfo) {
$collectionName = $collectionInfo->getName();

// Skip system collections
if (str_starts_with($collectionName, 'system.')) {
continue;
}
// Skip views it doesnt suport aggregate
$isView = ($collectionInfo['type'] ?? '') === 'view';
$stats = null;

if (! $isView) {
// Only run aggregation if it's a normal collection
$stats = $db->selectCollection($collectionName)->aggregate([
['$collStats' => ['storageStats' => ['scale' => 1]]],
['$project' => ['storageStats.totalSize' => 1]],
])->toArray();
}

$collections[] = [
'name' => $collectionName,
'schema' => null,
'schema' => $db->getDatabaseName(),
'schema_qualified_name' => $db->getDatabaseName().'.'.$collectionName,
'size' => $stats[0]?->storageStats?->totalSize ?? null,
'comment' => null,
'collation' => null,
'engine' => null,
'engine' => $isView ? 'view' : 'collection',
];
}

usort($collections, function ($a, $b) {
return $a['name'] <=> $b['name'];
});
usort($collections, fn ($a, $b) => $a['name'] <=> $b['name']);

return $collections;
}

public function getTableListing()
/**
* @param string|null $schema
* @param bool $schemaQualified If a schema is provided, prefix the collection names with the schema name
* @return array
*/
public function getTableListing($schema = null, $schemaQualified = false)
{
$collections = iterator_to_array($this->connection->getMongoDB()->listCollectionNames());
$collections = [];

if ($schema === null || is_string($schema)) {
$collections[$schema ?? 0] = iterator_to_array($this->connection->getDatabase($schema)->listCollectionNames());
} elseif (is_array($schema)) {
foreach ($schema as $db) {
$collections[$db] = iterator_to_array($this->connection->getDatabase($db)->listCollectionNames());
}
}

if ($schema && $schemaQualified) {
$collections = array_map(fn ($db, $collections) => array_map(static fn ($collection) => $db.'.'.$collection, $collections), array_keys($collections), $collections);
}

$collections = array_merge(...array_values($collections));

// Exclude system collections before sorting
$collections = array_filter($collections, fn ($name) => ! str_starts_with($name, 'system.'));

sort($collections);

@@ -157,7 +217,7 @@ public function getTableListing()

public function getColumns($table)
{
$stats = $this->connection->getMongoDB()->selectCollection($table)->aggregate([
$stats = $this->connection->getDatabase()->selectCollection($table)->aggregate([
// Sample 1,000 documents to get a representative sample of the collection
['$sample' => ['size' => 1_000]],
// Convert each document to an array of fields
@@ -187,16 +247,21 @@ public function getColumns($table)
foreach ($stats as $stat) {
sort($stat->types);
$type = implode(', ', $stat->types);
$name = $stat->_id;
if ($name === '_id') {
$name = 'id';
}

$columns[] = [
'name' => $stat->_id,
'name' => $name,
'type_name' => $type,
'type' => $type,
'collation' => null,
'nullable' => $stat->_id !== '_id',
'nullable' => $name !== 'id',
'default' => null,
'auto_increment' => false,
'comment' => sprintf('%d occurrences', $stat->total),
'generation' => $stat->_id === '_id' ? ['type' => 'objectId', 'expression' => null] : null,
'generation' => $name === 'id' ? ['type' => 'objectId', 'expression' => null] : null,
];
}

@@ -205,9 +270,11 @@ public function getColumns($table)

public function getIndexes($table)
{
$indexes = $this->connection->getMongoDB()->selectCollection($table)->listIndexes();

$collection = $this->connection->getDatabase()->selectCollection($table);
assert($collection instanceof Collection);
$indexList = [];

$indexes = $collection->listIndexes();
foreach ($indexes as $index) {
assert($index instanceof IndexInfo);
$indexList[] = [
@@ -218,12 +285,40 @@ public function getIndexes($table)
$index->isText() => 'text',
$index->is2dSphere() => '2dsphere',
$index->isTtl() => 'ttl',
default => 'default',
default => null,
},
'unique' => $index->isUnique(),
];
}

try {
$indexes = $collection->listSearchIndexes(['typeMap' => ['root' => 'array', 'array' => 'array', 'document' => 'array']]);
foreach ($indexes as $index) {
// Status 'DOES_NOT_EXIST' means the index has been dropped but is still in the process of being removed
if ($index['status'] === 'DOES_NOT_EXIST') {
continue;
}

$indexList[] = [
'name' => $index['name'],
'columns' => match ($index['type']) {
'search' => array_merge(
$index['latestDefinition']['mappings']['dynamic'] ? ['dynamic'] : [],
array_keys($index['latestDefinition']['mappings']['fields'] ?? []),
),
'vectorSearch' => array_column($index['latestDefinition']['fields'], 'path'),
},
'type' => $index['type'],
'primary' => false,
'unique' => false,
];
}
} catch (ServerException $exception) {
if (! self::isAtlasSearchNotSupportedException($exception)) {
throw $exception;
}
}

return $indexList;
}

@@ -232,7 +327,7 @@ public function getForeignKeys($table)
return [];
}

/** @inheritdoc */
/** {@inheritdoc} */
protected function createBlueprint($table, ?Closure $callback = null)
{
return new Blueprint($this->connection, $table);
@@ -241,13 +336,12 @@ protected function createBlueprint($table, ?Closure $callback = null)
/**
* Get collection.
*
* @param string $name
*
* @param string $name
* @return bool|CollectionInfo
*/
public function getCollection($name)
{
$db = $this->connection->getMongoDB();
$db = $this->connection->getDatabase();

$collections = iterator_to_array($db->listCollections([
'filter' => ['name' => $name],
@@ -264,10 +358,29 @@ public function getCollection($name)
protected function getAllCollections()
{
$collections = [];
foreach ($this->connection->getMongoDB()->listCollections() as $collection) {
$collections[] = $collection->getName();
foreach ($this->connection->getDatabase()->listCollections() as $collection) {
$name = $collection->getName();

// Skip system collections
if (str_starts_with($name, 'system.')) {
continue;
}

$collections[] = $name;
}

return $collections;
}

/** @internal */
public static function isAtlasSearchNotSupportedException(ServerException $e): bool
{
return in_array($e->getCode(), [
59, // MongoDB 4 to 6, 7-community: no such command: 'createSearchIndexes'
40324, // MongoDB 4 to 6: Unrecognized pipeline stage name: '$listSearchIndexes'
115, // MongoDB 7-ent: Search index commands are only supported with Atlas.
6047401, // MongoDB 7: $listSearchIndexes stage is only allowed on MongoDB Atlas
31082, // MongoDB 8: Using Atlas Search Database Commands and the $listSearchIndexes aggregation stage requires additional configuration.
], true);
}
}
74 changes: 74 additions & 0 deletions tests/SchemaTest.php
View file
Original file line number Diff line number Diff line change
@@ -19,6 +19,9 @@ public function tearDown(): void
{
Schema::drop('newcollection');
Schema::drop('newcollection_two');
// View type
Schema::drop('test_view');

}

public function testCreate(): void
@@ -397,6 +400,13 @@ public function testGetTables()
DB::connection('mongodb')->table('newcollection')->insert(['test' => 'value']);
DB::connection('mongodb')->table('newcollection_two')->insert(['test' => 'value']);

// Create a view (this creates system.views)
DB::connection('mongodb')->getDatabase()->command([
'create' => 'test_view',
'viewOn' => 'newcollection',
'pipeline' => []
]);

$tables = Schema::getTables();
$this->assertIsArray($tables);
$this->assertGreaterThanOrEqual(2, count($tables));
@@ -409,6 +419,8 @@ public function testGetTables()
$this->assertEquals(8192, $table['size']);
$found = true;
}
// Ensure system collections are excluded
$this->assertFalse(str_starts_with($table['name'], 'system.'));
}

if (! $found) {
@@ -421,12 +433,74 @@ public function testGetTableListing()
DB::connection('mongodb')->table('newcollection')->insert(['test' => 'value']);
DB::connection('mongodb')->table('newcollection_two')->insert(['test' => 'value']);

// Create a view (this creates system.views)
DB::connection('mongodb')->getDatabase()->command([
'create' => 'test_view',
'viewOn' => 'newcollection',
'pipeline' => []
]);

$tables = Schema::getTableListing();

$this->assertIsArray($tables);
$this->assertGreaterThanOrEqual(2, count($tables));
$this->assertContains('newcollection', $tables);
$this->assertContains('newcollection_two', $tables);

// Ensure system collections are excluded
$this->assertNotContains('system.views', $tables);
}
public function testGetAllCollections()
{

DB::connection('mongodb')->table('newcollection')->insert(['test' => 'value']);
DB::connection('mongodb')->table('newcollection_two')->insert(['test' => 'value']);

// Create a view (this creates system.views)
DB::connection('mongodb')->getDatabase()->command([
'create' => 'test_view',
'viewOn' => 'newcollection',
'pipeline' => []
]);

$collections = Schema::getAllCollections();

$this->assertIsArray($collections);
$this->assertGreaterThanOrEqual(2, count($collections));


$this->assertContains('newcollection', $collections);
$this->assertContains('newcollection_two', $collections);

// Ensure system collections are excluded
$this->assertNotContains('system.views', $collections);
}

public function testSystemCollectionsArePresentButFiltered()
{

// Create a view to trigger system.views collection
DB::connection('mongodb')->getDatabase()->command([
'create' => 'test_view',
'viewOn' => 'newcollection',
'pipeline' => []
]);

// Get all collections directly from MongoDB
$allCollections = $db->getDatabase()->listCollectionNames();

// Ensure the system.views collection exists in MongoDB
$this->assertContains('system.views', $allCollections);

// Ensure Schema::getTables does NOT include system collections
$tables = Schema::getTables();
foreach ($tables as $table) {
$this->assertFalse(str_starts_with($table['name'], 'system.'));
}

// Ensure Schema::getTableListing does NOT include system collections
$tableListing = Schema::getTableListing();
$this->assertNotContains('system.views', $tableListing);
}

public function testGetColumns()