diff --git a/db/migrations/20240915221912_software_inventory.php b/db/migrations/20240915221912_software_inventory.php new file mode 100644 index 000000000..ba6a9b0d3 --- /dev/null +++ b/db/migrations/20240915221912_software_inventory.php @@ -0,0 +1,29 @@ +isMigratingUp()) { + // Add new settings + $this->execute("INSERT INTO `Instance_Settings`(`IS_ID`, `IS_Name`) VALUES (14, 'Software Assets Snapshot')"); + $this->execute("INSERT INTO `Instance_Settings`(`IS_ID`, `IS_Name`) VALUES (15, 'Software Assets Snapshot Days Duration')"); + + // Dont enable software asset monitoring by default & clear software asset management monitoring logs after 7 days + $this->execute("INSERT INTO `Instance_Settings_Values`(`ISV_IS_ID`, `ISV_Value`) VALUES (14, '0')"); + $this->execute("INSERT INTO `Instance_Settings_Values`(`ISV_IS_ID`, `ISV_Value`) VALUES (15, '7')"); + + // Create table to store software assets monitor results + $this->table('Software_Assets_Snapshots', ['id' => "SAS_ID", 'primary_key' => ["SAS_ID"]]) + ->addColumn('SAS_Last_Updated', 'datetime', ['default' => 'CURRENT_TIMESTAMP']) + ->addColumn('SAS_Date', 'date', ['null' => false]) + ->addColumn('SAS_Data', 'json', ['null' => false]) + ->addIndex(['SAS_Date'], ['unique' => true, 'name' => 'unique_software_assets_monitor']) + ->create(); + } + } +} diff --git a/src/classes/Constants/InstanceSettingsKeys.php b/src/classes/Constants/InstanceSettingsKeys.php index 4d1c8d4d7..8cdc94ec9 100644 --- a/src/classes/Constants/InstanceSettingsKeys.php +++ b/src/classes/Constants/InstanceSettingsKeys.php @@ -16,4 +16,6 @@ class InstanceSettingsKeys const TIMEZONE = 11; const BACKUP_HISTORY = 12; const INSTANCE_METRIC_HISTORY = 13; + const SOFTWARE_INVENTORY_MONITOR = 14; + const SOFTWARE_INVENTORY_MONITOR_DAYS_DURATION = 15; } diff --git a/src/classes/Controllers/Hosts/SoftwareAssets/GetSnapshotSoftwareListController.php b/src/classes/Controllers/Hosts/SoftwareAssets/GetSnapshotSoftwareListController.php new file mode 100755 index 000000000..2c247e0f2 --- /dev/null +++ b/src/classes/Controllers/Hosts/SoftwareAssets/GetSnapshotSoftwareListController.php @@ -0,0 +1,30 @@ +fetchUserDetails = $fetchUserDetails; + $this->getSnapshotSoftwareList = $getSnapshotSoftwareList; + } + + public function get(int $userId, string $date) + { + $isAdmin = $this->fetchUserDetails->isAdmin($userId) === '1'; + if (!$isAdmin) { + throw new \Exception("No access", 1); + } + $date = new \DateTimeImmutable($date); + return $this->getSnapshotSoftwareList->get($date); + } +} diff --git a/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAssetsOverviewController.php b/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAssetsOverviewController.php new file mode 100755 index 000000000..d2535cc8b --- /dev/null +++ b/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAssetsOverviewController.php @@ -0,0 +1,30 @@ +fetchUserDetails = $fetchUserDetails; + $this->getSoftwareSnapshotOverview = $getSoftwareSnapshotOverview; + } + + public function get(int $userId, string $date) + { + $isAdmin = $this->fetchUserDetails->isAdmin($userId) === '1'; + if (!$isAdmin) { + throw new \Exception("No access", 1); + } + $date = new \DateTimeImmutable($date); + return $this->getSoftwareSnapshotOverview->get($date); + } +} diff --git a/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAsssetsHeadersController.php b/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAsssetsHeadersController.php new file mode 100755 index 000000000..e4da0a4ed --- /dev/null +++ b/src/classes/Controllers/Hosts/SoftwareAssets/GetSoftwareAsssetsHeadersController.php @@ -0,0 +1,29 @@ +fetchUserDetails = $fetchUserDetails; + $this->fetchSoftwareAssetSnapshots = $fetchSoftwareAssetSnapshots; + } + + public function get(int $userId) + { + $isAdmin = $this->fetchUserDetails->isAdmin($userId) === '1'; + if (!$isAdmin) { + throw new \Exception("No access", 1); + } + return $this->fetchSoftwareAssetSnapshots->fetchLastSevenHeaders(); + } +} diff --git a/src/classes/Model/Hosts/GetDetails.php b/src/classes/Model/Hosts/GetDetails.php index 82275fc12..a4fee4677 100644 --- a/src/classes/Model/Hosts/GetDetails.php +++ b/src/classes/Model/Hosts/GetDetails.php @@ -48,6 +48,22 @@ public function fetchAlias(int $hostId) return $do->fetchColumn(); } + public function fetchAliases(array $hostIds) + { + $qMarks = join(',', array_fill(0, count($hostIds), '?')); + $sql = "SELECT + `Host_ID`, + COALESCE(`Host_Alias`, `Host_Url_And_Port`) + FROM + `Hosts` + WHERE + `Host_ID` IN ($qMarks) + "; + $do = $this->database->prepare($sql); + $do->execute($hostIds); + return $do->fetchAll(\PDO::FETCH_KEY_PAIR); + } + public function fetchHost($hostId) { $sql = "SELECT diff --git a/src/classes/Model/Hosts/SoftwareAssets/FetchSoftwareAssetSnapshots.php b/src/classes/Model/Hosts/SoftwareAssets/FetchSoftwareAssetSnapshots.php new file mode 100644 index 000000000..ef3f3fcde --- /dev/null +++ b/src/classes/Model/Hosts/SoftwareAssets/FetchSoftwareAssetSnapshots.php @@ -0,0 +1,46 @@ +database = $database->dbObject; + } + + public function fetchForDate(\DateTimeImmutable $date) + { + $sql = "SELECT + JSON_UNQUOTE(`SAS_Data`) as `data` + FROM + `Software_Assets_Snapshots` + WHERE + `SAS_Date` = :date + "; + $do = $this->database->prepare($sql); + $do->execute([ + ":date" => $date->format("Y-m-d") + ]); + return $do->fetch(\PDO::FETCH_ASSOC); + } + + public function fetchLastSevenHeaders() + { + $sql = "SELECT + `SAS_ID` as `id`, + `SAS_Date` as `date`, + `SAS_Last_Updated` as `lastUpdate` + FROM + `Software_Assets_Snapshots` + ORDER BY + `SAS_Date` DESC + LIMIT 7 + "; + return $this->database->query($sql)->fetchAll(\PDO::FETCH_ASSOC); + } +} diff --git a/src/classes/Model/Hosts/SoftwareAssets/InsertSoftwareAssetsSnapshot.php b/src/classes/Model/Hosts/SoftwareAssets/InsertSoftwareAssetsSnapshot.php new file mode 100644 index 000000000..a71bbca1d --- /dev/null +++ b/src/classes/Model/Hosts/SoftwareAssets/InsertSoftwareAssetsSnapshot.php @@ -0,0 +1,41 @@ +database = $database->dbObject; + } + + public function insert(\DateTimeImmutable $date, array $data) + { + $sql = "INSERT INTO `Software_Assets_Snapshots` + ( + `SAS_Last_Updated`, + `SAS_Date`, + `SAS_Data` + ) VALUES ( + CURRENT_TIMESTAMP(), + :date, + :data + ) ON DUPLICATE KEY UPDATE + `SAS_Last_Updated` = CURRENT_TIMESTAMP(), + `SAS_Data` = :data; + "; + $do = $this->database->prepare($sql); + $do->execute([ + ":date"=>$date->format("Y-m-d"), + ":data"=>json_encode($data), + ]); + return $do->rowCount() ? true : false; + } +} + + + diff --git a/src/classes/Tools/Hosts/SoftwareAssets/GetSnapshotSoftwareList.php b/src/classes/Tools/Hosts/SoftwareAssets/GetSnapshotSoftwareList.php new file mode 100644 index 000000000..84616570d --- /dev/null +++ b/src/classes/Tools/Hosts/SoftwareAssets/GetSnapshotSoftwareList.php @@ -0,0 +1,54 @@ +fetchSoftwareAssetSnapshots = $fetchSoftwareAssetSnapshots; + $this->getDetails = $getDetails; + } + + public function get(\DateTimeImmutable $date) + { + $snapshot = $this->fetchSoftwareAssetSnapshots->fetchForDate($date); + + if (empty($snapshot)) { + return []; + } + + $snapshot = json_decode($snapshot["data"], true); + + $output = []; + + if (empty($snapshot)) { + return $output; + } + + $hostAliases = $this->getDetails->fetchAliases(array_keys($snapshot)); + + foreach ($snapshot as $hostId => $projects) { + foreach ($projects as $project => $instances) { + foreach ($instances as $instance => $packages) { + foreach ($packages as $package) { + $output[] = array_merge([ + "hostName" => $hostAliases[$hostId], + "project" => $project, + "instance" => $instance + ], $package); + } + } + } + } + return $output; + } +} diff --git a/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareAssetsSnapshotData.php b/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareAssetsSnapshotData.php new file mode 100644 index 000000000..0c2a8c7ba --- /dev/null +++ b/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareAssetsSnapshotData.php @@ -0,0 +1,301 @@ + "aptParse", + // Alpine + "apk info" => "apkParse", + // General package managers + "snap list" => "snapParse", + "npm list -g -l --parseable" => "npmParse", + "pip list --format=columns" => "pipParse", + "composer g show" => "composerParse" + ]; + + public function __construct( + HostList $hostList, + HasExtension $hasExtension + ) { + $this->hostList = $hostList; + $this->hasExtension = $hasExtension; + } + + public function get() + { + $hosts = $this->hostList->getOnlineHostsWithDetails(); + + $output = []; + + foreach ($hosts as $host) { + $supportsProjects = $this->hasExtension->checkWithHost($host, "projects"); + $output[$host->getHostId()] = []; + $allProjects = [["name" => "default", "config" => []]]; + + if ($supportsProjects) { + $allProjects = $host->projects->all(2); + } + + foreach ($allProjects as $project) { + $projectName = $project["name"]; + + $output[$host->getHostId()][$projectName] = []; + + + $lxd = $host->host->info()["environment"]["server"] === "lxd"; + + // See https://github.com/turtle0x1/LxdMosaic/issues/576#issuecomment-2351794173 + // to see when LXD VM's are supported + $supportsVms = $lxd ? false : true; + + if ($supportsProjects) { + $host->setProject($projectName); + } + + $instances = $host->instances->all(1); + foreach ($instances as $instance) { + if ($instance["status_code"] !== "103") { + continue; + } else if ($instance["type"] == "virtual-machine" && !$supportsVms) { + continue; + } + $instacePackages = []; + foreach ($this->commandMap as $command => $fn) { + try { + $result = $host->instances->execute($instance["name"], $command, true, [], true); + $result = $host->instances->logs->read( + $instance["name"], + $result["output"][0] + ); + $parsedPackages = call_user_func([$this, $fn], $result); + $instacePackages = array_merge($instacePackages, $parsedPackages); + $output[$host->getHostId()][$projectName][$instance["name"]] = $instacePackages; + } catch (\Throwable $th) { + continue; + } + } + } + } + } + return $output; + } + + public function aptParse($result) + { + $lines = explode("\n", trim($result)); + $packages = []; + + foreach ($lines as $line) { + // Use regex to extract package details + if (preg_match('/^(?P[\w\.\-\/]+),now (?P[\w\:\-\.]+) (?P\w+) \[(?P[^\]]+)\]$/', $line, $matches)) { + $packages[] = [ + 'manager' => 'apt', + 'name' => $matches['name'], + 'version' => $matches['version'], + 'rev' => null, + 'tracking' => null, + 'publisher' => null, + 'notes' => null, + 'architecture' => $matches['architecture'], + 'status' => explode(',', $matches['status']), + ]; + } + } + + return $packages; + } + + public function snapParse($data) + { + // Split the input data into lines + $lines = explode("\n", trim($data)); + + // Remove the header line + array_shift($lines); + + $packages = []; + + foreach ($lines as $line) { + // Use regex to extract fields + if (preg_match('/^(?P[\w\-]+)\s+(?P[\w\.\-]+)\s+(?P\d+)\s+(?P[\w\/\-\.]+)\s+(?P[\w\.\-✓✪]+)\s+(?P.*)$/', $line, $matches)) { + $packages[] = [ + 'manager' => 'SNAP', + 'name' => trim($matches['name']), + 'version' => trim($matches['version']), + 'rev' => trim($matches['rev']), + 'tracking' => trim($matches['tracking']), + 'publisher' => trim($matches['publisher']), + 'notes' => trim($matches['notes']), + 'architecture' => null, + 'status' => 'installed', + ]; + } + } + + return $packages; + } + + public function apkParse($data) + { + $lines = explode("\n", $data); + array_shift($lines); // Remove the first line + + $packages = []; + foreach ($lines as $line) { + $line = trim($line); + if (!empty($line)) { + // Split the package and version using regex + if (preg_match('/^(.*?)-(.+)$/', $line, $matches)) { + $packageName = $matches[1]; // Package name + $version = $matches[2]; // Version + $packages[] = [ + 'manager' => 'apk', + 'name' => $packageName, + 'version' => $version, + 'rev' => null, + 'tracking' => null, + 'publisher' => null, + 'notes' => null, + 'architecture' => null, + 'status' => 'installed', + ]; + } + } + } + return $packages; + } + + public function npmParse($data) + { + // Split the output into lines + $lines = explode("\n", trim($data)); + + // Initialize an array to hold package details + $packages = []; + + // Loop through each line to extract package information + foreach ($lines as $line) { + // Skip empty lines + if (empty($line)) { + continue; + } + + // Split the line into parts + $parts = explode(':', $line); + + // Get the package path and name/version + $packagePath = $parts[0]; + $packageInfo = end($parts); // Get the last part which is the package name and version + + $packageParts = explode('@', $packageInfo); + $packageName = $packageParts[0] ?? null; + $packageVersion = $packageParts[1] ?? null; + + if($packageName == null){ + continue; + } + + // Fill the array with the specified keys + $packages[] = [ + 'manager' => 'npm', + 'name' => $packageName, + 'version' => $packageVersion, + 'rev' => null, // Placeholder for revision, if applicable + 'tracking' => null, // Placeholder for tracking, if applicable + 'publisher' => null, // Placeholder for publisher, if applicable + 'notes' => null, // Placeholder for notes, if applicable + 'architecture' => null, // Placeholder for architecture, if applicable + 'status' => 'installed' // Default status + ]; + } + return $packages; + } + + public function pipParse($data) + { + // Split the output into lines + $lines = explode("\n", trim($data)); + + // Initialize an array to hold package details + $packages = []; + + // Loop through each line to extract package information + foreach ($lines as $line) { + // Skip the header and empty lines + if (strpos($line, 'Package') !== false || empty(trim($line))) { + continue; + } + + // Split the line into parts based on whitespace + $parts = preg_split('/\s+/', trim($line)); + + // Extract package name and version + $packageName = $parts[0]; + $packageVersion = $parts[1]; + + // Fill the array with the specified keys + $packages[] = [ + 'manager' => 'pip', + 'name' => $packageName, + 'version' => $packageVersion, + 'rev' => null, // Placeholder for revision, if applicable + 'tracking' => null, // Placeholder for tracking, if applicable + 'publisher' => null, // Placeholder for publisher, if applicable + 'notes' => null, // Placeholder for notes, if applicable + 'architecture' => null, // Placeholder for architecture, if applicable + 'status' => 'installed' // Default status + ]; + } + return $packages; + } + + public function composerParse($data) + { + // Split the output into data + $lines = explode("\n", trim($data)); + + // Initialize an array to hold package details + $packages = []; + + // Loop through each line to extract package information + foreach ($lines as $line) { + // Skip the header and empty lines + if (empty(trim($line)) || strpos($line, 'Changed current directory') !== false) { + continue; + } + + // Split the line into parts based on whitespace + $parts = preg_split('/\s+/', trim($line), 3); // Limit to 3 parts to separate name/version from description + + // Extract package name, version, and description + if (count($parts) >= 2) { + $packageName = $parts[0]; + $packageVersion = $parts[1]; + $packageDescription = isset($parts[2]) ? $parts[2] : ''; + + // Fill the array with the specified keys + $packages[] = [ + 'manager' => 'composer', + 'name' => $packageName, + 'version' => $packageVersion, + 'rev' => null, // Placeholder for revision, if applicable + 'tracking' => null, // Placeholder for tracking, if applicable + 'publisher' => null, // Placeholder for publisher, if applicable + 'notes' => $packageDescription, // Use description as notes + 'architecture' => null, // Placeholder for architecture, if applicable + 'status' => 'installed' // Default status + ]; + } + } + return $packages; + } +} diff --git a/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareSnapshotOverview.php b/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareSnapshotOverview.php new file mode 100644 index 000000000..837ce254e --- /dev/null +++ b/src/classes/Tools/Hosts/SoftwareAssets/GetSoftwareSnapshotOverview.php @@ -0,0 +1,121 @@ +fetchSoftwareAssetSnapshots = $fetchSoftwareAssetSnapshots; + $this->getDetails = $getDetails; + } + + public function get(\DateTimeImmutable $date = null) + { + if ($date == null) { + $date = new \DateTimeImmutable(); + } + + $snapshot = $this->fetchSoftwareAssetSnapshots->fetchForDate($date); + + $output = [ + "date" => $date->format("Y-m-d"), + "totalPackages" => 0, + "managerMetrics" => [], + "hostMetrics" => [], + "projectMetrics" => [], + "packages" => [] + ]; + + if ($snapshot == false) { + return $output; + } + + $snapshot = json_decode($snapshot["data"], true); + + if (empty($snapshot)) { + return $output; + } + + $hostAliases = $this->getDetails->fetchAliases(array_keys($snapshot)); + + foreach ($snapshot as $hostId => $projects) { + foreach ($projects as $project => $instances) { + foreach ($instances as $instance => $packages) { + foreach ($packages as $package) { + + $output["totalPackages"]++; + + $manager = $package["manager"]; + if (!isset($output["managerMetrics"][$manager])) { + $output["managerMetrics"][$manager] = [ + "name" => $manager, + "packages" => 0 + ]; + } + $output["managerMetrics"][$manager]["packages"]++; + + if (!isset($output["hostMetrics"][$hostId])) { + $output["hostMetrics"][$hostId] = [ + "name" => $hostAliases[$hostId], + "packages" => 0, + ]; + } + $output["hostMetrics"][$hostId]["packages"]++; + + if (!isset($output["projectMetrics"][$project])) { + $output["projectMetrics"][$project] = [ + "name" => $project, + "packages" => 0, + ]; + } + $output["projectMetrics"][$project]["packages"]++; + + $packageKey = $package["name"]; + + if (!isset($output["packages"][$packageKey])) { + $output["packages"][$packageKey] = [ + "name" => $package["name"], + "totalInstalls" => 0, + "versions" => [] + ]; + } + + $output["packages"][$packageKey]["totalInstalls"]++; + + if (!isset($output["packages"][$packageKey]["versions"][$package["version"]])) { + $output["packages"][$packageKey]["versions"][$package["version"]] = [ + "version" => $package["version"], + "installs" => 0 + ]; + } + $output["packages"][$packageKey]["versions"][$package["version"]]["installs"]++; + } + } + } + } + $output["managerMetrics"] = array_values($output["managerMetrics"]); + $output["hostMetrics"] = array_values($output["hostMetrics"]); + $output["projectMetrics"] = array_values($output["projectMetrics"]); + $output["packages"] = array_values($output["packages"]); + + usort($output["packages"], [$this, "sortInstalls"]); + + $output["packages"] = array_slice($output["packages"], 0, 20); + + return $output; + } + + private function sortInstalls($a, $b) + { + return $a["totalInstalls"] > $b["totalInstalls"] ? -1 : 1; + } +} diff --git a/src/cronJobs/scripts/storeSoftwareAssetsSnapshot.php b/src/cronJobs/scripts/storeSoftwareAssetsSnapshot.php new file mode 100644 index 000000000..3ebc925a0 --- /dev/null +++ b/src/cronJobs/scripts/storeSoftwareAssetsSnapshot.php @@ -0,0 +1,17 @@ +build(); + +$getSoftwareAssetsSnapshotData = $container->make("dhope0000\LXDClient\Tools\Hosts\SoftwareAssets\GetSoftwareAssetsSnapshotData"); +$insertSoftwareAssets = $container->make("dhope0000\LXDClient\Model\Hosts\SoftwareAssets\InsertSoftwareAssetsSnapshot"); + +$date = new \DateTimeImmutable(); + +$softwareAssets = $getSoftwareAssetsSnapshotData->get(); +$insertSoftwareAssets->insert($date, $softwareAssets); diff --git a/src/cronJobs/takeSoftwareAssetsSnapshotTask.php b/src/cronJobs/takeSoftwareAssetsSnapshotTask.php new file mode 100644 index 000000000..57d6f7cfd --- /dev/null +++ b/src/cronJobs/takeSoftwareAssetsSnapshotTask.php @@ -0,0 +1,26 @@ +build(); + +$env = new Dotenv\Dotenv(__DIR__ . "/../../"); +$env->load(); + +$getInstanceSetting = $container->make("dhope0000\LXDClient\Model\InstanceSettings\GetSetting"); + +$monitorSoftwareAssets = $getInstanceSetting->getSettingLatestValue(dhope0000\LXDClient\Constants\InstanceSettingsKeys::SOFTWARE_INVENTORY_MONITOR); + +if (empty($monitorSoftwareAssets) || $monitorSoftwareAssets == 0) { + return new Schedule(); +} + +$schedule = new Schedule(); +$task = $schedule->run(PHP_BINARY . ' ' . __DIR__ . '/scripts/storeSoftwareAssetsSnapshot.php'); +$task + ->daily() + ->at("23:59:50") + ->description('Take daily snapshot of software assets'); + +return $schedule; diff --git a/src/views/boxes/boxComponents/settings/softwareAssets.html b/src/views/boxes/boxComponents/settings/softwareAssets.html new file mode 100644 index 000000000..6b9b20d7f --- /dev/null +++ b/src/views/boxes/boxComponents/settings/softwareAssets.html @@ -0,0 +1,256 @@ +
+
+

+
+
+
+
+
+

All Snapshots

+
+
+
+ + + + + + + + +
Snapshot Date
+
+
+
+
+
+

Snapshot:

+
+
+
+
+
    +
  • +
    + Overview +
    +
    • +
  • +
    + All Software +
    +
    • +
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + +
PackageInstallsNo Versions
+
+
+
+ +
+
+
+ \ No newline at end of file diff --git a/src/views/boxes/settings.php b/src/views/boxes/settings.php index 900856524..ec443681e 100644 --- a/src/views/boxes/settings.php +++ b/src/views/boxes/settings.php @@ -317,6 +317,7 @@ +