From 68b1189ca5c02de46a1db4cd525bcf109bd2da35 Mon Sep 17 00:00:00 2001 From: Ran Nozik Date: Sun, 12 Jan 2025 12:55:48 +0200 Subject: [PATCH] feat: expose image content extraction --- lib/index.ts | 3 +- lib/scan.ts | 97 ++++++++++++++++--- .../application-scans/gomodules.spec.ts | 16 ++- test/system/application-scans/node.spec.ts | 20 +++- .../application-scans/python/pip.spec.ts | 14 ++- 5 files changed, 132 insertions(+), 18 deletions(-) diff --git a/lib/index.ts b/lib/index.ts index c6dad185..40323a4f 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -12,7 +12,7 @@ import { UpdateDockerfileBaseImageNameErrorCode, } from "./dockerfile/types"; import * as facts from "./facts"; -import { scan } from "./scan"; +import { extractContent, scan } from "./scan"; import { AutoDetectedUserInstructions, ContainerTarget, @@ -28,6 +28,7 @@ export { scan, display, dockerFile, + extractContent, facts, ScanResult, PluginResponse, diff --git a/lib/scan.ts b/lib/scan.ts index 727e38fd..46dafc41 100644 --- a/lib/scan.ts +++ b/lib/scan.ts @@ -4,7 +4,9 @@ import * as path from "path"; import { getImageArchive } from "./analyzer/image-inspector"; import { readDockerfileAndAnalyse } from "./dockerfile"; import { DockerFileAnalysis } from "./dockerfile/types"; +import { extractImageContent } from "./extractor"; import { ImageName } from "./extractor/image"; +import { ExtractAction, ExtractionResult } from "./extractor/types"; import { fullImageSavePath } from "./image-save-path"; import { getArchivePath, getImageType } from "./image-type"; import { isNumber, isTrue } from "./option-utils"; @@ -19,9 +21,14 @@ export function mergeEnvVarsIntoCredentials( options.password = options.password || process.env.SNYK_REGISTRY_PASSWORD; } -export async function scan( +async function getAnalysisParameters( options?: Partial, -): Promise { +): Promise<{ + targetImage: string; + imageType: ImageType; + dockerfileAnalysis: DockerFileAnalysis | undefined; + options: Partial; +}> { if (!options) { throw new Error("No plugin options provided"); } @@ -67,6 +74,23 @@ export async function scan( const dockerfileAnalysis = await readDockerfileAndAnalyse(dockerfilePath); const imageType = getImageType(targetImage); + return { + targetImage, + imageType, + dockerfileAnalysis, + options, + }; +} + +export async function scan( + options?: Partial, +): Promise { + const { + targetImage, + imageType, + dockerfileAnalysis, + options: updatedOptions, + } = await getAnalysisParameters(options); switch (imageType) { case ImageType.DockerArchive: case ImageType.OciArchive: @@ -74,14 +98,14 @@ export async function scan( targetImage, imageType, dockerfileAnalysis, - options, + updatedOptions, ); case ImageType.Identifier: return imageIdentifierAnalysis( targetImage, imageType, dockerfileAnalysis, - options, + updatedOptions, ); default: @@ -89,6 +113,20 @@ export async function scan( } } +function getAndValidateArchivePath(targetImage: string) { + const archivePath = getArchivePath(targetImage); + if (!fs.existsSync(archivePath)) { + throw new Error( + "The provided archive path does not exist on the filesystem", + ); + } + if (!fs.lstatSync(archivePath).isFile()) { + throw new Error("The provided archive path is not a file"); + } + + return archivePath; +} + async function localArchiveAnalysis( targetImage: string, imageType: ImageType, @@ -100,16 +138,7 @@ async function localArchiveAnalysis( exclude: options.globsToFind?.exclude || [], }; - const archivePath = getArchivePath(targetImage); - if (!fs.existsSync(archivePath)) { - throw new Error( - "The provided archive path does not exist on the filesystem", - ); - } - if (!fs.lstatSync(archivePath).isFile()) { - throw new Error("The provided archive path is not a file"); - } - + const archivePath = getAndValidateArchivePath(targetImage); const imageIdentifier = options.imageNameAndTag || // The target image becomes the base of the path, e.g. "archive.tar" for "/var/tmp/archive.tar" @@ -183,3 +212,43 @@ export function appendLatestTagIfMissing(targetImage: string): string { } return targetImage; } + +export async function extractContent( + extractActions: ExtractAction[], + options?: Partial, +): Promise { + const { + targetImage, + imageType, + options: updatedOptions, + } = await getAnalysisParameters(options); + + const { username, password, platform, imageSavePath } = updatedOptions; + let imagePath: string; + switch (imageType) { + case ImageType.DockerArchive: + case ImageType.OciArchive: + imagePath = getAndValidateArchivePath(targetImage); + break; + case ImageType.Identifier: + const imageSavePathFull = fullImageSavePath(imageSavePath); + const archiveResult = await getImageArchive( + targetImage, + imageSavePathFull, + username, + password, + platform, + ); + imagePath = archiveResult.path; + break; + default: + throw new Error("Unhandled image type for image " + targetImage); + } + + return extractImageContent( + imageType, + imagePath, + extractActions, + updatedOptions, + ); +} diff --git a/test/system/application-scans/gomodules.spec.ts b/test/system/application-scans/gomodules.spec.ts index 0b69f680..eec1e173 100644 --- a/test/system/application-scans/gomodules.spec.ts +++ b/test/system/application-scans/gomodules.spec.ts @@ -1,6 +1,7 @@ import * as elf from "elfy"; -import { scan } from "../../../lib"; +import { extractContent, scan } from "../../../lib"; +import { getGoModulesContentAction } from "../../../lib/go-parser"; import { getFixture } from "../../util"; describe("gomodules binaries scanning", () => { @@ -23,6 +24,19 @@ describe("gomodules binaries scanning", () => { expect(pluginResult).toMatchSnapshot(); }); + it("should extract image content successfully", async () => { + const fixturePath = getFixture( + "docker-archives/docker-save/testgo-1.17.tar", + ); + const imageNameAndTag = `docker-archive:${fixturePath}`; + const result = await extractContent([getGoModulesContentAction], { + path: imageNameAndTag, + }); + const testgoBinary = result.extractedLayers["/testgo"]; + expect(testgoBinary).toBeTruthy(); + expect("gomodules" in testgoBinary).toBeTruthy(); + }); + it("return plugin result when Go binary cannot be parsed do not break layer iterator", async () => { const elfParseMock = jest.spyOn(elf, "parse").mockImplementation(() => { throw new Error("Cannot read property 'type' of undefined"); diff --git a/test/system/application-scans/node.spec.ts b/test/system/application-scans/node.spec.ts index 44802bfb..2dde8d3d 100644 --- a/test/system/application-scans/node.spec.ts +++ b/test/system/application-scans/node.spec.ts @@ -4,7 +4,7 @@ import { legacy } from "@snyk/dep-graph"; import * as lockFileParser from "snyk-nodejs-lockfile-parser"; import { NodeLockfileVersion } from "snyk-nodejs-lockfile-parser"; import * as resolveDeps from "snyk-resolve-deps"; -import { scan } from "../../../lib"; +import { extractContent, scan } from "../../../lib"; import { getLockFileVersion, shouldBuildDepTree, @@ -12,6 +12,7 @@ import { import * as nodeUtils from "../../../lib/analyzer/applications/node-modules-utils"; import { getAppFilesRootDir } from "../../../lib/analyzer/applications/runtime-common"; import { FilePathToContent } from "../../../lib/analyzer/applications/types"; +import { getNodeAppFileContentAction } from "../../../lib/inputs/node/static"; import { getFixture, getObjFromFixture } from "../../util"; describe("node application scans", () => { @@ -215,6 +216,23 @@ describe("node application scans", () => { expect(depGraphNpmFromGlobalNodeModules.rootPkg.name).toEqual("lib"); }); + it("should extract image content successfully", async () => { + const fixturePath = getFixture("npm/multi-project-image.tar"); + const imageNameAndTag = `docker-archive:${fixturePath}`; + const result = await extractContent([getNodeAppFileContentAction], { + path: imageNameAndTag, + }); + expect(Object.keys(result.extractedLayers).length).toEqual(608); + Object.keys(result.extractedLayers).forEach((fileName) => { + expect( + fileName.endsWith("/package.json") || + fileName.endsWith("/package-lock.json") || + fileName.endsWith("/yarn.lock"), + ).toBeTruthy(); + expect("node-app-files" in result.extractedLayers[fileName]).toBeTruthy(); + }); + }); + it("should generate a scanResult that contains a npm7 depGraph generated from node modules manifest files", async () => { const imageWithManifestFiles = getFixture( "npm/npm-without-lockfiles/npm7-with-package-lock-file.tar", diff --git a/test/system/application-scans/python/pip.spec.ts b/test/system/application-scans/python/pip.spec.ts index d1c57caa..44971015 100644 --- a/test/system/application-scans/python/pip.spec.ts +++ b/test/system/application-scans/python/pip.spec.ts @@ -1,5 +1,6 @@ -import { scan } from "../../../../lib"; +import { extractContent, scan } from "../../../../lib"; import { getAppFilesRootDir } from "../../../../lib/analyzer/applications/runtime-common"; +import { getPythonAppFileContentAction } from "../../../../lib/inputs/python/static"; import { getFixture } from "../../../util"; describe("pip application scan", () => { @@ -60,6 +61,17 @@ describe("pip application scan", () => { expect(pluginResultExcludeAppVulnsTrueBoolean.scanResults).toHaveLength(1); }); + it("should extract image content successfully", async () => { + const fixturePath = getFixture("docker-archives/docker-save/pip-flask.tar"); + const imageNameAndTag = `docker-archive:${fixturePath}`; + const result = await extractContent([getPythonAppFileContentAction], { + path: imageNameAndTag, + }); + const serverPyFile = result.extractedLayers["/app/server.py"]; + expect(serverPyFile).toBeTruthy(); + expect("python-app-files" in serverPyFile).toBeTruthy(); + }); + it("should handle --collect-application-files", async () => { const fixturePath = getFixture("docker-archives/docker-save/pip-flask.tar"); const imageNameAndTag = `docker-archive:${fixturePath}`;