Merge pull request #640 from snyk/expose_image_content_extraction

Expose image content extraction

lib/index.ts

@@ -12,7 +12,7 @@ import {
   UpdateDockerfileBaseImageNameErrorCode,
 } from "./dockerfile/types";
 import * as facts from "./facts";
-import { scan } from "./scan";
+import { extractContent, scan } from "./scan";
 import {
   AutoDetectedUserInstructions,
   ContainerTarget,
@@ -28,6 +28,7 @@ export {
   scan,
   display,
   dockerFile,
+  extractContent,
   facts,
   ScanResult,
   PluginResponse,

lib/scan.ts

@@ -4,7 +4,9 @@ import * as path from "path";
 import { getImageArchive } from "./analyzer/image-inspector";
 import { readDockerfileAndAnalyse } from "./dockerfile";
 import { DockerFileAnalysis } from "./dockerfile/types";
+import { extractImageContent } from "./extractor";
 import { ImageName } from "./extractor/image";
+import { ExtractAction, ExtractionResult } from "./extractor/types";
 import { fullImageSavePath } from "./image-save-path";
 import { getArchivePath, getImageType } from "./image-type";
 import { isNumber, isTrue } from "./option-utils";
@@ -19,9 +21,14 @@ export function mergeEnvVarsIntoCredentials(
   options.password = options.password || process.env.SNYK_REGISTRY_PASSWORD;
 }
 
-export async function scan(
+async function getAnalysisParameters(
   options?: Partial<PluginOptions>,
-): Promise<PluginResponse> {
+): Promise<{
+  targetImage: string;
+  imageType: ImageType;
+  dockerfileAnalysis: DockerFileAnalysis | undefined;
+  options: Partial<PluginOptions>;
+}> {
   if (!options) {
     throw new Error("No plugin options provided");
   }
@@ -67,28 +74,59 @@ export async function scan(
   const dockerfileAnalysis = await readDockerfileAndAnalyse(dockerfilePath);
 
   const imageType = getImageType(targetImage);
+  return {
+    targetImage,
+    imageType,
+    dockerfileAnalysis,
+    options,
+  };
+}
+
+export async function scan(
+  options?: Partial<PluginOptions>,
+): Promise<PluginResponse> {
+  const {
+    targetImage,
+    imageType,
+    dockerfileAnalysis,
+    options: updatedOptions,
+  } = await getAnalysisParameters(options);
   switch (imageType) {
     case ImageType.DockerArchive:
     case ImageType.OciArchive:
       return localArchiveAnalysis(
         targetImage,
         imageType,
         dockerfileAnalysis,
-        options,
+        updatedOptions,
       );
     case ImageType.Identifier:
       return imageIdentifierAnalysis(
         targetImage,
         imageType,
         dockerfileAnalysis,
-        options,
+        updatedOptions,
       );
 
     default:
       throw new Error("Unhandled image type for image " + targetImage);
   }
 }
 
+function getAndValidateArchivePath(targetImage: string) {
+  const archivePath = getArchivePath(targetImage);
+  if (!fs.existsSync(archivePath)) {
+    throw new Error(
+      "The provided archive path does not exist on the filesystem",
+    );
+  }
+  if (!fs.lstatSync(archivePath).isFile()) {
+    throw new Error("The provided archive path is not a file");
+  }
+
+  return archivePath;
+}
+
 async function localArchiveAnalysis(
   targetImage: string,
   imageType: ImageType,
@@ -100,16 +138,7 @@ async function localArchiveAnalysis(
     exclude: options.globsToFind?.exclude || [],
   };
 
-  const archivePath = getArchivePath(targetImage);
-  if (!fs.existsSync(archivePath)) {
-    throw new Error(
-      "The provided archive path does not exist on the filesystem",
-    );
-  }
-  if (!fs.lstatSync(archivePath).isFile()) {
-    throw new Error("The provided archive path is not a file");
-  }
-
+  const archivePath = getAndValidateArchivePath(targetImage);
   const imageIdentifier =
     options.imageNameAndTag ||
     // The target image becomes the base of the path, e.g. "archive.tar" for "/var/tmp/archive.tar"
@@ -183,3 +212,43 @@ export function appendLatestTagIfMissing(targetImage: string): string {
   }
   return targetImage;
 }
+
+export async function extractContent(
+  extractActions: ExtractAction[],
+  options?: Partial<PluginOptions>,
+): Promise<ExtractionResult> {
+  const {
+    targetImage,
+    imageType,
+    options: updatedOptions,
+  } = await getAnalysisParameters(options);
+
+  const { username, password, platform, imageSavePath } = updatedOptions;
+  let imagePath: string;
+  switch (imageType) {
+    case ImageType.DockerArchive:
+    case ImageType.OciArchive:
+      imagePath = getAndValidateArchivePath(targetImage);
+      break;
+    case ImageType.Identifier:
+      const imageSavePathFull = fullImageSavePath(imageSavePath);
+      const archiveResult = await getImageArchive(
+        targetImage,
+        imageSavePathFull,
+        username,
+        password,
+        platform,
+      );
+      imagePath = archiveResult.path;
+      break;
+    default:
+      throw new Error("Unhandled image type for image " + targetImage);
+  }
+
+  return extractImageContent(
+    imageType,
+    imagePath,
+    extractActions,
+    updatedOptions,
+  );
+}

test/system/application-scans/gomodules.spec.ts

@@ -1,6 +1,7 @@
 import * as elf from "elfy";
 
-import { scan } from "../../../lib";
+import { extractContent, scan } from "../../../lib";
+import { getGoModulesContentAction } from "../../../lib/go-parser";
 import { getFixture } from "../../util";
 
 describe("gomodules binaries scanning", () => {
@@ -23,6 +24,19 @@ describe("gomodules binaries scanning", () => {
     expect(pluginResult).toMatchSnapshot();
   });
 
+  it("should extract image content successfully", async () => {
+    const fixturePath = getFixture(
+      "docker-archives/docker-save/testgo-1.17.tar",
+    );
+    const imageNameAndTag = `docker-archive:${fixturePath}`;
+    const result = await extractContent([getGoModulesContentAction], {
+      path: imageNameAndTag,
+    });
+    const testgoBinary = result.extractedLayers["/testgo"];
+    expect(testgoBinary).toBeTruthy();
+    expect("gomodules" in testgoBinary).toBeTruthy();
+  });
+
   it("return plugin result when Go binary cannot be parsed do not break layer iterator", async () => {
     const elfParseMock = jest.spyOn(elf, "parse").mockImplementation(() => {
       throw new Error("Cannot read property 'type' of undefined");

test/system/application-scans/node.spec.ts

@@ -4,14 +4,15 @@ import { legacy } from "@snyk/dep-graph";
 import * as lockFileParser from "snyk-nodejs-lockfile-parser";
 import { NodeLockfileVersion } from "snyk-nodejs-lockfile-parser";
 import * as resolveDeps from "snyk-resolve-deps";
-import { scan } from "../../../lib";
+import { extractContent, scan } from "../../../lib";
 import {
   getLockFileVersion,
   shouldBuildDepTree,
 } from "../../../lib/analyzer/applications/node";
 import * as nodeUtils from "../../../lib/analyzer/applications/node-modules-utils";
 import { getAppFilesRootDir } from "../../../lib/analyzer/applications/runtime-common";
 import { FilePathToContent } from "../../../lib/analyzer/applications/types";
+import { getNodeAppFileContentAction } from "../../../lib/inputs/node/static";
 import { getFixture, getObjFromFixture } from "../../util";
 
 describe("node application scans", () => {
@@ -215,6 +216,23 @@ describe("node application scans", () => {
     expect(depGraphNpmFromGlobalNodeModules.rootPkg.name).toEqual("lib");
   });
 
+  it("should extract image content successfully", async () => {
+    const fixturePath = getFixture("npm/multi-project-image.tar");
+    const imageNameAndTag = `docker-archive:${fixturePath}`;
+    const result = await extractContent([getNodeAppFileContentAction], {
+      path: imageNameAndTag,
+    });
+    expect(Object.keys(result.extractedLayers).length).toEqual(608);
+    Object.keys(result.extractedLayers).forEach((fileName) => {
+      expect(
+        fileName.endsWith("/package.json") ||
+          fileName.endsWith("/package-lock.json") ||
+          fileName.endsWith("/yarn.lock"),
+      ).toBeTruthy();
+      expect("node-app-files" in result.extractedLayers[fileName]).toBeTruthy();
+    });
+  });
+
   it("should generate a scanResult that contains a npm7 depGraph generated from node modules manifest files", async () => {
     const imageWithManifestFiles = getFixture(
       "npm/npm-without-lockfiles/npm7-with-package-lock-file.tar",

test/system/application-scans/python/pip.spec.ts

@@ -1,5 +1,6 @@
-import { scan } from "../../../../lib";
+import { extractContent, scan } from "../../../../lib";
 import { getAppFilesRootDir } from "../../../../lib/analyzer/applications/runtime-common";
+import { getPythonAppFileContentAction } from "../../../../lib/inputs/python/static";
 import { getFixture } from "../../../util";
 
 describe("pip application scan", () => {
@@ -60,6 +61,17 @@ describe("pip application scan", () => {
     expect(pluginResultExcludeAppVulnsTrueBoolean.scanResults).toHaveLength(1);
   });
 
+  it("should extract image content successfully", async () => {
+    const fixturePath = getFixture("docker-archives/docker-save/pip-flask.tar");
+    const imageNameAndTag = `docker-archive:${fixturePath}`;
+    const result = await extractContent([getPythonAppFileContentAction], {
+      path: imageNameAndTag,
+    });
+    const serverPyFile = result.extractedLayers["/app/server.py"];
+    expect(serverPyFile).toBeTruthy();
+    expect("python-app-files" in serverPyFile).toBeTruthy();
+  });
+
   it("should handle --collect-application-files", async () => {
     const fixturePath = getFixture("docker-archives/docker-save/pip-flask.tar");
     const imageNameAndTag = `docker-archive:${fixturePath}`;