Make chen 2 the default (#68)

* Switch to java 21 (#59) Signed-off-by: Prabhu Subramanian <[email protected]> * Switch to cpg2 and overflowdb2 (#60) * Use cpg2 Signed-off-by: Prabhu Subramanian <[email protected]> * Use odb2 1.0.0 Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]> * Use nop logger (#61) Signed-off-by: Prabhu Subramanian <[email protected]> * Fix/issue 62 (#63) * Fix importCode predef. Default to reachables slicing by default Signed-off-by: Prabhu Subramanian <[email protected]> * native image fixes for java Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]> * Update packages Signed-off-by: Prabhu Subramanian <[email protected]> * Automatically invoke cdxgen while importing code (#65) * Automatically invoke cdxgen while importing code Signed-off-by: Prabhu Subramanian <[email protected]> * Use dynamic source Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]> * Feature/js cli source (#66) * Added cli source for js Signed-off-by: Prabhu Subramanian <[email protected]> * Added cli source for js Signed-off-by: Prabhu Subramanian <[email protected]> * Improves method full name construction for js Signed-off-by: Prabhu Subramanian <[email protected]> * Revert back changes to get tests working back Signed-off-by: Prabhu Subramanian <[email protected]> * Better way to resolve a method full name Signed-off-by: Prabhu Subramanian <[email protected]> * Fix tests Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]> * Update java version Signed-off-by: Prabhu Subramanian <[email protected]> * Tags for crypto libs and methods (#67) Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Jan 29, 2024 · 9eb99c1 · 9eb99c1
1 parent 0e9d1e5
commit 9eb99c1
Show file tree

Hide file tree

Showing 49 changed files with 718 additions and 608 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -41,7 +41,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20.x'
+          node-version: '21.x'
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20.x'
+          node-version: '21.x'
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -6,7 +6,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        jvm: ['17', '19', '20', '21']
+        jvm: ['21']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -29,7 +29,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20.x'
+          node-version: '21.x'
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20.x'
+          node-version: '21.x'
       - name: Delete `.rustup` directory
         run: rm -rf /home/runner/.rustup # to save disk space
         if: runner.os == 'Linux'

diff --git a/.github/workflows/win_compat.yml b/.github/workflows/win_compat.yml
@@ -21,4 +21,4 @@ jobs:
       run: |
         python -m pip install poetry
         python -m poetry install
-        python -m poetry run chen ${{ matrix.with-science }}
+        python -m poetry run chen ${{ matrix.with-science }}
diff --git a/.gitignore b/.gitignore
@@ -58,3 +58,5 @@ chen.zip
 project/metals.sbt
 conda-out/
 notebooks/.ipynb_checkpoints/
+*.slices.json
+
diff --git a/README.md b/README.md
@@ -144,7 +144,7 @@ Refer to the documentation site to learn more about the commands.
 
 ## Languages supported
 
-- C/C++ (Requires Java 17 or above)
+- C/C++ (Requires Java 21 or above)
 - H (C/C++ Header files alone)
 - Java (Requires compilation) - 8 to 17
 - Jar

diff --git a/build.sbt b/build.sbt
@@ -1,9 +1,9 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "1.1.6"
+ThisBuild / version      := "2.0.6"
 ThisBuild / scalaVersion := "3.3.1"
 
-val cpgVersion = "1.4.22"
+val cpgVersion = "1.0.0"
 
 lazy val platform          = Projects.platform
 lazy val console           = Projects.console
@@ -35,26 +35,24 @@ lazy val aggregatedProjects: Seq[ProjectReference] = Seq(
 
 ThisBuild / libraryDependencies ++= Seq(
   "org.slf4j"                % "slf4j-api"         % "2.0.11",
-  "org.apache.logging.log4j" % "log4j-slf4j2-impl" % "2.22.1" % Optional,
-  "org.apache.logging.log4j" % "log4j-core"        % "2.22.1" % Optional
-  // `Optional` means "not transitive", but still included in "stage/lib"
+  "org.slf4j"                % "slf4j-nop"         % "2.0.11" % Optional,
 )
 
 ThisBuild / compile / javacOptions ++= Seq(
   "-g", // debug symbols
   "-Xlint",
-  "--release=17"
+  "--release=21"
 ) ++ {
   // fail early if users with JDK8 try to run this
   val javaVersion = sys.props("java.specification.version").toFloat
-  assert(javaVersion.toInt >= 17, s"this build requires JDK17+ - you're using $javaVersion")
+  assert(javaVersion.toInt >= 21, s"this build requires JDK21+ - you're using $javaVersion")
   Nil
 }
 
 ThisBuild / scalacOptions ++= Seq(
   "-deprecation", // Emit warning and location for usages of deprecated APIs.
   "--release",
-  "17",
+  "21"
 )
 
 

diff --git a/chenpy/cli.py b/chenpy/cli.py
@@ -93,7 +93,7 @@ def fix_envs():
         )
         if not os.getenv("JAVA_HOME"):
             LOG.info(
-                "Ensure Java >= 17 up to 20 is installed. Set the environment variable JAVA_HOME to point the correct "
+                "Ensure Java >= 21 is installed. Set the environment variable JAVA_HOME to point the correct "
                 "java directory."
             )
         LOG.info(

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.authors="Team AppThreat <[email protected]>" \
       org.opencontainers.image.source="https://github.com/appthreat/chen" \
       org.opencontainers.image.url="https://github.com/appthreat/chen" \
-      org.opencontainers.image.version="1.1.x" \
+      org.opencontainers.image.version="2.0.x" \
       org.opencontainers.image.vendor="appthreat" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="chen" \

diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/chen",
   "issueTracker": "https://github.com/AppThreat/chen/issues",
   "name": "chen",
-  "version": "1.1.6",
+  "version": "2.0.6",
   "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
   "applicationCategory": "code-analysis",
   "keywords": [

diff --git a/console/build.sbt b/console/build.sbt
@@ -17,7 +17,7 @@ dependsOn(
 )
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"         %% "codepropertygraph" % Versions.cpg,
+  "io.appthreat"         %% "cpg2" % Versions.cpg,
   "com.michaelpollmeier" %% "scala-repl-pp-server" % Versions.scalaReplPP,
   "com.github.scopt"     %% "scopt"             % ScoptVersion,
   "org.typelevel"        %% "cats-effect"       % Versions.cats,

diff --git a/console/src/main/scala/io/appthreat/console/BridgeBase.scala b/console/src/main/scala/io/appthreat/console/BridgeBase.scala
@@ -290,7 +290,7 @@ trait ScriptExecution:
       */
     private def importCpgCode(config: Config): List[String] =
         config.cpgToLoad.map { cpgFile =>
-            "importCpg(\"" + cpgFile + "\")"
+            "importAtom(\"" + cpgFile + "\")"
         }.toList ++ config.forInputPath.map { name =>
             s"""
          |openForInputPath(\"$name\")
@@ -340,9 +340,8 @@ trait PluginHandling:
 
         val storeCode = if config.store then "save"
         else ""
-        val runDataflow = if productName == "ocular" then "run.dataflow"
-        else "run.ossdataflow"
-        val argsString = argsStringFromConfig(config)
+        val runDataflow = "run.ossdataflow"
+        val argsString  = argsStringFromConfig(config)
 
         s"""
        | if (${config.overwrite} || !workspace.projectExists("$src")) {

diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala
@@ -522,6 +522,8 @@ class Console[T <: Project](
                                         .add(
                                           c.methodFullName + (if c.callee(
                                                                     NoResolve
+                                                                  ).nonEmpty && c.callee(
+                                                                    NoResolve
                                                                   ).head.nonEmpty && c.callee(
                                                                     NoResolve
                                                                   ).head.isExternal
@@ -530,6 +532,7 @@ class Console[T <: Project](
                                         )
                                     addedMethods += c.methodFullName -> true
                             )
+                    end if
                 )
                 rootTree.add(childTree)
             }

diff --git a/console/src/main/scala/io/appthreat/console/ConsoleConfig.scala b/console/src/main/scala/io/appthreat/console/ConsoleConfig.scala
@@ -12,12 +12,12 @@ import scala.collection.mutable
   */
 class InstallConfig(environment: Map[String, String] = sys.env):
 
-    /** determining the root path of the joern/ocular installation is rather complex unfortunately,
-      * because we support a variety of use cases:
+    /** determining the root path of the installation is rather complex unfortunately, because we
+      * support a variety of use cases:
       *   - running the installed distribution from the install dir
       *   - running the installed distribution anywhere else on the system
-      *   - running a locally staged ocular/joern build (via `sbt stage` and then either `./joern`
-      *     or `cd platform/target/universal/stage; ./joern`)
+      *   - running a locally staged build (via `sbt stage` and then either `./chennai` or `cd
+      *     platform/target/universal/stage; ./chennai`)
       *   - running a unit/integration test (note: the jars would be in the local cache, e.g. in
       *     ~/.coursier/cache)
       */

diff --git a/console/src/main/scala/io/appthreat/console/Run.scala b/console/src/main/scala/io/appthreat/console/Run.scala
@@ -3,8 +3,8 @@ package io.appthreat.console
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.HasStoreMethod
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
-import org.reflections8.Reflections
-import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder}
+import org.reflections.Reflections
+import org.reflections.util.{ClasspathHelper, ConfigurationBuilder}
 
 import scala.jdk.CollectionConverters.*
 

diff --git a/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala b/console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala
@@ -1,5 +1,6 @@
 package io.appthreat.console.cpgcreation
 
+import better.files.File
 import io.appthreat.console.FrontendConfig
 import io.shiftleft.codepropertygraph.Cpg
 
@@ -10,26 +11,41 @@ case class AtomGenerator(
   config: FrontendConfig,
   rootPath: Path,
   language: String,
-  sliceMode: String = "usages",
-  slicesFile: String = "usages.json"
+  sliceMode: String = "reachables",
+  slicesFile: String = "reachables.slices.json"
 ) extends CpgGenerator:
-    private lazy val command: String = "atom"
+    private lazy val command: String       = sys.env.getOrElse("ATOM_CMD", "atom")
+    private lazy val cdxgenCommand: String = sys.env.getOrElse("CDXGEN_CMD", "cdxgen")
 
     /** Generate an atom for the given input path. Returns the output path, or None, if no CPG was
       * generated.
       */
     override def generate(inputPath: String, outputPath: String = "app.atom"): Try[String] =
+        // If there is no bom.json file in the root directory, attempt to automatically invoke cdxgen
+        val bomPath = File(inputPath) / "bom.json"
+        if !bomPath.exists then
+            val cdxLanguage = language.toLowerCase().replace("src", "")
+            val arguments = Seq(
+              "-t",
+              cdxLanguage,
+              "--deep",
+              "-o",
+              (File(inputPath) / "bom.json").pathAsString,
+              inputPath
+            )
+            runShellCommand(cdxgenCommand, arguments)
         val arguments = Seq(
           sliceMode,
           "-s",
-          slicesFile,
+          (File(inputPath) / slicesFile).pathAsString,
           "--output",
-          outputPath,
+          (File(inputPath) / outputPath).pathAsString,
           "--language",
           language,
           inputPath
         ) ++ config.cmdLineParams
-        runShellCommand(command, arguments).map(_ => outputPath)
+        runShellCommand(command, arguments).map(_ => (File(inputPath) / outputPath).pathAsString)
+    end generate
 
     override def isAvailable: Boolean = true
 

diff --git a/console/src/main/scala/io/appthreat/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/appthreat/console/cpgcreation/ImportCode.scala
@@ -1,16 +1,12 @@
 package io.appthreat.console.cpgcreation
 
 import better.files.File
-import io.appthreat.console.{Console, FrontendConfig, Reporting}
 import io.appthreat.console.workspacehandling.Project
-import io.appthreat.console.{ConsoleException, FrontendConfig, Reporting}
+import io.appthreat.console.{Console, ConsoleException, FrontendConfig, Reporting}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.Languages
-import overflowdb.traversal.help.Table
 import me.shadaj.scalapy.py
-import me.shadaj.scalapy.py.SeqConverters
-import py.PyQuote
-import me.shadaj.scalapy.interpreter.CPythonInterpreter
+import overflowdb.traversal.help.Table
 
 import java.nio.file.Path
 import scala.util.{Failure, Success, Try}

diff --git a/console/src/test/scala/io/appthreat/console/LanguageHelperTests.scala b/console/src/test/scala/io/appthreat/console/LanguageHelperTests.scala
@@ -26,47 +26,47 @@ class LanguageHelperTests extends AnyWordSpec with Matchers {
     }
 
     "guess `JavaSrc` for a directory containing `.java`" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "ServiceIdentifierComposerVisitorBasedStrategy.java")
         guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.JAVASRC)
       }
     }
 
     "guess `Go` for a directory containing `Gopkg.lock`" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "Gopkg.lock")
         guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.GOLANG)
       }
     }
 
     "guess `Go` for a directory containing `Gopkg.toml`" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "Gopkg.toml")
         guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.GOLANG)
       }
     }
 
     "guess `Javascript` for a directory containing `package.json`" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "package.json")
         guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.JSSRC)
       }
     }
 
     "guess `C` for a directory containing .ll (LLVM) file" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "foobar.ll")
         guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.LLVM)
       }
     }
 
     "guess the language with the largest number of files" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         val subdir = mkdir(tmpDir / "subdir")
         touch(subdir / "source.c")
         touch(subdir / "source.java")
@@ -79,7 +79,7 @@ class LanguageHelperTests extends AnyWordSpec with Matchers {
     }
 
     "not find anything for an empty directory" in {
-      File.usingTemporaryDirectory("oculartests") { tmpDir =>
+      File.usingTemporaryDirectory("chentests") { tmpDir =>
         guessLanguage(tmpDir.pathAsString) shouldBe None
       }
     }

diff --git a/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/Engine.scala b/dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/Engine.scala
@@ -28,7 +28,7 @@ class Engine(context: EngineContext):
 
     private val logger: Logger = LoggerFactory.getLogger(this.getClass)
     private val executorService: ExecutorService =
-        Executors.newWorkStealingPool(2)
+        Executors.newVirtualThreadPerTaskExecutor()
     private val completionService =
         new ExecutorCompletionService[TaskSummary](executorService)
 

diff --git a/macros/build.sbt b/macros/build.sbt
@@ -3,7 +3,7 @@ name := "macros"
 dependsOn(Projects.semanticcpg % Test)
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "io.appthreat"  %% "cpg2" % Versions.cpg,
   "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 

diff --git a/macros/src/main/scala/io/appthreat/console/QueryDatabase.scala b/macros/src/main/scala/io/appthreat/console/QueryDatabase.scala
@@ -1,7 +1,7 @@
 package io.appthreat.console
 
-import org.reflections8.Reflections
-import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder}
+import org.reflections.Reflections
+import org.reflections.util.{ClasspathHelper, ConfigurationBuilder}
 
 import java.lang.reflect.{Method, Parameter}
 import scala.annotation.unused
@@ -44,7 +44,7 @@ class QueryDatabase(
             // the namespace currently looks like `io.appthreat.scanners.c.CopyLoops`
             val namespaceParts = bundleNamespace.split('.')
             val language =
-                if bundleNamespace.startsWith("io.appthreat.ocular.scanners") then
+                if bundleNamespace.startsWith("io.appthreat.chen.scanners") then
                     namespaceParts(4)
                 else if namespaceParts.length > 3 then
                     namespaceParts(3)

diff --git a/meta.yaml b/meta.yaml
@@ -1,4 +1,4 @@
-{% set version = "1.1.6" %}
+{% set version = "2.0.6" %}
 
 package:
   name: chen