Merge pull request #19813 from github/kaspersv/overlay-java-discarding

Overlay: Add manual Java overlay annotations & discard predicates

Author: kaspersv

java/ql/lib/semmle/code/Location.qll

@@ -8,6 +8,7 @@ module;
 
 import FileSystem
 import semmle.code.java.Element
+private import semmle.code.java.Overlay
 private import semmle.code.SMAP
 
 /** Holds if element `e` has name `name`. */
@@ -221,3 +222,16 @@ private predicate fixedHasLocation(Top l, Location loc, File f) {
   not hasSourceLocation(l, _, _) and
   locations_default(loc, f, _, _, _, _)
 }
+
+overlay[local]
+private predicate discardableLocation(string file, @location l) {
+  not isOverlay() and
+  file = getRawFileForLoc(l) and
+  not exists(@file f | hasLocation(f, l))
+}
+
+/** Discard base locations in files fully extracted in the overlay. */
+overlay[discard_entity]
+private predicate discardLocation(@location l) {
+  exists(string file | discardableLocation(file, l) and extractedInOverlay(file))
+}

java/ql/lib/semmle/code/java/Expr.qll

@@ -7,6 +7,7 @@ module;
 import java
 private import semmle.code.java.frameworks.android.Compose
 private import semmle.code.java.Constants
+private import semmle.code.java.Overlay
 
 /** A common super-class that represents all kinds of expressions. */
 class Expr extends ExprParent, @expr {
@@ -2701,3 +2702,6 @@ class RecordPatternExpr extends Expr, @recordpatternexpr {
     )
   }
 }
+
+overlay[local]
+private class DiscardableExpr extends DiscardableLocatable, @expr { }

java/ql/lib/semmle/code/java/Javadoc.qll

@@ -5,6 +5,7 @@ overlay[local?]
 module;
 
 import semmle.code.Location
+private import semmle.code.java.Overlay
 
 /** A Javadoc parent is an element whose child can be some Javadoc documentation. */
 class JavadocParent extends @javadocParent, Top {
@@ -196,3 +197,6 @@ class KtCommentSection extends @ktcommentsection {
   /** Gets the string representation of this section. */
   string toString() { result = this.getContent() }
 }
+
+overlay[local]
+private class DiscardableJavadoc extends DiscardableLocatable, @javadoc { }

java/ql/lib/semmle/code/java/Member.qll

@@ -11,6 +11,7 @@ import Annotation
 import Exception
 import metrics.MetricField
 private import dispatch.VirtualDispatch
+private import semmle.code.java.Overlay
 
 /**
  * A common abstraction for type member declarations,
@@ -623,7 +624,13 @@ class SrcMethod extends Method {
       then implementsInterfaceMethod(result, this)
       else result.getASourceOverriddenMethod*() = this
     ) and
-    (exists(result.getBody()) or result.hasModifier("native"))
+    (
+      // We allow empty method bodies for the local overlay variant to allow
+      // calls to methods only fully extracted in base.
+      isOverlay() or
+      exists(result.getBody()) or
+      result.hasModifier("native")
+    )
   }
 }
 
@@ -897,3 +904,13 @@ class ExtensionMethod extends Method {
     else result = 0
   }
 }
+
+overlay[local]
+private class DiscardableAnonymousMethod extends DiscardableLocatable, @method {
+  DiscardableAnonymousMethod() {
+    exists(@classorinterface c | methods(this, _, _, _, c, _) and isAnonymClass(c, _))
+  }
+}
+
+overlay[local]
+private class DiscardableMethod extends DiscardableReferableLocatable, @method { }

java/ql/lib/semmle/code/java/Overlay.qll

@@ -0,0 +1,83 @@
+/**
+ * Defines entity discard predicates for Java overlay analysis.
+ */
+overlay[local?]
+module;
+
+import java
+
+/**
+ * A local predicate that always holds for the overlay variant and
+ * never holds for the base variant. This is used to define local
+ * predicates that behave differently for the base and overlay variant.
+ */
+overlay[local]
+predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+/** Gets the raw file for a locatable. */
+overlay[local]
+string getRawFile(@locatable el) {
+  exists(@location loc, @file file |
+    hasLocation(el, loc) and
+    locations_default(loc, file, _, _, _, _) and
+    files(file, result)
+  )
+}
+
+/** Gets the raw file for a location. */
+overlay[local]
+string getRawFileForLoc(@location l) {
+  exists(@file f | locations_default(l, f, _, _, _, _) and files(f, result))
+}
+
+/** Holds for files fully extracted in the overlay. */
+overlay[local]
+predicate extractedInOverlay(string file) {
+  isOverlay() and
+  // numlines is used to restrict attention to fully extracted files and
+  // ignore skeleton extracted files in the overlay
+  exists(@locatable l | numlines(l, _, _, _) and file = getRawFile(l))
+}
+
+/**
+ * A `@locatable` that should be discarded in the base variant if its file is
+ * extracted in the overlay variant.
+ */
+overlay[local]
+abstract class DiscardableLocatable extends @locatable {
+  /** Gets the raw file for a locatable in base. */
+  string getRawFileInBase() { not isOverlay() and result = getRawFile(this) }
+
+  /** Gets a textual representation of this discardable locatable. */
+  string toString() { none() }
+}
+
+overlay[discard_entity]
+private predicate discardLocatable(@locatable el) {
+  extractedInOverlay(el.(DiscardableLocatable).getRawFileInBase())
+}
+
+/**
+ * A `@locatable` that should be discarded in the base variant if its file is
+ * extracted in the overlay variant and it is itself not extracted in the
+ * overlay, that is, it is deleted in the overlay.
+ */
+overlay[local]
+abstract class DiscardableReferableLocatable extends @locatable {
+  /** Gets the raw file for a locatable in base. */
+  string getRawFileInBase() { not isOverlay() and result = getRawFile(this) }
+
+  /** Holds if the locatable exists in the overlay. */
+  predicate existsInOverlay() { isOverlay() and exists(this) }
+
+  /** Gets a textual representation of this discardable locatable. */
+  string toString() { none() }
+}
+
+overlay[discard_entity]
+private predicate discardReferableLocatable(@locatable el) {
+  exists(DiscardableReferableLocatable drl | drl = el |
+    extractedInOverlay(drl.getRawFileInBase()) and
+    not drl.existsInOverlay()
+  )
+}

java/ql/lib/semmle/code/java/Statement.qll

@@ -6,6 +6,7 @@ module;
 
 import Expr
 import metrics.MetricStmt
+private import semmle.code.java.Overlay
 
 /** A common super-class of all statements. */
 class Stmt extends StmtParent, ExprParent, @stmt {
@@ -987,3 +988,6 @@ class SuperConstructorInvocationStmt extends Stmt, ConstructorCall, @superconstr
 
   override string getAPrimaryQlClass() { result = "SuperConstructorInvocationStmt" }
 }
+
+overlay[local]
+private class DiscardableStmt extends DiscardableLocatable, @stmt { }

java/ql/lib/semmle/code/java/Variable.qll

@@ -5,6 +5,7 @@ overlay[local?]
 module;
 
 import Element
+private import semmle.code.java.Overlay
 
 /** A variable is a field, a local variable or a parameter. */
 class Variable extends @variable, Annotatable, Element, Modifiable {
@@ -133,3 +134,6 @@ class Parameter extends Element, @param, LocalScopeVariable {
   /** Holds if this is an anonymous parameter, `_` */
   predicate isAnonymous() { this.getName() = "" }
 }
+
+overlay[local]
+private class DiscardableLocalScopeVariable extends DiscardableLocatable, @localscopevariable { }

java/ql/src/experimental/Security/CWE/CWE-020/Log4jJndiInjection.ql

@@ -22,6 +22,7 @@ import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.security.Sanitizers
 import Log4jInjectionFlow::PathGraph
 
+overlay[local?]
 deprecated private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "log4j-injection" }
 }

java/ql/src/experimental/Security/CWE/CWE-036/OpenStream.ql

@@ -17,6 +17,7 @@ import semmle.code.java.dataflow.FlowSources
 import semmle.code.java.dataflow.ExternalFlow
 import RemoteUrlToOpenStreamFlow::PathGraph
 
+overlay[local?]
 deprecated private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "openstream-called-on-tainted-url" }
 }

java/ql/src/experimental/Security/CWE/CWE-073/FilePathInjection.ql

@@ -22,6 +22,7 @@ import semmle.code.java.security.PathSanitizer
 private import semmle.code.java.security.Sanitizers
 import InjectFilePathFlow::PathGraph
 
+overlay[local?]
 deprecated private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "file-path-injection" }
 }

java/ql/src/experimental/Security/CWE/CWE-078/ExecTainted.ql

@@ -18,6 +18,7 @@ import semmle.code.java.security.CommandLineQuery
 import InputToArgumentToExecFlow::PathGraph
 private import semmle.code.java.dataflow.ExternalFlow
 
+overlay[local?]
 deprecated private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "jsch-os-injection" }
 }

java/ql/src/experimental/Security/CWE/CWE-200/AndroidWebResourceResponse.qll

@@ -7,6 +7,7 @@ private import semmle.code.java.dataflow.ExternalFlow
 private import semmle.code.java.dataflow.FlowSteps
 private import semmle.code.java.frameworks.android.WebView
 
+overlay[local?]
 private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "android-web-resource-response" }
 }

java/ql/src/experimental/Security/CWE/CWE-400/ThreadResourceAbuse.qll

@@ -8,6 +8,7 @@ import semmle.code.java.arithmetic.Overflow
 import semmle.code.java.dataflow.FlowSteps
 import semmle.code.java.controlflow.Guards
 
+overlay[local?]
 private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "thread-resource-abuse" }
 }

java/ql/src/experimental/Security/CWE/CWE-625/PermissiveDotRegexQuery.qll

@@ -9,6 +9,7 @@ import semmle.code.java.controlflow.Guards
 import semmle.code.java.security.UrlRedirect
 import Regex
 
+overlay[local?]
 private class ActivateModels extends ActiveExperimentalModels {
   ActivateModels() { this = "permissive-dot-regex-query" }
 }

shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll

@@ -291,6 +291,7 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
      * to `lambdaCall`, if any. That is, `lastCall` is able to target the enclosing
      * callable of `lambdaCall`.
      */
+    overlay[global]
     pragma[nomagic]
     predicate revLambdaFlow(
       Call lambdaCall, LambdaCallKind kind, Node node, Type t, boolean toReturn, boolean toJump,