Skip to content

Commit 13d87f8

Browse files
oxistooxisto
committed
Fixed all tests
1 parent f6ccafd commit 13d87f8

File tree

5 files changed

+350
-271
lines changed

5 files changed

+350
-271
lines changed

Diff for: ‎cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt

+2-3
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa
344344
* e.g. [LoopStatement]s or [BreakStatement].
345345
*/
346346
protected fun handleEOG(node: Node?) {
347-
if (node == null || alreadySeen.contains(node)) {
347+
if (node == null) {
348348
return
349349
}
350350

@@ -543,8 +543,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa
543543
} else if (declaration is FunctionDeclaration) {
544544
// save the current EOG stack, because we can have a function declaration within an
545545
// existing function and the EOG handler for handling function declarations will
546-
// reset the
547-
// stack
546+
// reset the stack
548547
val oldEOG = currentPredecessors.toMutableList()
549548

550549
// analyze the defaults

Diff for: ‎cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt

+331
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,331 @@
1+
/*
2+
* Copyright (c) 2025, Fraunhofer AISEC. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* $$$$$$\ $$$$$$$\ $$$$$$\
17+
* $$ __$$\ $$ __$$\ $$ __$$\
18+
* $$ / \__|$$ | $$ |$$ / \__|
19+
* $$ | $$$$$$$ |$$ |$$$$\
20+
* $$ | $$ ____/ $$ |\_$$ |
21+
* $$ | $$\ $$ | $$ | $$ |
22+
* \$$$$$ |$$ | \$$$$$ |
23+
* \______/ \__| \______/
24+
*
25+
*/
26+
package de.fraunhofer.aisec.cpg.frontends.python
27+
28+
import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading
29+
import de.fraunhofer.aisec.cpg.frontends.isKnownOperatorName
30+
import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_KEYWORD_ONLY_ARGUMENT
31+
import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_POSITIONAL_ONLY_ARGUMENT
32+
import de.fraunhofer.aisec.cpg.graph.*
33+
import de.fraunhofer.aisec.cpg.graph.declarations.*
34+
import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope
35+
import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope
36+
import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
37+
import de.fraunhofer.aisec.cpg.graph.types.FunctionType
38+
import de.fraunhofer.aisec.cpg.helpers.Util
39+
40+
/**
41+
* In Python, all declarations are statements. This class handles the parsing of the statements
42+
* represented as [Declaration] nodes in our CPG.
43+
*
44+
* For declarations encountered directly on a namespace and classes, we directly invoke the
45+
* [DeclarationHandler], for others, the [StatementHandler] will forward these statements to us.
46+
*/
47+
class DeclarationHandler(frontend: PythonLanguageFrontend) :
48+
PythonHandler<Declaration, Python.AST.BaseStmt>(::ProblemDeclaration, frontend) {
49+
override fun handleNode(node: Python.AST.BaseStmt): Declaration {
50+
return when (node) {
51+
is Python.AST.FunctionDef -> handleFunctionDef(node)
52+
is Python.AST.AsyncFunctionDef -> handleFunctionDef(node)
53+
else -> {
54+
return handleNotSupported(node, node.javaClass.simpleName)
55+
}
56+
}
57+
}
58+
59+
private fun handleNotSupported(node: Python.AST.BaseStmt, name: String): Declaration {
60+
Util.errorWithFileLocation(
61+
frontend,
62+
node,
63+
log,
64+
"Parsing of type $name is not supported (yet)",
65+
)
66+
67+
val cpgNode = this.configConstructor.get()
68+
if (cpgNode is ProblemNode) {
69+
cpgNode.problem = "Parsing of type $name is not supported (yet)"
70+
}
71+
72+
return cpgNode
73+
}
74+
75+
/**
76+
* We have to consider multiple things when matching Python's FunctionDef to the CPG:
77+
* - A [Python.AST.FunctionDef] could be one of
78+
* - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__`
79+
* - a [MethodeDeclaration] if it appears in a record, and it isn't a
80+
* [ConstructorDeclaration]
81+
* - a [FunctionDeclaration] if neither of the above apply
82+
*
83+
* In case of a [ConstructorDeclaration] or[MethodDeclaration]: the first argument is the
84+
* `receiver` (most often called `self`).
85+
*/
86+
private fun handleFunctionDef(s: Python.AST.NormalOrAsyncFunctionDef): FunctionDeclaration {
87+
var recordDeclaration =
88+
(frontend.scopeManager.currentScope as? RecordScope)?.astNode as? RecordDeclaration
89+
val language = language
90+
val result =
91+
if (recordDeclaration != null) {
92+
if (s.name == "__init__") {
93+
newConstructorDeclaration(
94+
name = s.name,
95+
recordDeclaration = recordDeclaration,
96+
rawNode = s,
97+
)
98+
} else if (language is HasOperatorOverloading && s.name.isKnownOperatorName) {
99+
var decl =
100+
newOperatorDeclaration(
101+
name = s.name,
102+
recordDeclaration = recordDeclaration,
103+
operatorCode = language.operatorCodeFor(s.name) ?: "",
104+
rawNode = s,
105+
)
106+
if (decl.operatorCode == "") {
107+
Util.warnWithFileLocation(
108+
decl,
109+
log,
110+
"Could not find operator code for operator {}. This will most likely result in a failure",
111+
s.name,
112+
)
113+
}
114+
decl
115+
} else {
116+
newMethodDeclaration(
117+
name = s.name,
118+
recordDeclaration = recordDeclaration,
119+
isStatic = false,
120+
rawNode = s,
121+
)
122+
}
123+
} else {
124+
newFunctionDeclaration(name = s.name, rawNode = s)
125+
}
126+
frontend.scopeManager.enterScope(result)
127+
128+
frontend.statementHandler.addAsyncWarning(s, result)
129+
130+
// Handle decorators (which are translated into CPG "annotations")
131+
result.annotations += frontend.statementHandler.handleAnnotations(s)
132+
133+
// Handle return type and calculate function type
134+
if (result is ConstructorDeclaration) {
135+
// Return type of the constructor is always its record declaration type
136+
result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType())
137+
} else {
138+
result.returnTypes = listOf(frontend.typeOf(s.returns))
139+
}
140+
result.type = FunctionType.computeType(result)
141+
142+
handleArguments(s.args, result, recordDeclaration)
143+
144+
if (s.body.isNotEmpty()) {
145+
result.body = frontend.statementHandler.makeBlock(s.body, parentNode = s)
146+
}
147+
148+
frontend.scopeManager.leaveScope(result)
149+
150+
// We want functions inside functions to be wrapped in a declaration statement, so we are
151+
// not adding them to scope's AST
152+
if (frontend.scopeManager.currentScope is FunctionScope) {
153+
frontend.scopeManager.addDeclaration(result, addToAST = false)
154+
} else {
155+
// In any other cases, we add them to the "declarations", otherwise we will not
156+
// correctly resolve them.
157+
frontend.scopeManager.addDeclaration(result)
158+
}
159+
160+
return result
161+
}
162+
163+
/** Adds the arguments to [result] which might be located in a [recordDeclaration]. */
164+
private fun handleArguments(
165+
args: Python.AST.arguments,
166+
result: FunctionDeclaration,
167+
recordDeclaration: RecordDeclaration?,
168+
) {
169+
// We can merge posonlyargs and args because both are positional arguments. We do not
170+
// enforce that posonlyargs can ONLY be used in a positional style, whereas args can be used
171+
// both in positional and keyword style.
172+
var positionalArguments = args.posonlyargs + args.args
173+
174+
// Handle receiver if it exists and if it is not a static method
175+
if (
176+
recordDeclaration != null &&
177+
result.annotations.none { it.name.localName == "staticmethod" }
178+
) {
179+
handleReceiverArgument(positionalArguments, args, result, recordDeclaration)
180+
// Skip the receiver argument for further processing
181+
positionalArguments = positionalArguments.drop(1)
182+
}
183+
184+
// Handle remaining arguments
185+
handlePositionalArguments(positionalArguments, args)
186+
187+
args.vararg?.let { handleArgument(it, isPosOnly = false, isVariadic = true) }
188+
args.kwarg?.let { handleArgument(it, isPosOnly = false, isVariadic = false) }
189+
190+
handleKeywordOnlyArguments(args)
191+
}
192+
193+
/**
194+
* This function creates a [newParameterDeclaration] for the argument, setting any modifiers
195+
* (like positional-only or keyword-only) and [defaultValue] if applicable.
196+
*/
197+
internal fun handleArgument(
198+
node: Python.AST.arg,
199+
isPosOnly: Boolean = false,
200+
isVariadic: Boolean = false,
201+
isKwoOnly: Boolean = false,
202+
defaultValue: Expression? = null,
203+
): ParameterDeclaration {
204+
val type = frontend.typeOf(node.annotation)
205+
val arg =
206+
newParameterDeclaration(
207+
name = node.arg,
208+
type = type,
209+
variadic = isVariadic,
210+
rawNode = node,
211+
)
212+
defaultValue?.let { arg.default = it }
213+
if (isPosOnly) {
214+
arg.modifiers += MODIFIER_POSITIONAL_ONLY_ARGUMENT
215+
}
216+
217+
if (isKwoOnly) {
218+
arg.modifiers += MODIFIER_KEYWORD_ONLY_ARGUMENT
219+
}
220+
221+
frontend.scopeManager.addDeclaration(arg)
222+
223+
return arg
224+
}
225+
226+
/**
227+
* This method retrieves the first argument of the [positionalArguments], which is typically the
228+
* receiver object.
229+
*
230+
* A receiver can also have a default value. However, this case is not handled and is therefore
231+
* passed with a problem expression.
232+
*/
233+
private fun handleReceiverArgument(
234+
positionalArguments: List<Python.AST.arg>,
235+
args: Python.AST.arguments,
236+
result: FunctionDeclaration,
237+
recordDeclaration: RecordDeclaration,
238+
) {
239+
// first argument is the receiver
240+
val recvPythonNode = positionalArguments.firstOrNull()
241+
if (recvPythonNode == null) {
242+
result.additionalProblems += newProblemExpression("Expected a receiver", rawNode = args)
243+
} else {
244+
val tpe = recordDeclaration.toType()
245+
val recvNode =
246+
newVariableDeclaration(
247+
name = recvPythonNode.arg,
248+
type = tpe,
249+
implicitInitializerAllowed = false,
250+
rawNode = recvPythonNode,
251+
)
252+
253+
// If the number of defaults equals the number of positional arguments, the receiver has
254+
// a default value
255+
if (args.defaults.size == positionalArguments.size) {
256+
val defaultValue =
257+
args.defaults.getOrNull(0)?.let { frontend.expressionHandler.handle(it) }
258+
defaultValue?.let {
259+
frontend.scopeManager.addDeclaration(recvNode)
260+
result.additionalProblems +=
261+
newProblemExpression("Receiver with default value", rawNode = args)
262+
}
263+
}
264+
265+
when (result) {
266+
is ConstructorDeclaration,
267+
is MethodDeclaration -> result.receiver = recvNode
268+
else ->
269+
result.additionalProblems +=
270+
newProblemExpression(
271+
problem =
272+
"Expected a constructor or method declaration. Got something else.",
273+
rawNode = result,
274+
)
275+
}
276+
}
277+
}
278+
279+
/**
280+
* This method extracts the [positionalArguments] including those that may have default values.
281+
*
282+
* In Python only the arguments with default values are stored in `args.defaults`.
283+
* https://docs.python.org/3/library/ast.html#ast.arguments
284+
*
285+
* For example: `def my_func(a, b=1, c=2): pass`
286+
*
287+
* In this case, `args.defaults` contains only the defaults for `b` and `c`, while `args.args`
288+
* includes all arguments (`a`, `b` and `c`). The number of arguments without defaults is
289+
* determined by subtracting the size of `args.defaults` from the total number of arguments.
290+
* This matches each default to its corresponding argument.
291+
*
292+
* From the Python docs: "If there are fewer defaults, they correspond to the last n arguments."
293+
*/
294+
private fun handlePositionalArguments(
295+
positionalArguments: List<Python.AST.arg>,
296+
args: Python.AST.arguments,
297+
) {
298+
val nonDefaultArgsCount = positionalArguments.size - args.defaults.size
299+
300+
for (idx in positionalArguments.indices) {
301+
val arg = positionalArguments[idx]
302+
val defaultIndex = idx - nonDefaultArgsCount
303+
val defaultValue =
304+
if (defaultIndex >= 0) {
305+
args.defaults.getOrNull(defaultIndex)?.let {
306+
frontend.expressionHandler.handle(it)
307+
}
308+
} else {
309+
null
310+
}
311+
handleArgument(arg, isPosOnly = arg in args.posonlyargs, defaultValue = defaultValue)
312+
}
313+
}
314+
315+
/**
316+
* This method extracts the keyword-only arguments from [args] and maps them to the
317+
* corresponding function parameters.
318+
*/
319+
private fun handleKeywordOnlyArguments(args: Python.AST.arguments) {
320+
for (idx in args.kwonlyargs.indices) {
321+
val arg = args.kwonlyargs[idx]
322+
val default = args.kw_defaults.getOrNull(idx)
323+
handleArgument(
324+
arg,
325+
isPosOnly = false,
326+
isKwoOnly = true,
327+
defaultValue = default?.let { frontend.expressionHandler.handle(it) },
328+
)
329+
}
330+
}
331+
}

Diff for: ‎cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -568,7 +568,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) :
568568
val function = newFunctionDeclaration(name = "", rawNode = node)
569569
frontend.scopeManager.enterScope(function)
570570
for (arg in node.args.args) {
571-
this.frontend.statementHandler.handleArgument(arg)
571+
this.frontend.declarationHandler.handleArgument(arg)
572572
}
573573
function.body = handle(node.body)
574574
frontend.scopeManager.leaveScope(function)

Diff for: ‎cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt

+6-3
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
6666
private val tokenTypeIndex = 0
6767
private val jep = JepSingleton // configure Jep
6868

69-
// val declarationHandler = DeclarationHandler(this)
70-
// val specificationHandler = SpecificationHandler(this)
69+
internal val declarationHandler = DeclarationHandler(this)
7170
internal var statementHandler = StatementHandler(this)
7271
internal var expressionHandler = ExpressionHandler(this)
7372

@@ -313,7 +312,11 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
313312

314313
if (lastNamespace != null) {
315314
for (stmt in pythonASTModule.body) {
316-
lastNamespace.statements += statementHandler.handle(stmt)
315+
if (stmt is Python.AST.FunctionDef || stmt is Python.AST.AsyncFunctionDef) {
316+
declarationHandler.handleNode(stmt)
317+
} else {
318+
lastNamespace.statements += statementHandler.handle(stmt)
319+
}
317320
}
318321
}
319322

0 commit comments

Comments
 (0)