Skip to content

Commit 332c215

Browse files
mmvpmmmvpm
authored
Add codeforces examples to contest estimator (#1750)
1 parent 9b2619a commit 332c215

File tree

8 files changed

+134
-17
lines changed

8 files changed

+134
-17
lines changed

‎utbot-junit-contest/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ The projects are provided to Contest estimator in advance.
2828
All available projects are placed in the [resources][resources] folder, which contains:
2929
- [projects][projects] consisting of the folders with the project JAR files in them.
3030
- [classes][classes] consisting of the folders — each named after the project and containing the `list` file with the fully qualified class names.
31+
It also may contain an `exceptions` file with the description of the expected exceptions, that utbot should find.
32+
Description is presented in the format: `<class fully qualified name>.<method name>: <expected exception fqn> <another fqn> ...`.
33+
For example, see this [file](src/main/resources/classes/codeforces/exceptions).
3134

3235
### How to add a new project
3336
You should add both the JAR files to the `projects` folder and the file with a list of classes to the `classes` folder.

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking
1414
import org.utbot.framework.codegen.domain.StaticsMocking
1515
import org.utbot.framework.codegen.domain.junitByVersion
1616
import org.utbot.framework.codegen.CodeGenerator
17-
import org.utbot.framework.plugin.api.CodegenLanguage
18-
import org.utbot.framework.plugin.api.Coverage
19-
import org.utbot.framework.plugin.api.ExecutableId
20-
import org.utbot.framework.plugin.api.MockStrategyApi
21-
import org.utbot.framework.plugin.api.TestCaseGenerator
22-
import org.utbot.framework.plugin.api.UtError
23-
import org.utbot.framework.plugin.api.UtExecution
24-
import org.utbot.framework.plugin.api.UtMethodTestSet
2517
import org.utbot.framework.plugin.api.util.UtContext
2618
import org.utbot.framework.plugin.api.util.executableId
2719
import org.utbot.framework.plugin.api.util.id
@@ -59,10 +51,10 @@ import kotlinx.coroutines.launch
5951
import kotlinx.coroutines.newSingleThreadContext
6052
import kotlinx.coroutines.runBlocking
6153
import kotlinx.coroutines.withTimeoutOrNull
62-
import kotlinx.coroutines.yield
6354
import org.utbot.framework.SummariesGenerationType
6455
import org.utbot.framework.codegen.services.language.CgLanguageAssistant
6556
import org.utbot.framework.minimization.minimizeExecutions
57+
import org.utbot.framework.plugin.api.*
6658
import org.utbot.framework.plugin.api.util.isSynthetic
6759
import org.utbot.framework.util.jimpleBody
6860
import org.utbot.summary.summarize
@@ -154,6 +146,7 @@ fun main(args: Array<String>) {
154146
fuzzingRatio = 0.1,
155147
classpathString,
156148
runFromEstimator = false,
149+
expectedExceptions = ExpectedExceptionsForClass(),
157150
methodNameFilter = null
158151
)
159152
println("${ContestMessage.READY}")
@@ -187,6 +180,7 @@ fun runGeneration(
187180
fuzzingRatio: Double,
188181
classpathString: String,
189182
runFromEstimator: Boolean,
183+
expectedExceptions: ExpectedExceptionsForClass,
190184
methodNameFilter: String? = null // For debug purposes you can specify method name
191185
): StatsForClass = runBlocking {
192186
val testsByMethod: MutableMap<ExecutableId, MutableList<UtExecution>> = mutableMapOf()
@@ -266,7 +260,10 @@ fun runGeneration(
266260
val methodJob = currentCoroutineContext().job
267261

268262
logger.debug { " ... " }
269-
val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}")
263+
val statsForMethod = StatsForMethod(
264+
"${method.classId.simpleName}#${method.name}",
265+
expectedExceptions.getForMethod(method.name).exceptionNames
266+
)
270267
statsForClass.statsForMethods.add(statsForMethod)
271268

272269

@@ -337,6 +334,9 @@ fun runGeneration(
337334
val className = Type.getInternalName(method.classId.jClass)
338335
logger.debug { "--new testCase collected, to generate: $testMethodName" }
339336
statsForMethod.testsGeneratedCount++
337+
result.result.exceptionOrNull()?.let { exception ->
338+
statsForMethod.detectedExceptionFqns += exception::class.java.name
339+
}
340340
result.coverage?.let {
341341
statsForClass.updateCoverage(
342342
newCoverage = it,

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ enum class Tool {
137137
methodNameFilter: String?,
138138
statsForProject: StatsForProject,
139139
compiledTestDir: File,
140-
classFqn: String
140+
classFqn: String,
141+
expectedExceptions: ExpectedExceptionsForClass
141142
) = withUtContext(ContextManager.createNewContext(project.classloader)) {
142143
val classStats: StatsForClass = try {
143144
runGeneration(
@@ -147,6 +148,7 @@ enum class Tool {
147148
fuzzingRatio,
148149
project.sootClasspathString,
149150
runFromEstimator = true,
151+
expectedExceptions,
150152
methodNameFilter
151153
)
152154
} catch (e: CancellationException) {
@@ -206,7 +208,8 @@ enum class Tool {
206208
methodNameFilter: String?,
207209
statsForProject: StatsForProject,
208210
compiledTestDir: File,
209-
classFqn: String
211+
classFqn: String,
212+
expectedExceptions: ExpectedExceptionsForClass
210213
) {
211214
// EvoSuite has several phases, the variable below is responsible for assert generation
212215
// timeout. We want to give 10s for a big time budgets and timeLimit / 5 for small budgets.
@@ -277,7 +280,8 @@ enum class Tool {
277280
methodNameFilter: String?,
278281
statsForProject: StatsForProject,
279282
compiledTestDir: File,
280-
classFqn: String
283+
classFqn: String,
284+
expectedExceptions: ExpectedExceptionsForClass
281285
)
282286

283287
abstract fun moveProducedFilesIfNeeded()
@@ -391,7 +395,13 @@ fun runEstimator(
391395
if (updatedMethodFilter != null)
392396
logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" }
393397

394-
val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() }
398+
val projectDirs = classesLists.listFiles()!!
399+
val projectToClassFQNs = projectDirs.associate {
400+
it.name to File(it, "list").readLines()
401+
}
402+
val projectToExpectedExceptions = projectDirs.associate {
403+
it.name to parseExceptionsFile(File(it, "exceptions"))
404+
}
395405

396406
val projects = mutableListOf<ProjectToEstimate>()
397407

@@ -400,6 +410,7 @@ fun runEstimator(
400410
val project = ProjectToEstimate(
401411
name,
402412
classesFQN,
413+
projectToExpectedExceptions.getValue(name),
403414
File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") },
404415
testCandidatesDir,
405416
unzippedJars
@@ -463,7 +474,8 @@ fun runEstimator(
463474
methodNameFilter,
464475
statsForProject,
465476
compiledTestDir,
466-
classFqn
477+
classFqn,
478+
project.expectedExceptions.getForClass(classFqn)
467479
)
468480
}
469481
catch (e: Throwable) {
@@ -526,6 +538,7 @@ private fun classNamesByJar(jar: File): List<String> {
526538
class ProjectToEstimate(
527539
val name: String,
528540
val classFQNs: List<String>,
541+
val expectedExceptions: ExpectedExceptionsForProject,
529542
private val jars: List<File>,
530543
testCandidatesDir: File,
531544
unzippedJars: File

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/ExpectedExceptions.kt‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package org.utbot.contest
2+
3+
import java.io.File
4+
5+
class ExpectedExceptionsForMethod(
6+
val exceptionNames: List<String>
7+
)
8+
9+
class ExpectedExceptionsForClass {
10+
11+
private val byMethodName: MutableMap<String, ExpectedExceptionsForMethod> =
12+
mutableMapOf()
13+
14+
fun getForMethod(methodName: String): ExpectedExceptionsForMethod =
15+
byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf())
16+
17+
fun setForMethod(methodName: String, exceptionNames: List<String>) {
18+
byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames)
19+
}
20+
}
21+
22+
class ExpectedExceptionsForProject {
23+
24+
private val byClassName: MutableMap<String, ExpectedExceptionsForClass> =
25+
mutableMapOf()
26+
27+
fun getForClass(className: String): ExpectedExceptionsForClass =
28+
byClassName[className] ?: ExpectedExceptionsForClass()
29+
30+
fun setForClass(className: String, methodName: String, exceptionNames: List<String>) {
31+
val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() }
32+
forClass.setForMethod(methodName, exceptionNames)
33+
}
34+
}
35+
36+
fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject {
37+
if (!exceptionsDescriptionFile.exists()) {
38+
return ExpectedExceptionsForProject()
39+
}
40+
41+
val forProject = ExpectedExceptionsForProject()
42+
for (methodDescription in exceptionsDescriptionFile.readLines()) {
43+
val methodFqn = methodDescription.substringBefore(':').trim()
44+
val classFqn = methodFqn.substringBeforeLast('.')
45+
val methodName = methodFqn.substringAfterLast('.')
46+
val exceptionFqns = methodDescription.substringAfter(':')
47+
.split(' ')
48+
.map { it.trim() }
49+
.filter { it.isNotBlank() }
50+
forProject.setForClass(classFqn, methodName, exceptionFqns)
51+
}
52+
53+
return forProject
54+
}

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,15 @@ class StatsForProject(val project: String) {
7575
else this
7676
}
7777

78+
val detectedExceptionsCount: Int
79+
get() = statsForClasses.sumOf { it.detectedExceptionsCount }
80+
val expectedExceptionsCount: Int
81+
get() = statsForClasses.sumOf { it.expectedExceptionsCount }
82+
7883
override fun toString(): String = "\n<StatsForProject($project)> :" +
7984
"\n\t#classes for generation = $classesForGeneration" +
8085
"\n\t#tc generated = $testCasesGenerated" +
86+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
8187
"\n\t#classes without problems = $classesWithoutProblems" +
8288
"\n\t#classes canceled by timeout = $classesCanceledByTimeout" +
8389
"\n----------------------------------------" +
@@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) {
128134
val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() }
129135
val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount }
130136

137+
val detectedExceptionsCount: Int
138+
get() = statsForMethods.sumOf { it.detectedExceptionsCount }
139+
val expectedExceptionsCount: Int
140+
get() = statsForMethods.sumOf { it.expectedExceptionsCount }
141+
131142
var coverage = CoverageInstructionsSet()
132143
var fuzzedCoverage = CoverageInstructionsSet()
133144
var concolicCoverage = CoverageInstructionsSet()
@@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) {
149160
"\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" +
150161
"\n\t#methods with exceptions = $methodsWithAtLeastOneException" +
151162
"\n\t#generated TC = $testcasesGenerated" +
163+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
152164
"\n\t#total coverage = ${coverage.prettyInfo()}" +
153165
"\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" +
154166
"\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}"
155167
}
156168

157169

158-
class StatsForMethod(val methodName: String) {
170+
class StatsForMethod(
171+
val methodName: String,
172+
val expectedExceptionFqns: List<String>
173+
) {
159174
var testsGeneratedCount = 0
160175

161176
val failReasons: MutableMultiset<FailReason> = mutableMultisetOf()
162177

178+
val detectedExceptionFqns: MutableSet<String> = mutableSetOf()
179+
163180
//generated no TC, nor exception
164181
val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0
165182

183+
val detectedExceptionsCount: Int
184+
get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size
185+
186+
val expectedExceptionsCount: Int =
187+
expectedExceptionFqns.size
188+
166189

167190
override fun toString(): String = "\n<StatsForMethod> :" + (if (isSuspicious) " SUSPICIOUS" else "") +
168-
"\n\t#generatedTC=$testsGeneratedCount\n\t" +
191+
"\n\t#generated TC = $testsGeneratedCount" +
192+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
193+
"\n\t" +
169194
(if (failReasons.isEmpty()) "WITH NO EXCEPTIONS"
170195
else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"")
171196

‎utbot-junit-contest/src/main/resources/classes/codeforces/exceptions‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException

‎utbot-junit-contest/src/main/resources/classes/codeforces/list‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B

‎utbot-junit-contest/src/main/resources/projects/codeforces/Codeforces-1.0-SNAPSHOT.jar‎

14.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)