Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 332c215

Browse files
authored
Add codeforces examples to contest estimator (#1750)
1 parent 9b2619a commit 332c215

File tree

8 files changed

+134
-17
lines changed

8 files changed

+134
-17
lines changed

‎utbot-junit-contest/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ The projects are provided to Contest estimator in advance.
2828
All available projects are placed in the [resources][resources] folder, which contains:
2929
- [projects][projects] consisting of the folders with the project JAR files in them.
3030
- [classes][classes] consisting of the folders — each named after the project and containing the `list` file with the fully qualified class names.
31+
It also may contain an `exceptions` file with the description of the expected exceptions, that utbot should find.
32+
Description is presented in the format: `<class fully qualified name>.<method name>: <expected exception fqn> <another fqn> ...`.
33+
For example, see this [file](src/main/resources/classes/codeforces/exceptions).
3134

3235
### How to add a new project
3336
You should add both the JAR files to the `projects` folder and the file with a list of classes to the `classes` folder.

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,6 @@ import org.utbot.framework.codegen.domain.ForceStaticMocking
1414
import org.utbot.framework.codegen.domain.StaticsMocking
1515
import org.utbot.framework.codegen.domain.junitByVersion
1616
import org.utbot.framework.codegen.CodeGenerator
17-
import org.utbot.framework.plugin.api.CodegenLanguage
18-
import org.utbot.framework.plugin.api.Coverage
19-
import org.utbot.framework.plugin.api.ExecutableId
20-
import org.utbot.framework.plugin.api.MockStrategyApi
21-
import org.utbot.framework.plugin.api.TestCaseGenerator
22-
import org.utbot.framework.plugin.api.UtError
23-
import org.utbot.framework.plugin.api.UtExecution
24-
import org.utbot.framework.plugin.api.UtMethodTestSet
2517
import org.utbot.framework.plugin.api.util.UtContext
2618
import org.utbot.framework.plugin.api.util.executableId
2719
import org.utbot.framework.plugin.api.util.id
@@ -59,10 +51,10 @@ import kotlinx.coroutines.launch
5951
import kotlinx.coroutines.newSingleThreadContext
6052
import kotlinx.coroutines.runBlocking
6153
import kotlinx.coroutines.withTimeoutOrNull
62-
import kotlinx.coroutines.yield
6354
import org.utbot.framework.SummariesGenerationType
6455
import org.utbot.framework.codegen.services.language.CgLanguageAssistant
6556
import org.utbot.framework.minimization.minimizeExecutions
57+
import org.utbot.framework.plugin.api.*
6658
import org.utbot.framework.plugin.api.util.isSynthetic
6759
import org.utbot.framework.util.jimpleBody
6860
import org.utbot.summary.summarize
@@ -154,6 +146,7 @@ fun main(args: Array<String>) {
154146
fuzzingRatio = 0.1,
155147
classpathString,
156148
runFromEstimator = false,
149+
expectedExceptions = ExpectedExceptionsForClass(),
157150
methodNameFilter = null
158151
)
159152
println("${ContestMessage.READY}")
@@ -187,6 +180,7 @@ fun runGeneration(
187180
fuzzingRatio: Double,
188181
classpathString: String,
189182
runFromEstimator: Boolean,
183+
expectedExceptions: ExpectedExceptionsForClass,
190184
methodNameFilter: String? = null // For debug purposes you can specify method name
191185
): StatsForClass = runBlocking {
192186
val testsByMethod: MutableMap<ExecutableId, MutableList<UtExecution>> = mutableMapOf()
@@ -266,7 +260,10 @@ fun runGeneration(
266260
val methodJob = currentCoroutineContext().job
267261

268262
logger.debug { " ... " }
269-
val statsForMethod = StatsForMethod("${method.classId.simpleName}#${method.name}")
263+
val statsForMethod = StatsForMethod(
264+
"${method.classId.simpleName}#${method.name}",
265+
expectedExceptions.getForMethod(method.name).exceptionNames
266+
)
270267
statsForClass.statsForMethods.add(statsForMethod)
271268

272269

@@ -337,6 +334,9 @@ fun runGeneration(
337334
val className = Type.getInternalName(method.classId.jClass)
338335
logger.debug { "--new testCase collected, to generate: $testMethodName" }
339336
statsForMethod.testsGeneratedCount++
337+
result.result.exceptionOrNull()?.let { exception ->
338+
statsForMethod.detectedExceptionFqns += exception::class.java.name
339+
}
340340
result.coverage?.let {
341341
statsForClass.updateCoverage(
342342
newCoverage = it,

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ enum class Tool {
137137
methodNameFilter: String?,
138138
statsForProject: StatsForProject,
139139
compiledTestDir: File,
140-
classFqn: String
140+
classFqn: String,
141+
expectedExceptions: ExpectedExceptionsForClass
141142
) = withUtContext(ContextManager.createNewContext(project.classloader)) {
142143
val classStats: StatsForClass = try {
143144
runGeneration(
@@ -147,6 +148,7 @@ enum class Tool {
147148
fuzzingRatio,
148149
project.sootClasspathString,
149150
runFromEstimator = true,
151+
expectedExceptions,
150152
methodNameFilter
151153
)
152154
} catch (e: CancellationException) {
@@ -206,7 +208,8 @@ enum class Tool {
206208
methodNameFilter: String?,
207209
statsForProject: StatsForProject,
208210
compiledTestDir: File,
209-
classFqn: String
211+
classFqn: String,
212+
expectedExceptions: ExpectedExceptionsForClass
210213
) {
211214
// EvoSuite has several phases, the variable below is responsible for assert generation
212215
// timeout. We want to give 10s for a big time budgets and timeLimit / 5 for small budgets.
@@ -277,7 +280,8 @@ enum class Tool {
277280
methodNameFilter: String?,
278281
statsForProject: StatsForProject,
279282
compiledTestDir: File,
280-
classFqn: String
283+
classFqn: String,
284+
expectedExceptions: ExpectedExceptionsForClass
281285
)
282286

283287
abstract fun moveProducedFilesIfNeeded()
@@ -391,7 +395,13 @@ fun runEstimator(
391395
if (updatedMethodFilter != null)
392396
logger.info { "Filtering: class='$classFqnFilter', method ='$methodNameFilter'" }
393397

394-
val projectToClassFQNs = classesLists.listFiles()!!.associate { it.name to File(it, "list").readLines() }
398+
val projectDirs = classesLists.listFiles()!!
399+
val projectToClassFQNs = projectDirs.associate {
400+
it.name to File(it, "list").readLines()
401+
}
402+
val projectToExpectedExceptions = projectDirs.associate {
403+
it.name to parseExceptionsFile(File(it, "exceptions"))
404+
}
395405

396406
val projects = mutableListOf<ProjectToEstimate>()
397407

@@ -400,6 +410,7 @@ fun runEstimator(
400410
val project = ProjectToEstimate(
401411
name,
402412
classesFQN,
413+
projectToExpectedExceptions.getValue(name),
403414
File(classpathDir, name).listFiles()!!.filter { it.toString().endsWith("jar") },
404415
testCandidatesDir,
405416
unzippedJars
@@ -463,7 +474,8 @@ fun runEstimator(
463474
methodNameFilter,
464475
statsForProject,
465476
compiledTestDir,
466-
classFqn
477+
classFqn,
478+
project.expectedExceptions.getForClass(classFqn)
467479
)
468480
}
469481
catch (e: Throwable) {
@@ -526,6 +538,7 @@ private fun classNamesByJar(jar: File): List<String> {
526538
class ProjectToEstimate(
527539
val name: String,
528540
val classFQNs: List<String>,
541+
val expectedExceptions: ExpectedExceptionsForProject,
529542
private val jars: List<File>,
530543
testCandidatesDir: File,
531544
unzippedJars: File
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package org.utbot.contest
2+
3+
import java.io.File
4+
5+
class ExpectedExceptionsForMethod(
6+
val exceptionNames: List<String>
7+
)
8+
9+
class ExpectedExceptionsForClass {
10+
11+
private val byMethodName: MutableMap<String, ExpectedExceptionsForMethod> =
12+
mutableMapOf()
13+
14+
fun getForMethod(methodName: String): ExpectedExceptionsForMethod =
15+
byMethodName[methodName] ?: ExpectedExceptionsForMethod(listOf())
16+
17+
fun setForMethod(methodName: String, exceptionNames: List<String>) {
18+
byMethodName[methodName] = ExpectedExceptionsForMethod(exceptionNames)
19+
}
20+
}
21+
22+
class ExpectedExceptionsForProject {
23+
24+
private val byClassName: MutableMap<String, ExpectedExceptionsForClass> =
25+
mutableMapOf()
26+
27+
fun getForClass(className: String): ExpectedExceptionsForClass =
28+
byClassName[className] ?: ExpectedExceptionsForClass()
29+
30+
fun setForClass(className: String, methodName: String, exceptionNames: List<String>) {
31+
val forClass = byClassName.getOrPut(className) { ExpectedExceptionsForClass() }
32+
forClass.setForMethod(methodName, exceptionNames)
33+
}
34+
}
35+
36+
fun parseExceptionsFile(exceptionsDescriptionFile: File): ExpectedExceptionsForProject {
37+
if (!exceptionsDescriptionFile.exists()) {
38+
return ExpectedExceptionsForProject()
39+
}
40+
41+
val forProject = ExpectedExceptionsForProject()
42+
for (methodDescription in exceptionsDescriptionFile.readLines()) {
43+
val methodFqn = methodDescription.substringBefore(':').trim()
44+
val classFqn = methodFqn.substringBeforeLast('.')
45+
val methodName = methodFqn.substringAfterLast('.')
46+
val exceptionFqns = methodDescription.substringAfter(':')
47+
.split(' ')
48+
.map { it.trim() }
49+
.filter { it.isNotBlank() }
50+
forProject.setForClass(classFqn, methodName, exceptionFqns)
51+
}
52+
53+
return forProject
54+
}

‎utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,15 @@ class StatsForProject(val project: String) {
7575
else this
7676
}
7777

78+
val detectedExceptionsCount: Int
79+
get() = statsForClasses.sumOf { it.detectedExceptionsCount }
80+
val expectedExceptionsCount: Int
81+
get() = statsForClasses.sumOf { it.expectedExceptionsCount }
82+
7883
override fun toString(): String = "\n<StatsForProject($project)> :" +
7984
"\n\t#classes for generation = $classesForGeneration" +
8085
"\n\t#tc generated = $testCasesGenerated" +
86+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
8187
"\n\t#classes without problems = $classesWithoutProblems" +
8288
"\n\t#classes canceled by timeout = $classesCanceledByTimeout" +
8389
"\n----------------------------------------" +
@@ -128,6 +134,11 @@ class StatsForClass(val project: String, val className: String) {
128134
val methodsWithAtLeastOneException: Int get() = statsForMethods.count { it.failReasons.isNotEmpty() }
129135
val testcasesGenerated: Int get() = statsForMethods.sumOf { it.testsGeneratedCount }
130136

137+
val detectedExceptionsCount: Int
138+
get() = statsForMethods.sumOf { it.detectedExceptionsCount }
139+
val expectedExceptionsCount: Int
140+
get() = statsForMethods.sumOf { it.expectedExceptionsCount }
141+
131142
var coverage = CoverageInstructionsSet()
132143
var fuzzedCoverage = CoverageInstructionsSet()
133144
var concolicCoverage = CoverageInstructionsSet()
@@ -149,23 +160,37 @@ class StatsForClass(val project: String, val className: String) {
149160
"\n\t#methods with at least one TC = ${statsForMethods.count { it.testsGeneratedCount > 0 }}" +
150161
"\n\t#methods with exceptions = $methodsWithAtLeastOneException" +
151162
"\n\t#generated TC = $testcasesGenerated" +
163+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
152164
"\n\t#total coverage = ${coverage.prettyInfo()}" +
153165
"\n\t#fuzzed coverage = ${fuzzedCoverage.prettyInfo()}" +
154166
"\n\t#concolic coverage = ${concolicCoverage.prettyInfo()}"
155167
}
156168

157169

158-
class StatsForMethod(val methodName: String) {
170+
class StatsForMethod(
171+
val methodName: String,
172+
val expectedExceptionFqns: List<String>
173+
) {
159174
var testsGeneratedCount = 0
160175

161176
val failReasons: MutableMultiset<FailReason> = mutableMultisetOf()
162177

178+
val detectedExceptionFqns: MutableSet<String> = mutableSetOf()
179+
163180
//generated no TC, nor exception
164181
val isSuspicious: Boolean get() = failReasons.isEmpty() && testsGeneratedCount == 0
165182

183+
val detectedExceptionsCount: Int
184+
get() = expectedExceptionFqns.toSet().intersect(detectedExceptionFqns).size
185+
186+
val expectedExceptionsCount: Int =
187+
expectedExceptionFqns.size
188+
166189

167190
override fun toString(): String = "\n<StatsForMethod> :" + (if (isSuspicious) " SUSPICIOUS" else "") +
168-
"\n\t#generatedTC=$testsGeneratedCount\n\t" +
191+
"\n\t#generated TC = $testsGeneratedCount" +
192+
(if (expectedExceptionsCount > 0) "\n\t#detected exceptions = $detectedExceptionsCount/$expectedExceptionsCount" else "") +
193+
"\n\t" +
169194
(if (failReasons.isEmpty()) "WITH NO EXCEPTIONS"
170195
else "FAILED ${failReasons.sumOfMultiplicities} time(s) with ${failReasons.size} different exception(s)\"")
171196

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A.solve: java.lang.StackOverflowError
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A.solve: java.lang.ArrayIndexOutOfBoundsException
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B.solve: java.lang.ArrayIndexOutOfBoundsException
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C.solve: java.lang.ArrayIndexOutOfBoundsException
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C.solve: java.lang.ArrayIndexOutOfBoundsException
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B.solve: java.lang.ArithmeticException
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2.solve: java.lang.AssertionError
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C.solve: java.lang.IndexOutOfBoundsException
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D.solve: java.lang.NullPointerException
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A.solve: java.lang.StringIndexOutOfBoundsException
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B.solve: java.lang.StringIndexOutOfBoundsException
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
org.utbot.examples.codeforces.error.stackof.Task_70A
2+
org.utbot.examples.codeforces.exception.aiobe.Task_70A
3+
org.utbot.examples.codeforces.exception.aiobe.Task_71B
4+
org.utbot.examples.codeforces.exception.aiobe.Task_1712C
5+
org.utbot.examples.codeforces.exception.aiobe.Task_1749C
6+
org.utbot.examples.codeforces.exception.arithmetic.Task_1715B
7+
org.utbot.examples.codeforces.exception.error.Task_1718A2
8+
org.utbot.examples.codeforces.exception.iobe.Task_1740C
9+
org.utbot.examples.codeforces.exception.npe.Task_1703D
10+
org.utbot.examples.codeforces.exception.siobe.Task_75A
11+
org.utbot.examples.codeforces.exception.siobe.Task_1702B
14.3 KB
Binary file not shown.

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /