Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit e6e39d3

Browse files
Python codegen improvement: timeouted tests and exceptions (#2101)
* Add skipping of tests with too long execution * Add asserts for exception * Add timeouted tests limit and fix setting fail/pass exceptions * Fix handleTimeoutResult * Fix pytest skip decorator * Fix mistake in pytest error message
1 parent f26d423 commit e6e39d3

File tree

17 files changed

+227
-44
lines changed

17 files changed

+227
-44
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
418418
return { +actual[streamConsumingMethodId]() }
419419
}
420420

421-
protected fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
421+
protected openfun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
422422
if (exception is AccessControlException) return false
423423
// tests with timeout or crash should be processed differently
424424
if (exception is TimeoutException || exception is InstrumentedProcessDeathException) return false

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogProcessor.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ object PythonDialogProcessor {
205205
val message = it.fold(StringBuilder()) { acc, line -> acc.appendHtmlLine(line) }
206206
WarningTestsReportNotifier.notify(message.toString())
207207
},
208+
runtimeExceptionTestsBehaviour = model.runtimeExceptionTestsBehaviour,
208209
startedCleaningAction = { indicator.text = "Cleaning up..." }
209210
)
210211
} finally {

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonDialogWindow.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ class PythonDialogWindow(val model: PythonTestsModel) : DialogWrapper(model.proj
177177
val settings = model.project.service<Settings>()
178178
with(settings) {
179179
model.timeoutForRun = hangingTestsTimeout.timeoutMs
180+
model.runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour
180181
}
181182

182183
super.doOKAction()

‎utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/language/python/PythonTestsModel.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import com.intellij.openapi.project.Project
55
import com.jetbrains.python.psi.PyClass
66
import com.jetbrains.python.psi.PyFile
77
import com.jetbrains.python.psi.PyFunction
8+
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
89
import org.utbot.framework.codegen.domain.TestFramework
910
import org.utbot.framework.codegen.services.language.CgLanguageAssistant
1011
import org.utbot.intellij.plugin.models.BaseTestsModel
@@ -31,4 +32,5 @@ class PythonTestsModel(
3132
lateinit var testSourceRootPath: String
3233
lateinit var testFramework: TestFramework
3334
lateinit var selectedFunctions: Set<PyFunction>
35+
lateinit var runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour
3436
}

‎utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ class PythonEngine(
4747
// can be improved
4848
description.name
4949
}
50-
is UtExplicitlyThrownException -> "${description.name}_with_exception"
50+
is UtExecutionFailure -> "${description.name}_with_exception"
5151
else -> description.name
5252
}
5353
val testName = "test_$testSuffix"
@@ -81,6 +81,37 @@ class PythonEngine(
8181
return Pair(stateThisObject, modelList)
8282
}
8383

84+
private fun handleTimeoutResult(
85+
arguments: List<PythonFuzzedValue>,
86+
methodUnderTestDescription: PythonMethodDescription,
87+
): FuzzingExecutionFeedback {
88+
val summary = arguments
89+
.zip(methodUnderTest.arguments)
90+
.mapNotNull { it.first.summary?.replace("%var%", it.second.name) }
91+
val executionResult = UtTimeoutException(TimeoutException("Execution is too long"))
92+
val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult)
93+
94+
val hasThisObject = methodUnderTest.hasThisArgument
95+
val (beforeThisObjectTree, beforeModelListTree) = if (hasThisObject) {
96+
arguments.first() to arguments.drop(1)
97+
} else {
98+
null to arguments
99+
}
100+
val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) }
101+
val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) }
102+
103+
val utFuzzedExecution = PythonUtExecution(
104+
stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
105+
stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
106+
stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()),
107+
diffIds = emptyList(),
108+
result = executionResult,
109+
testMethodName = testMethodName.testName?.camelToSnakeCase(),
110+
displayName = testMethodName.displayName,
111+
summary = summary.map { DocRegularStmt(it) }
112+
)
113+
return ValidExecution(utFuzzedExecution)
114+
}
84115
private fun handleSuccessResult(
85116
arguments: List<PythonFuzzedValue>,
86117
types: List<Type>,
@@ -110,7 +141,7 @@ class PythonEngine(
110141

111142
val executionResult =
112143
if (evaluationResult.isException) {
113-
UtExplicitlyThrownException(Throwable(resultModel.type.toString()), false)
144+
UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false)
114145
}
115146
else {
116147
UtExecutionSuccess(PythonTreeModel(resultModel))
@@ -157,8 +188,7 @@ class PythonEngine(
157188
serverSocket,
158189
pythonPath,
159190
until,
160-
{ constructEvaluationInput(it) },
161-
)
191+
) { constructEvaluationInput(it) }
162192
} catch (_: TimeoutException) {
163193
return@flow
164194
}
@@ -199,8 +229,8 @@ class PythonEngine(
199229
}
200230

201231
is PythonEvaluationTimeout -> {
202-
val utError = UtError(evaluationResult.message, Throwable())
203-
PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS))
232+
val utTimeoutException = handleTimeoutResult(arguments, description)
233+
PythonExecutionResult(utTimeoutException, PythonFeedback(control = Control.PASS))
204234
}
205235

206236
is PythonEvaluationSuccess -> {

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import java.io.File
3131

3232
private val logger = KotlinLogging.logger {}
3333
private const val RANDOM_TYPE_FREQUENCY = 6
34+
private const val MAX_EMPTY_COVERAGE_TESTS = 5
3435

3536
class PythonTestCaseGenerator(
3637
private val withMinimization: Boolean = true,
@@ -262,12 +263,15 @@ class PythonTestCaseGenerator(
262263
}
263264

264265
logger.info("Collect all test executions for ${method.name}")
265-
val (successfulExecutions, failedExecutions) = executions.partition { it.result is UtExecutionSuccess }
266+
val (emptyCoverageExecutions, coverageExecutions) = executions.partition { it.coverage == null }
267+
val (successfulExecutions, failedExecutions) = coverageExecutions.partition { it.result is UtExecutionSuccess }
266268

267269
return PythonTestSet(
268270
method,
269271
if (withMinimization)
270-
minimizeExecutions(successfulExecutions) + minimizeExecutions(failedExecutions)
272+
minimizeExecutions(successfulExecutions) +
273+
minimizeExecutions(failedExecutions) +
274+
emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS)
271275
else
272276
executions,
273277
errors,

‎utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package org.utbot.python
33
import com.squareup.moshi.Moshi
44
import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
55
import org.parsers.python.PythonParser
6+
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
67
import org.utbot.python.framework.codegen.model.PythonSysPathImport
78
import org.utbot.python.framework.codegen.model.PythonSystemImport
89
import org.utbot.python.framework.codegen.model.PythonUserImport
@@ -50,6 +51,7 @@ object PythonTestGenerationProcessor {
5051
pythonRunRoot: Path,
5152
doNotCheckRequirements: Boolean = false,
5253
withMinimization: Boolean = true,
54+
runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.FAIL,
5355
isCanceled: () -> Boolean = { false },
5456
checkingRequirementsAction: () -> Unit = {},
5557
installingRequirementsAction: () -> Unit = {},
@@ -202,6 +204,7 @@ object PythonTestGenerationProcessor {
202204
paramNames = paramNames,
203205
testFramework = testFramework,
204206
testClassPackageName = "",
207+
runtimeExceptionTestsBehaviour = runtimeExceptionTestsBehaviour,
205208
)
206209
val testCode = codegen.pythonGenerateAsStringWithTestReport(
207210
notEmptyTests.map { testSet ->

‎utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
package org.utbot.python.evaluation
22

3+
import java.net.SocketException
4+
35
class UtExecutorThread : Thread() {
46
override fun run() {
5-
response = pythonWorker?.receiveMessage()
7+
response = try {
8+
pythonWorker?.receiveMessage()
9+
} catch (ex: SocketException) {
10+
null
11+
}
612
}
713

814
enum class Status {

‎utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ class PythonClassId(
3434
override val simpleName: String = typeName
3535
override val canonicalName = name
3636
override val packageName = moduleName
37+
val prettyName: String = if (rootModuleName == pythonBuiltinsModuleName)
38+
name.split(".", limit=2).last()
39+
else
40+
name
3741
}
3842

3943
open class RawPythonAnnotation(

‎utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ val pythonDictClassId = PythonClassId("builtins.dict")
1919
val pythonSetClassId = PythonClassId("builtins.set")
2020
val pythonBytearrayClassId = PythonClassId("builtins.bytearray")
2121
val pythonBytesClassId = PythonClassId("builtins.bytes")
22+
val pythonExceptionClassId = PythonClassId("builtins.Exception")

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /