Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 228072f

Browse files
Added NumPy support for utbot-python (#2742)
1 parent 2b37f07 commit 228072f

File tree

30 files changed

+595
-43
lines changed

30 files changed

+595
-43
lines changed

‎utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/TestWriter.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ class TestWriter {
99

1010
fun generateTestCode(): String {
1111
val (importLines, code) = testCode.fold(mutableListOf<String>() to StringBuilder()) { acc, s ->
12-
val lines = s.split(System.lineSeparator())
12+
// val lines = s.split(System.lineSeparator())
13+
val lines = s.split("(\\r\\n|\\r|\\n)".toRegex())
1314
val firstClassIndex = lines.indexOfFirst { it.startsWith("class") }
1415
lines.take(firstClassIndex).forEach { line -> if (line !in acc.first) acc.first.add(line) }
1516
lines.drop(firstClassIndex).forEach { line -> acc.second.append(line + System.lineSeparator()) }

‎utbot-python-executor/src/main/python/utbot_executor/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "utbot-executor"
3-
version = "1.9.19"
3+
version = "1.10.0"
44
description = ""
55
authors = ["Vyacheslav Tamarin <vyacheslav.tamarin@yandex.ru>"]
66
readme = "README.md"

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/deep_serialization.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ def serialize_objects_dump(objs: List[Any], clear_visited: bool = False) -> Tupl
5050
serializer.write_object_to_memory(obj)
5151
for obj in objs
5252
]
53+
5354
return ids, serializer.memory, serialize_memory_dump(serializer.memory)
5455

5556

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,14 @@
1111
ListMemoryObject,
1212
DictMemoryObject,
1313
ReduceMemoryObject,
14-
MemoryDump, IteratorMemoryObject,
14+
MemoryDump, IteratorMemoryObject,NdarrayMemoryObject,
1515
)
1616
from utbot_executor.deep_serialization.utils import PythonId, TypeInfo
17+
try:
18+
import numpy as np
19+
except ImportError:
20+
import sys
21+
print("numpy is not installed", file=sys.stderr)
1722

1823

1924
class MemoryObjectEncoder(json.JSONEncoder):
@@ -27,6 +32,10 @@ def default(self, o):
2732
}
2833
if isinstance(o, ReprMemoryObject):
2934
base_json["value"] = o.value
35+
elif isinstance(o, NdarrayMemoryObject):
36+
base_json["items"] = o.items
37+
base_json["comparable"] = True
38+
base_json["dimensions"] = o.dimensions
3039
elif isinstance(o, (ListMemoryObject, DictMemoryObject)):
3140
base_json["items"] = o.items
3241
elif isinstance(o, IteratorMemoryObject):
@@ -53,6 +62,10 @@ def default(self, o):
5362
"kind": o.kind,
5463
"module": o.module,
5564
}
65+
if isinstance(o, np.ndarray):
66+
raise NotImplementedError("np.ndarray is not supported")
67+
if isinstance(o, np.bool_):
68+
return bool(o)
5669
return json.JSONEncoder.default(self, o)
5770

5871

@@ -75,6 +88,17 @@ def as_reduce_object(dct: Dict) -> Union[MemoryObject, Dict]:
7588
)
7689
obj.comparable = dct["comparable"]
7790
return obj
91+
92+
if dct["strategy"] == "ndarray":
93+
obj = NdarrayMemoryObject.__new__(NdarrayMemoryObject)
94+
obj.items = dct["items"]
95+
obj.typeinfo = TypeInfo(
96+
kind=dct["typeinfo"]["kind"], module=dct["typeinfo"]["module"]
97+
)
98+
obj.comparable = dct["comparable"]
99+
obj.dimensions = dct["dimensions"]
100+
return obj
101+
78102
if dct["strategy"] == "dict":
79103
obj = DictMemoryObject.__new__(DictMemoryObject)
80104
obj.items = dct["items"]
@@ -138,6 +162,11 @@ def reload_id(self) -> MemoryDump:
138162
new_memory_object.items = [
139163
self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
140164
]
165+
elif isinstance(new_memory_object, NdarrayMemoryObject):
166+
new_memory_object.items = [
167+
self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
168+
]
169+
new_memory_object.dimensions = obj.dimensions
141170
elif isinstance(new_memory_object, IteratorMemoryObject):
142171
new_memory_object.items = [
143172
self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
@@ -198,6 +227,27 @@ def load_object(self, python_id: PythonId) -> object:
198227

199228
for item in dump_object.items:
200229
real_object.append(self.load_object(item))
230+
231+
elif isinstance(dump_object, NdarrayMemoryObject):
232+
real_object = []
233+
234+
id_ = PythonId(str(id(real_object)))
235+
self.dump_id_to_real_id[python_id] = id_
236+
self.memory[id_] = real_object
237+
238+
temp_list = []
239+
for item in dump_object.items:
240+
temp_list.append(self.load_object(item))
241+
242+
dt = np.dtype(type(temp_list[0]) if len(temp_list) > 0 else np.int64)
243+
temp_list = np.array(temp_list, dtype=dt)
244+
245+
real_object = np.ndarray(
246+
shape=dump_object.dimensions,
247+
dtype=dt,
248+
buffer=temp_list
249+
)
250+
201251
elif isinstance(dump_object, DictMemoryObject):
202252
real_object = {}
203253

@@ -250,7 +300,7 @@ def load_object(self, python_id: PythonId) -> object:
250300
for key, dictitem in dictitems.items():
251301
real_object[key] = dictitem
252302
else:
253-
raise TypeError(f"Invalid type {dump_object}")
303+
raise TypeError(f"Invalid type {dump_object}, type: {type(dump_object)}")
254304

255305
id_ = PythonId(str(id(real_object)))
256306
self.dump_id_to_real_id[python_id] = id_
@@ -279,6 +329,10 @@ def main():
279329
"builtins.tuple",
280330
"builtins.bytes",
281331
"builtins.type",
332+
"numpy.ndarray"
282333
]
283334
)
284335
print(loader.load_object(PythonId("140239390887040")))
336+
337+
if __name__ == '__main__':
338+
main()

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py

Lines changed: 52 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
import typing
99
from itertools import zip_longest
1010
from typing import Any, Callable, Dict, List, Optional, Set, Type, Iterable
11+
try:
12+
import numpy as np
13+
except ImportError:
14+
import sys
15+
print("numpy is not installed", file=sys.stderr)
1116

1217
from utbot_executor.deep_serialization.config import PICKLE_PROTO
1318
from utbot_executor.deep_serialization.iterator_wrapper import IteratorWrapper
@@ -41,7 +46,7 @@ def __init__(self, obj: object) -> None:
4146
self.id_ = PythonId(str(id(self.obj)))
4247

4348
def _initialize(
44-
self, deserialized_obj: object = None, comparable: bool = True
49+
self, deserialized_obj: object = None, comparable: bool = True
4550
) -> None:
4651
self.deserialized_obj = deserialized_obj
4752
self.comparable = comparable
@@ -111,14 +116,42 @@ def initialize(self) -> None:
111116
elif self.typeinfo.fullname == "builtins.set":
112117
deserialized_obj = set(deserialized_obj)
113118

119+
114120
comparable = all(serializer.get_by_id(elem).comparable for elem in self.items)
115121

116122
super()._initialize(deserialized_obj, comparable)
117123

124+
class NdarrayMemoryObject(MemoryObject):
125+
strategy: str = "ndarray"
126+
items: List[PythonId] = []
127+
dimensions: List[int] = []
128+
129+
def __init__(self, ndarray_object: object) -> None:
130+
self.items: List[PythonId] = []
131+
super().__init__(ndarray_object)
132+
133+
def initialize(self) -> None:
134+
serializer = PythonSerializer()
135+
self.deserialized_obj = [] # for recursive collections
136+
self.comparable = False # for recursive collections
137+
138+
temp_object = self.obj.copy().flatten()
139+
140+
self.dimensions = self.obj.shape
141+
if temp_object.shape != (0, ):
142+
for elem in temp_object:
143+
elem_id = serializer.write_object_to_memory(elem)
144+
self.items.append(elem_id)
145+
self.deserialized_obj.append(serializer[elem_id])
146+
147+
deserialized_obj = self.deserialized_obj
148+
comparable = all(serializer.get_by_id(elem).comparable for elem in self.items) if self.deserialized_obj != [] else True
149+
super()._initialize(deserialized_obj, comparable)
150+
118151
def __repr__(self) -> str:
119152
if hasattr(self, "obj"):
120153
return str(self.obj)
121-
return f"{self.typeinfo.kind}{self.items}"
154+
return f"{self.typeinfo.kind}{self.items}{self.dimensions}"
122155

123156

124157
class DictMemoryObject(MemoryObject):
@@ -264,10 +297,10 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
264297

265298
is_reconstructor = constructor_kind.qualname == "copyreg._reconstructor"
266299
is_reduce_user_type = (
267-
len(self.reduce_value[1]) == 3
268-
and isinstance(self.reduce_value[1][0], type(self.obj))
269-
and self.reduce_value[1][1] is object
270-
and self.reduce_value[1][2] is None
300+
len(self.reduce_value[1]) == 3
301+
and isinstance(self.reduce_value[1][0], type(self.obj))
302+
and self.reduce_value[1][1] is object
303+
and self.reduce_value[1][2] is None
271304
)
272305
is_reduce_ex_user_type = len(self.reduce_value[1]) == 1 and isinstance(
273306
self.reduce_value[1][0], type(self.obj)
@@ -294,8 +327,8 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
294327
len(inspect.signature(init_method).parameters),
295328
)
296329
if (
297-
not init_from_object
298-
and len(inspect.signature(init_method).parameters) == 1
330+
not init_from_object
331+
and len(inspect.signature(init_method).parameters) == 1
299332
) or init_from_object:
300333
logging.debug("init with one argument! %s", init_method)
301334
constructor_arguments = []
@@ -317,9 +350,9 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
317350
if is_reconstructor and is_user_type:
318351
constructor_arguments = self.reduce_value[1]
319352
if (
320-
len(constructor_arguments) == 3
321-
and constructor_arguments[-1] is None
322-
and constructor_arguments[-2] == object
353+
len(constructor_arguments) == 3
354+
and constructor_arguments[-1] is None
355+
and constructor_arguments[-2] == object
323356
):
324357
del constructor_arguments[1:]
325358
callable_constructor = object.__new__
@@ -392,6 +425,12 @@ def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
392425
return ListMemoryObject
393426
return None
394427

428+
class NdarrayMemoryObjectProvider(MemoryObjectProvider):
429+
@staticmethod
430+
def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
431+
if type(obj) == np.ndarray:
432+
return NdarrayMemoryObject
433+
return None
395434

396435
class DictMemoryObjectProvider(MemoryObjectProvider):
397436
@staticmethod
@@ -425,6 +464,7 @@ def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
425464
return None
426465

427466

467+
428468
class ReprMemoryObjectProvider(MemoryObjectProvider):
429469
@staticmethod
430470
def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
@@ -450,6 +490,7 @@ class PythonSerializer:
450490
visited: Set[PythonId] = set()
451491

452492
providers: List[MemoryObjectProvider] = [
493+
NdarrayMemoryObjectProvider,
453494
ListMemoryObjectProvider,
454495
DictMemoryObjectProvider,
455496
IteratorMemoryObjectProvider,

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/utils.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ def get_constructor_info(constructor: object, obj: object) -> TypeInfo:
9595

9696

9797
def has_reduce(py_object: object) -> bool:
98+
if get_kind(py_object).module == "numpy":
99+
return False
98100
reduce = getattr(py_object, "__reduce__", None)
99101
if reduce is None:
100102
return False
@@ -161,6 +163,10 @@ def check_eval(py_object: object) -> bool:
161163
except Exception:
162164
return False
163165

166+
try:
167+
import numpy as np
168+
except ImportError:
169+
pass
164170

165171
def has_repr(py_object: object) -> bool:
166172
reprable_types = [
@@ -171,11 +177,13 @@ def has_repr(py_object: object) -> bool:
171177
bytes,
172178
bytearray,
173179
str,
174-
# tuple,
175-
# list,
176-
# dict,
177-
# set,
178-
# frozenset,
180+
np.int64,
181+
np.int32,
182+
np.int16,
183+
np.int8,
184+
np.float32,
185+
np.float16,
186+
np.float64,
179187
type,
180188
]
181189
if type(py_object) in reprable_types:

‎utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444

4545

4646
def _update_states(
47-
init_memory_dump: MemoryDump, before_memory_dump: MemoryDump
47+
init_memory_dump: MemoryDump, before_memory_dump: MemoryDump
4848
) -> MemoryDump:
4949
for id_, obj in before_memory_dump.objects.items():
5050
if id_ in init_memory_dump.objects:
@@ -87,8 +87,9 @@ def add_imports(imports: Iterable[str]):
8787
globals()[submodule_name] = importlib.import_module(
8888
submodule_name
8989
)
90-
except ModuleNotFoundError:
90+
except ModuleNotFoundErrorase:
9191
logging.warning("Import submodule %s failed", submodule_name)
92+
raise e
9293
logging.debug("Submodule #%d: OK", i)
9394

9495
def run_function(self, request: ExecutionRequest) -> ExecutionResponse:
@@ -115,6 +116,9 @@ def run_reduce_function(self, request: ExecutionRequest) -> ExecutionResponse:
115116
self.add_imports(request.imports)
116117
loader.add_syspaths(request.syspaths)
117118
loader.add_imports(request.imports)
119+
except ModuleNotFoundError as _:
120+
logging.debug("Error \n%s", traceback.format_exc())
121+
return ExecutionFailResponse("fail", traceback.format_exc())
118122
except Exception as _:
119123
logging.debug("Error \n%s", traceback.format_exc())
120124
return ExecutionFailResponse("fail", traceback.format_exc())
@@ -246,9 +250,9 @@ def run_pickle_function(self, request: ExecutionRequest) -> ExecutionResponse:
246250

247251

248252
def _serialize_state(
249-
args: List[Any],
250-
kwargs: Dict[str, Any],
251-
result: Any = None,
253+
args: List[Any],
254+
kwargs: Dict[str, Any],
255+
result: Any = None,
252256
) -> Tuple[List[PythonId], Dict[str, PythonId], PythonId, MemoryDump, str]:
253257
"""Serialize objects from args, kwargs and result.
254258
@@ -267,13 +271,13 @@ def _serialize_state(
267271

268272

269273
def _run_calculate_function_value(
270-
function: types.FunctionType,
271-
args: List[Any],
272-
kwargs: Dict[str, Any],
273-
fullpath: str,
274-
state_init: str,
275-
tracer: UtTracer,
276-
state_assertions: bool,
274+
function: types.FunctionType,
275+
args: List[Any],
276+
kwargs: Dict[str, Any],
277+
fullpath: str,
278+
state_init: str,
279+
tracer: UtTracer,
280+
state_assertions: bool,
277281
) -> ExecutionResponse:
278282
"""Calculate function evaluation result.
279283

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /