| OLD | NEW |
|---|
| 1 | 1 |
|
| 2 /* Execute compiled code */ | 2 /* Execute compiled code */ |
|
| 3 | 3 |
|
| 4 /* XXX TO DO: | 4 /* XXX TO DO: |
|
| 5 XXX speed up searching for keywords by using a dictionary | 5 XXX speed up searching for keywords by using a dictionary |
|
| 6 XXX document it! | 6 XXX document it! |
|
| 7 */ | 7 */ |
|
| 8 | 8 |
|
| 9 /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it | 9 /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it |
|
| 10 generally C. */ | 10 generally C. */ |
|
| (...skipping 332 matching lines...) | | Loading... |
| 343 static PyObject * load_args(PyObject ***, int); | 343 static PyObject * load_args(PyObject ***, int); |
|
| 344 | 344 |
|
| 345 #ifdef WITH_LLVM | 345 #ifdef WITH_LLVM |
|
| 346 static inline void mark_called(PyCodeObject *co); | 346 static inline void mark_called(PyCodeObject *co); |
|
| 347 static inline int maybe_compile(PyCodeObject *co, PyFrameObject *f); | 347 static inline int maybe_compile(PyCodeObject *co, PyFrameObject *f); |
|
| 348 | 348 |
|
| 349 /* Record data for use in generating optimized machine code. */ | 349 /* Record data for use in generating optimized machine code. */ |
|
| 350 static void record_type(PyCodeObject *, int, int, int, PyObject *); | 350 static void record_type(PyCodeObject *, int, int, int, PyObject *); |
|
| 351 static void record_func(PyCodeObject *, int, int, int, PyObject *); | 351 static void record_func(PyCodeObject *, int, int, int, PyObject *); |
|
| 352 static void record_object(PyCodeObject *, int, int, int, PyObject *); | 352 static void record_object(PyCodeObject *, int, int, int, PyObject *); |
|
| 353 static void inc_feedback_counter(PyCodeObject *, int, int, int); | 353 static void inc_feedback_counter(PyCodeObject *, int, int, int, int); |
|
| 354 #endif /* WITH_LLVM */ | 354 #endif /* WITH_LLVM */ |
|
| 355 | 355 |
|
| 356 int _Py_ProfilingPossible = 0; | 356 int _Py_ProfilingPossible = 0; |
|
| 357 | 357 |
|
| 358 /* Keep this in sync with llvm_fbuilder.cc */ | 358 /* Keep this in sync with llvm_fbuilder.cc */ |
|
| 359 #define CALL_FLAG_VAR 1 | 359 #define CALL_FLAG_VAR 1 |
|
| 360 #define CALL_FLAG_KW 2 | 360 #define CALL_FLAG_KW 2 |
|
| 361 | 361 |
|
| 362 #ifdef LLTRACE | 362 #ifdef LLTRACE |
|
| 363 static int lltrace; | 363 static int lltrace; |
|
| (...skipping 62 matching lines...) | | Loading... |
| 426 PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER | 426 PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER |
|
| 427 PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION | 427 PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION |
|
| 428 PCALL_METHOD > PCALL_BOUND_METHOD | 428 PCALL_METHOD > PCALL_BOUND_METHOD |
|
| 429 */ | 429 */ |
|
| 430 | 430 |
|
| 431 #define PCALL(POS) pcall[POS]++ | 431 #define PCALL(POS) pcall[POS]++ |
|
| 432 | 432 |
|
| 433 PyObject * | 433 PyObject * |
|
| 434 PyEval_GetCallStats(PyObject *self) | 434 PyEval_GetCallStats(PyObject *self) |
|
| 435 { | 435 { |
|
| 436 » return Py_BuildValue("iiiiiiiiiii", | 436 » return Py_BuildValue("iiiiiiiiiiiii", |
|
| 437 pcall[0], pcall[1], pcall[2], pcall[3], | 437 pcall[0], pcall[1], pcall[2], pcall[3], |
|
| 438 pcall[4], pcall[5], pcall[6], pcall[7], | 438 pcall[4], pcall[5], pcall[6], pcall[7], |
|
| 439 pcall[8], pcall[9], pcall[10]); | 439 pcall[8], pcall[9], pcall[10]); |
|
| 440 } | 440 } |
|
| 441 #else | 441 #else |
|
| 442 #define PCALL(O) | 442 #define PCALL(O) |
|
| 443 | 443 |
|
| 444 PyObject * | 444 PyObject * |
|
| 445 PyEval_GetCallStats(PyObject *self) | 445 PyEval_GetCallStats(PyObject *self) |
|
| 446 { | 446 { |
|
| (...skipping 514 matching lines...) | | Loading... |
| 961 #define JUMPBY(x) (next_instr += (x)) | 961 #define JUMPBY(x) (next_instr += (x)) |
|
| 962 | 962 |
|
| 963 /* Feedback-gathering macros */ | 963 /* Feedback-gathering macros */ |
|
| 964 #ifdef WITH_LLVM | 964 #ifdef WITH_LLVM |
|
| 965 #define RECORD_TYPE(arg_index, obj) \ | 965 #define RECORD_TYPE(arg_index, obj) \ |
|
| 966 if(rec_feedback){record_type(co, opcode, f->f_lasti, arg_index, obj);} | 966 if(rec_feedback){record_type(co, opcode, f->f_lasti, arg_index, obj);} |
|
| 967 #define RECORD_OBJECT(arg_index, obj) \ | 967 #define RECORD_OBJECT(arg_index, obj) \ |
|
| 968 if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);} | 968 if(rec_feedback){record_object(co, opcode, f->f_lasti, arg_index, obj);} |
|
| 969 #define RECORD_FUNC(obj) \ | 969 #define RECORD_FUNC(obj) \ |
|
| 970 if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);} | 970 if(rec_feedback){record_func(co, opcode, f->f_lasti, 0, obj);} |
|
| 971 #define INC_COUNTER(arg_index, counter_id) \ |
|
| 972 if (rec_feedback) { \ |
|
| 973 inc_feedback_counter(co, opcode, f->f_lasti, arg_index, \ |
|
| 974 counter_id); \ |
|
| 975 } |
|
| 971 #define RECORD_TRUE() \ | 976 #define RECORD_TRUE() \ |
|
| 972 » if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti, PY_FDO_JUM
P_TRUE);} | 977 » INC_COUNTER(0, PY_FDO_JUMP_TRUE) |
|
| 973 #define RECORD_FALSE() \ | 978 #define RECORD_FALSE() \ |
|
| 974 » if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti, PY_FDO_JUM
P_FALSE);} | 979 » INC_COUNTER(0, PY_FDO_JUMP_FALSE) |
|
| 975 #define RECORD_NONBOOLEAN() \ | 980 #define RECORD_NONBOOLEAN() \ |
|
| 976 » if(rec_feedback){inc_feedback_counter(co, opcode, f->f_lasti, PY_FDO_JUM
P_NON_BOOLEAN);} | 981 » INC_COUNTER(0, PY_FDO_JUMP_NON_BOOLEAN) |
|
| 977 #define UPDATE_HOTNESS_JABS() \ | 982 #define UPDATE_HOTNESS_JABS() \ |
|
| 978 do { if (oparg <= f->f_lasti) ++co->co_hotness; } while (0) | 983 do { if (oparg <= f->f_lasti) ++co->co_hotness; } while (0) |
|
| 979 #else | 984 #else |
|
| 980 #define RECORD_TYPE(arg_index, obj) | 985 #define RECORD_TYPE(arg_index, obj) |
|
| 981 #define RECORD_OBJECT(arg_index, obj) | 986 #define RECORD_OBJECT(arg_index, obj) |
|
| 982 #define RECORD_FUNC(obj) | 987 #define RECORD_FUNC(obj) |
|
| 988 #define INC_COUNTER(arg_index, counter_id) |
|
| 983 #define RECORD_TRUE() | 989 #define RECORD_TRUE() |
|
| 984 #define RECORD_FALSE() | 990 #define RECORD_FALSE() |
|
| 985 #define RECORD_NONBOOLEAN() | 991 #define RECORD_NONBOOLEAN() |
|
| 986 #define UPDATE_HOTNESS_JABS() | 992 #define UPDATE_HOTNESS_JABS() |
|
| 987 #endif /* WITH_LLVM */ | 993 #endif /* WITH_LLVM */ |
|
| 988 | 994 |
|
| 989 | 995 |
|
| 990 /* OpCode prediction macros | 996 /* OpCode prediction macros |
|
| 991 Some opcodes tend to come in pairs thus making it possible to | 997 Some opcodes tend to come in pairs thus making it possible to |
|
| 992 predict the second code when the first is run. For example, | 998 predict the second code when the first is run. For example, |
|
| (...skipping 128 matching lines...) | | Loading... |
| 1121 retval = co->co_native_function(f); | 1127 retval = co->co_native_function(f); |
|
| 1122 goto exit_eval_frame; | 1128 goto exit_eval_frame; |
|
| 1123 } | 1129 } |
|
| 1124 } | 1130 } |
|
| 1125 | 1131 |
|
| 1126 if (bail_reason != _PYFRAME_NO_BAIL) { | 1132 if (bail_reason != _PYFRAME_NO_BAIL) { |
|
| 1127 #ifdef Py_WITH_INSTRUMENTATION | 1133 #ifdef Py_WITH_INSTRUMENTATION |
|
| 1128 bail_count_stats->RecordBail(f, bail_reason); | 1134 bail_count_stats->RecordBail(f, bail_reason); |
|
| 1129 #endif | 1135 #endif |
|
| 1130 if (_Py_BailError) { | 1136 if (_Py_BailError) { |
|
| 1131 » » » PyErr_SetString(PyExc_RuntimeError, | 1137 » » » /* When we bail, we set f_lasti to the current opcode |
|
| 1132 » » "bailed to the interpreter"); | 1138 » » » * minus 1, so we add one back. */ |
|
| 1139 » » » int lasti = f->f_lasti + 1; |
|
| 1140 » » » PyErr_Format(PyExc_RuntimeError, "bailed to the " |
|
| 1141 » » » » "interpreter at opcode index %d", lasti); |
|
| 1133 goto exit_eval_frame; | 1142 goto exit_eval_frame; |
|
| 1134 } | 1143 } |
|
| 1135 } | 1144 } |
|
| 1136 | 1145 |
|
| 1137 » // Create co_runtime_feedback now that we're about to use it. You | 1146 » /* Create co_runtime_feedback now that we're about to use it. You |
|
| 1138 » // might think this would cause a problem if the user flips | 1147 » * might think this would cause a problem if the user flips |
|
| 1139 » // Py_JitControl from "never" to "whenhot", but since the value of | 1148 » * Py_JitControl from "never" to "whenhot", but since the value of |
|
| 1140 » // rec_feedback is constant for the duration of this frame's execution, | 1149 » * rec_feedback is constant for the duration of this frame's execution, |
|
| 1141 » // we will not accidentally try to record feedback without initializing | 1150 » * we will not accidentally try to record feedback without initializing |
|
| 1142 » // co_runtime_feedback. | 1151 » * co_runtime_feedback. */ |
|
| 1143 if (rec_feedback && co->co_runtime_feedback == NULL) { | 1152 if (rec_feedback && co->co_runtime_feedback == NULL) { |
|
| 1144 #if Py_WITH_INSTRUMENTATION | 1153 #if Py_WITH_INSTRUMENTATION |
|
| 1145 feedback_map_counter->IncCounter(); | 1154 feedback_map_counter->IncCounter(); |
|
| 1146 #endif | 1155 #endif |
|
| 1147 co->co_runtime_feedback = PyFeedbackMap_New(); | 1156 co->co_runtime_feedback = PyFeedbackMap_New(); |
|
| 1148 } | 1157 } |
|
| 1149 #endif /* WITH_LLVM */ | 1158 #endif /* WITH_LLVM */ |
|
| 1150 | 1159 |
|
| 1151 switch (bail_reason) { | 1160 switch (bail_reason) { |
|
| 1152 case _PYFRAME_NO_BAIL: | 1161 case _PYFRAME_NO_BAIL: |
|
| (...skipping 1309 matching lines...) | | Loading... |
| 2462 RECORD_TYPE(0, v); | 2471 RECORD_TYPE(0, v); |
|
| 2463 x = PyObject_GetAttr(v, w); | 2472 x = PyObject_GetAttr(v, w); |
|
| 2464 Py_DECREF(v); | 2473 Py_DECREF(v); |
|
| 2465 SET_TOP(x); | 2474 SET_TOP(x); |
|
| 2466 if (x == NULL) { | 2475 if (x == NULL) { |
|
| 2467 why = UNWIND_EXCEPTION; | 2476 why = UNWIND_EXCEPTION; |
|
| 2468 break; | 2477 break; |
|
| 2469 } | 2478 } |
|
| 2470 DISPATCH(); | 2479 DISPATCH(); |
|
| 2471 | 2480 |
|
| 2481 TARGET(LOAD_METHOD) |
|
| 2482 w = GETITEM(names, oparg); |
|
| 2483 v = TOP(); |
|
| 2484 RECORD_TYPE(0, v); |
|
| 2485 x = PyObject_GetMethod(v, w); |
|
| 2486 if (((long)x) & 1) { |
|
| 2487 /* Record that this was a regular method. */ |
|
| 2488 INC_COUNTER(1, PY_FDO_LOADMETHOD_METHOD); |
|
| 2489 |
|
| 2490 /* Set up the stack as if self were the first |
|
| 2491 * argument to the unbound method. */ |
|
| 2492 x = (PyObject*)(((Py_uintptr_t)x) & ~1); |
|
| 2493 SET_TOP(x); |
|
| 2494 PUSH(v); |
|
| 2495 } |
|
| 2496 else { |
|
| 2497 /* Record that this was not a regular method. |
|
| 2498 */ |
|
| 2499 INC_COUNTER(1, PY_FDO_LOADMETHOD_OTHER); |
|
| 2500 |
|
| 2501 /* Set up the stack as if there were no self |
|
| 2502 * argument. Pad the stack with a NULL so |
|
| 2503 * CALL_METHOD knows the method is bound. */ |
|
| 2504 Py_DECREF(v); |
|
| 2505 SET_TOP(NULL); |
|
| 2506 PUSH(x); |
|
| 2507 } |
|
| 2508 if (x == NULL) { |
|
| 2509 why = UNWIND_EXCEPTION; |
|
| 2510 break; |
|
| 2511 } |
|
| 2512 DISPATCH(); |
|
| 2513 |
|
| 2472 TARGET(COMPARE_OP) | 2514 TARGET(COMPARE_OP) |
|
| 2473 w = POP(); | 2515 w = POP(); |
|
| 2474 v = TOP(); | 2516 v = TOP(); |
|
| 2475 RECORD_TYPE(0, v); | 2517 RECORD_TYPE(0, v); |
|
| 2476 RECORD_TYPE(1, w); | 2518 RECORD_TYPE(1, w); |
|
| 2477 if (PyInt_CheckExact(w) && PyInt_CheckExact(v)) { | 2519 if (PyInt_CheckExact(w) && PyInt_CheckExact(v)) { |
|
| 2478 /* INLINE: cmp(int, int) */ | 2520 /* INLINE: cmp(int, int) */ |
|
| 2479 register long a, b; | 2521 register long a, b; |
|
| 2480 register int res; | 2522 register int res; |
|
| 2481 a = PyInt_AS_LONG(v); | 2523 a = PyInt_AS_LONG(v); |
|
| (...skipping 356 matching lines...) | | Loading... |
| 2838 * arguments. */ | 2880 * arguments. */ |
|
| 2839 stack_pointer -= num_stack_slots; | 2881 stack_pointer -= num_stack_slots; |
|
| 2840 PUSH(x); | 2882 PUSH(x); |
|
| 2841 if (x == NULL) { | 2883 if (x == NULL) { |
|
| 2842 why = UNWIND_EXCEPTION; | 2884 why = UNWIND_EXCEPTION; |
|
| 2843 break; | 2885 break; |
|
| 2844 } | 2886 } |
|
| 2845 DISPATCH(); | 2887 DISPATCH(); |
|
| 2846 } | 2888 } |
|
| 2847 | 2889 |
|
| 2890 TARGET(CALL_METHOD) |
|
| 2891 { |
|
| 2892 int num_args, num_kwargs, num_stack_slots; |
|
| 2893 PyObject *method; |
|
| 2894 PY_LOG_TSC_EVENT(CALL_START_EVAL); |
|
| 2895 PCALL(PCALL_ALL); |
|
| 2896 num_args = oparg & 0xff; |
|
| 2897 num_kwargs = (oparg>>8) & 0xff; |
|
| 2898 /* +1 for the actual function object, +1 for self. */ |
|
| 2899 num_stack_slots = num_args + 2 * num_kwargs + 1 + 1; |
|
| 2900 method = stack_pointer[-num_stack_slots]; |
|
| 2901 if (method != NULL) { |
|
| 2902 /* We loaded an unbound method. Adjust |
|
| 2903 * num_args to include the self argument pushed |
|
| 2904 * on the stack after the method. */ |
|
| 2905 num_args++; |
|
| 2906 } |
|
| 2907 #ifdef WITH_LLVM |
|
| 2908 else { |
|
| 2909 /* The method is really in the next slot. */ |
|
| 2910 method = stack_pointer[-num_stack_slots+1]; |
|
| 2911 } |
|
| 2912 /* We'll focus on these simple calls with only |
|
| 2913 * positional args for now (since they're easy to |
|
| 2914 * implement). */ |
|
| 2915 if (num_kwargs == 0) { |
|
| 2916 RECORD_FUNC(method); |
|
| 2917 } |
|
| 2918 #endif |
|
| 2919 x = _PyEval_CallFunction(stack_pointer, |
|
| 2920 num_args, num_kwargs); |
|
| 2921 /* Clear the stack of the function object and |
|
| 2922 * arguments. */ |
|
| 2923 stack_pointer -= num_stack_slots; |
|
| 2924 PUSH(x); |
|
| 2925 if (x == NULL) { |
|
| 2926 why = UNWIND_EXCEPTION; |
|
| 2927 break; |
|
| 2928 } |
|
| 2929 DISPATCH(); |
|
| 2930 } |
|
| 2931 |
|
| 2848 TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) | 2932 TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) |
|
| 2849 TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) | 2933 TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) |
|
| 2850 TARGET_WITH_IMPL(CALL_FUNCTION_VAR_KW, _call_function_var_kw) | 2934 TARGET_WITH_IMPL(CALL_FUNCTION_VAR_KW, _call_function_var_kw) |
|
| 2851 _call_function_var_kw: | 2935 _call_function_var_kw: |
|
| 2852 { | 2936 { |
|
| 2853 int num_args, num_kwargs, num_stack_slots, flags; | 2937 int num_args, num_kwargs, num_stack_slots, flags; |
|
| 2854 PY_LOG_TSC_EVENT(CALL_START_EVAL); | 2938 PY_LOG_TSC_EVENT(CALL_START_EVAL); |
|
| 2855 /* TODO(jyasskin): Add feedback gathering. */ | 2939 /* TODO(jyasskin): Add feedback gathering. */ |
|
| 2856 num_args = oparg & 0xff; | 2940 num_args = oparg & 0xff; |
|
| 2857 num_kwargs = (oparg>>8) & 0xff; | 2941 num_kwargs = (oparg>>8) & 0xff; |
|
| (...skipping 1352 matching lines...) | | Loading... |
| 4210 else if (PyClass_Check(func)) | 4294 else if (PyClass_Check(func)) |
|
| 4211 return " constructor"; | 4295 return " constructor"; |
|
| 4212 else if (PyInstance_Check(func)) { | 4296 else if (PyInstance_Check(func)) { |
|
| 4213 return " instance"; | 4297 return " instance"; |
|
| 4214 } else { | 4298 } else { |
|
| 4215 return " object"; | 4299 return " object"; |
|
| 4216 } | 4300 } |
|
| 4217 } | 4301 } |
|
| 4218 | 4302 |
|
| 4219 static void | 4303 static void |
|
| 4220 err_args(PyObject *func, int flags, int min_arity, int max_arity, int nargs) | 4304 err_args(PyMethodDef *ml, int flags, int min_arity, int max_arity, int nargs) |
|
| 4221 { | 4305 { |
|
| 4222 if (min_arity != max_arity) | 4306 if (min_arity != max_arity) |
|
| 4223 PyErr_Format(PyExc_TypeError, | 4307 PyErr_Format(PyExc_TypeError, |
|
| 4224 "%.200s() takes %d-%d arguments (%d given)", | 4308 "%.200s() takes %d-%d arguments (%d given)", |
|
| 4225 » » » » ((PyCFunctionObject *)func)->m_ml->ml_name, | 4309 » » » » ml->ml_name, min_arity, max_arity, nargs); |
|
| 4226 » » » » min_arity, max_arity, nargs); |
|
| 4227 else if (min_arity == 0) | 4310 else if (min_arity == 0) |
|
| 4228 PyErr_Format(PyExc_TypeError, | 4311 PyErr_Format(PyExc_TypeError, |
|
| 4229 "%.200s() takes no arguments (%d given)", | 4312 "%.200s() takes no arguments (%d given)", |
|
| 4230 » » » ((PyCFunctionObject *)func)->m_ml->ml_name, | 4313 » » » ml->ml_name, nargs); |
|
| 4231 » » » nargs); |
|
| 4232 else if (min_arity == 1) | 4314 else if (min_arity == 1) |
|
| 4233 PyErr_Format(PyExc_TypeError, | 4315 PyErr_Format(PyExc_TypeError, |
|
| 4234 "%.200s() takes exactly one argument (%d given)", | 4316 "%.200s() takes exactly one argument (%d given)", |
|
| 4235 » » » ((PyCFunctionObject *)func)->m_ml->ml_name, | 4317 » » » ml->ml_name, nargs); |
|
| 4236 » » » nargs); |
|
| 4237 else | 4318 else |
|
| 4238 PyErr_Format(PyExc_TypeError, | 4319 PyErr_Format(PyExc_TypeError, |
|
| 4239 "%.200s() takes exactly %d arguments (%d given)", | 4320 "%.200s() takes exactly %d arguments (%d given)", |
|
| 4240 » » » ((PyCFunctionObject *)func)->m_ml->ml_name, | 4321 » » » ml->ml_name, min_arity, nargs); |
|
| 4241 » » » min_arity, |
|
| 4242 » » » nargs); |
|
| 4243 } | 4322 } |
|
| 4244 | 4323 |
|
| 4245 #ifdef WITH_LLVM | 4324 #ifdef WITH_LLVM |
|
| 4246 static inline void | 4325 static inline void |
|
| 4247 mark_called(PyCodeObject *co) | 4326 mark_called(PyCodeObject *co) |
|
| 4248 { | 4327 { |
|
| 4249 co->co_hotness += 10; | 4328 co->co_hotness += 10; |
|
| 4250 } | 4329 } |
|
| 4251 | 4330 |
|
| 4252 // Decide whether to compile a code object's bytecode to native code based on | 4331 // Decide whether to compile a code object's bytecode to native code based on |
|
| (...skipping 153 matching lines...) | | Loading... |
| 4406 but the caller must adjust the stack pointer down by (na + 2*nk + 1). | 4485 but the caller must adjust the stack pointer down by (na + 2*nk + 1). |
|
| 4407 We put the stack change in the caller so that LLVM's optimizers can | 4486 We put the stack change in the caller so that LLVM's optimizers can |
|
| 4408 see it. */ | 4487 see it. */ |
|
| 4409 PyObject * | 4488 PyObject * |
|
| 4410 _PyEval_CallFunction(PyObject **stack_pointer, int na, int nk) | 4489 _PyEval_CallFunction(PyObject **stack_pointer, int na, int nk) |
|
| 4411 { | 4490 { |
|
| 4412 int n = na + 2 * nk; | 4491 int n = na + 2 * nk; |
|
| 4413 PyObject **pfunc = stack_pointer - n - 1; | 4492 PyObject **pfunc = stack_pointer - n - 1; |
|
| 4414 PyObject *func = *pfunc; | 4493 PyObject *func = *pfunc; |
|
| 4415 PyObject *x, *w; | 4494 PyObject *x, *w; |
|
| 4495 PyMethodDef *ml = NULL; |
|
| 4496 PyObject *self = NULL; |
|
| 4416 | 4497 |
|
| 4417 » /* Always dispatch PyCFunction first, because these are | 4498 » /* Always dispatch PyCFunction and PyMethodDescr first, because these |
|
| 4418 » presumed to be the most frequent callable object. | 4499 » both represent methods written in C, and are presumed to be the most |
|
| 4419 » */ | 4500 » frequently called objects. |
|
| 4420 » if (PyCFunction_Check(func) && nk == 0) { | 4501 » */ |
|
| 4421 » » int flags = PyCFunction_GET_FLAGS(func); | 4502 » if (nk == 0) { |
|
| 4503 » » if (PyCFunction_Check(func)) { |
|
| 4504 » » » ml = PyCFunction_GET_METHODDEF(func); |
|
| 4505 » » » self = PyCFunction_GET_SELF(func); |
|
| 4506 » » } |
|
| 4507 » » else if (PyMethodDescr_Check(func)) { |
|
| 4508 » » » ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4509 » » » /* The first argument on the stack (the one immediately |
|
| 4510 » » » after func) is self. We borrow the reference from |
|
| 4511 » » » the stack, which gets cleaned off and decrefed at |
|
| 4512 » » » the end of the function. |
|
| 4513 » » » */ |
|
| 4514 » » » self = pfunc[1]; |
|
| 4515 » » » na--; |
|
| 4516 » » } |
|
| 4517 » } |
|
| 4518 » if (ml != NULL) { |
|
| 4519 » » int flags = ml->ml_flags; |
|
| 4422 PyThreadState *tstate = PyThreadState_GET(); | 4520 PyThreadState *tstate = PyThreadState_GET(); |
|
| 4423 | 4521 |
|
| 4424 PCALL(PCALL_CFUNCTION); | 4522 PCALL(PCALL_CFUNCTION); |
|
| 4425 if (flags & METH_ARG_RANGE) { | 4523 if (flags & METH_ARG_RANGE) { |
|
| 4426 » » » PyCFunction meth = PyCFunction_GET_FUNCTION(func); | 4524 » » » PyCFunction meth = ml->ml_meth; |
|
| 4427 » » » PyObject *self = PyCFunction_GET_SELF(func); | 4525 » » » int min_arity = ml->ml_min_arity; |
|
| 4428 » » » int min_arity = PyCFunction_GET_MIN_ARITY(func); | 4526 » » » int max_arity = ml->ml_max_arity; |
|
| 4429 » » » int max_arity = PyCFunction_GET_MAX_ARITY(func); |
|
| 4430 PyObject *args[PY_MAX_ARITY] = {NULL}; | 4527 PyObject *args[PY_MAX_ARITY] = {NULL}; |
|
| 4431 | 4528 |
|
| 4432 switch (na) { | 4529 switch (na) { |
|
| 4433 default: | 4530 default: |
|
| 4434 PyErr_BadInternalCall(); | 4531 PyErr_BadInternalCall(); |
|
| 4435 return NULL; | 4532 return NULL; |
|
| 4436 case 3: args[2] = EXT_POP(stack_pointer); | 4533 case 3: args[2] = EXT_POP(stack_pointer); |
|
| 4437 case 2: args[1] = EXT_POP(stack_pointer); | 4534 case 2: args[1] = EXT_POP(stack_pointer); |
|
| 4438 case 1: args[0] = EXT_POP(stack_pointer); | 4535 case 1: args[0] = EXT_POP(stack_pointer); |
|
| 4439 case 0: break; | 4536 case 0: break; |
|
| 4440 } | 4537 } |
|
| 4441 | 4538 |
|
| 4442 /* But wait, you ask, what about {un,bin}ary functions? | 4539 /* But wait, you ask, what about {un,bin}ary functions? |
|
| 4443 Aren't we passing more arguments than it expects? | 4540 Aren't we passing more arguments than it expects? |
|
| 4444 Yes, but C allows this. Go C. */ | 4541 Yes, but C allows this. Go C. */ |
|
| 4445 if (min_arity <= na && na <= max_arity) { | 4542 if (min_arity <= na && na <= max_arity) { |
|
| 4446 C_TRACE(x, (*(PyCFunctionThreeArgs)meth) | 4543 C_TRACE(x, (*(PyCFunctionThreeArgs)meth) |
|
| 4447 » » » » (self, args[0], args[1], args[2])); | 4544 » » » » (self, args[0], args[1], args[2])); |
|
| 4448 » » » » Py_XDECREF(args[0]); |
|
| 4449 » » » » Py_XDECREF(args[1]); |
|
| 4450 » » » » Py_XDECREF(args[2]); |
|
| 4451 } | 4545 } |
|
| 4452 else { | 4546 else { |
|
| 4453 » » » » err_args(func, flags, min_arity, max_arity, na); | 4547 » » » » err_args(ml, flags, min_arity, max_arity, na); |
|
| 4454 x = NULL; | 4548 x = NULL; |
|
| 4455 } | 4549 } |
|
| 4550 Py_XDECREF(args[0]); |
|
| 4551 Py_XDECREF(args[1]); |
|
| 4552 Py_XDECREF(args[2]); |
|
| 4456 } | 4553 } |
|
| 4457 else { | 4554 else { |
|
| 4458 PyObject *callargs; | 4555 PyObject *callargs; |
|
| 4459 callargs = load_args(&stack_pointer, na); | 4556 callargs = load_args(&stack_pointer, na); |
|
| 4460 » » » C_TRACE(x, PyCFunction_Call(func,callargs,NULL)); | 4557 » » » C_TRACE(x, PyMethodDef_Call(ml, self, callargs,NULL)); |
|
| 4461 Py_XDECREF(callargs); | 4558 Py_XDECREF(callargs); |
|
| 4462 } | 4559 } |
|
| 4463 } else { | 4560 } else { |
|
| 4464 if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) { | 4561 if (PyMethod_Check(func) && PyMethod_GET_SELF(func) != NULL) { |
|
| 4465 /* optimize access to bound methods */ | 4562 /* optimize access to bound methods */ |
|
| 4466 PyObject *self = PyMethod_GET_SELF(func); | 4563 PyObject *self = PyMethod_GET_SELF(func); |
|
| 4467 PCALL(PCALL_METHOD); | 4564 PCALL(PCALL_METHOD); |
|
| 4468 PCALL(PCALL_BOUND_METHOD); | 4565 PCALL(PCALL_BOUND_METHOD); |
|
| 4469 Py_INCREF(self); | 4566 Py_INCREF(self); |
|
| 4470 func = PyMethod_GET_FUNCTION(func); | 4567 func = PyMethod_GET_FUNCTION(func); |
|
| 4471 Py_INCREF(func); |
|
| 4472 Py_DECREF(*pfunc); | 4568 Py_DECREF(*pfunc); |
|
| 4473 *pfunc = self; | 4569 *pfunc = self; |
|
| 4474 na++; | 4570 na++; |
|
| 4475 n++; | 4571 n++; |
|
| 4476 » » } else | 4572 » » } |
|
| 4477 » » » Py_INCREF(func); | 4573 » » Py_INCREF(func); |
|
| 4478 if (PyFunction_Check(func)) | 4574 if (PyFunction_Check(func)) |
|
| 4479 x = fast_function(func, &stack_pointer, n, na, nk); | 4575 x = fast_function(func, &stack_pointer, n, na, nk); |
|
| 4480 else | 4576 else |
|
| 4481 x = do_call(func, &stack_pointer, na, nk); | 4577 x = do_call(func, &stack_pointer, na, nk); |
|
| 4482 Py_DECREF(func); | 4578 Py_DECREF(func); |
|
| 4483 } | 4579 } |
|
| 4484 | 4580 |
|
| 4485 /* Clear the stack of the function object. Also removes | 4581 /* Clear the stack of the function object. Also removes |
|
| 4486 the arguments in case they weren't consumed already | 4582 » the arguments in case they weren't consumed already |
|
| 4487 (fast_function() and err_args() leave them on the stack). | 4583 » (fast_function() and err_args() leave them on the stack). |
|
| 4488 » */ | 4584 » */ |
|
| 4489 while (stack_pointer > pfunc) { | 4585 while (stack_pointer > pfunc) { |
|
| 4490 w = EXT_POP(stack_pointer); | 4586 w = EXT_POP(stack_pointer); |
|
| 4491 Py_DECREF(w); | 4587 Py_DECREF(w); |
|
| 4492 PCALL(PCALL_POP); | 4588 PCALL(PCALL_POP); |
|
| 4493 } | 4589 } |
|
| 4494 return x; | 4590 return x; |
|
| 4495 } | 4591 } |
|
| 4496 | 4592 |
|
| 4497 /* Consumes a reference to each of the arguments and the called function, but | 4593 /* Consumes a reference to each of the arguments and the called function, but |
|
| 4498 the caller must adjust the stack pointer down by (na + 2*nk + 1) + 1 for a | 4594 the caller must adjust the stack pointer down by (na + 2*nk + 1) + 1 for a |
|
| (...skipping 189 matching lines...) | | Loading... |
| 4688 } | 4784 } |
|
| 4689 return args; | 4785 return args; |
|
| 4690 } | 4786 } |
|
| 4691 | 4787 |
|
| 4692 static PyObject * | 4788 static PyObject * |
|
| 4693 do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) | 4789 do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) |
|
| 4694 { | 4790 { |
|
| 4695 PyObject *callargs = NULL; | 4791 PyObject *callargs = NULL; |
|
| 4696 PyObject *kwdict = NULL; | 4792 PyObject *kwdict = NULL; |
|
| 4697 PyObject *result = NULL; | 4793 PyObject *result = NULL; |
|
| 4794 PyMethodDef *ml = NULL; |
|
| 4795 PyObject *self = NULL; |
|
| 4796 |
|
| 4797 if (PyCFunction_Check(func)) { |
|
| 4798 ml = PyCFunction_GET_METHODDEF(func); |
|
| 4799 self = PyCFunction_GET_SELF(func); |
|
| 4800 } |
|
| 4801 else if (PyMethodDescr_Check(func)) { |
|
| 4802 ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4803 self = (*pp_stack)[-(na + 2 * nk)]; |
|
| 4804 na--; |
|
| 4805 } |
|
| 4698 | 4806 |
|
| 4699 if (nk > 0) { | 4807 if (nk > 0) { |
|
| 4700 kwdict = update_keyword_args(NULL, nk, pp_stack, func); | 4808 kwdict = update_keyword_args(NULL, nk, pp_stack, func); |
|
| 4701 if (kwdict == NULL) | 4809 if (kwdict == NULL) |
|
| 4702 goto call_fail; | 4810 goto call_fail; |
|
| 4703 } | 4811 } |
|
| 4704 callargs = load_args(pp_stack, na); | 4812 callargs = load_args(pp_stack, na); |
|
| 4705 if (callargs == NULL) | 4813 if (callargs == NULL) |
|
| 4706 goto call_fail; | 4814 goto call_fail; |
|
| 4707 #ifdef CALL_PROFILE | 4815 #ifdef CALL_PROFILE |
|
| 4708 /* At this point, we have to look at the type of func to | 4816 /* At this point, we have to look at the type of func to |
|
| 4709 update the call stats properly. Do it here so as to avoid | 4817 update the call stats properly. Do it here so as to avoid |
|
| 4710 exposing the call stats machinery outside eval.cc. | 4818 exposing the call stats machinery outside eval.cc. |
|
| 4711 */ | 4819 */ |
|
| 4712 if (PyFunction_Check(func)) | 4820 if (PyFunction_Check(func)) |
|
| 4713 PCALL(PCALL_FUNCTION); | 4821 PCALL(PCALL_FUNCTION); |
|
| 4714 else if (PyMethod_Check(func)) | 4822 else if (PyMethod_Check(func)) |
|
| 4715 PCALL(PCALL_METHOD); | 4823 PCALL(PCALL_METHOD); |
|
| 4716 else if (PyType_Check(func)) | 4824 else if (PyType_Check(func)) |
|
| 4717 PCALL(PCALL_TYPE); | 4825 PCALL(PCALL_TYPE); |
|
| 4718 else if (PyCFunction_Check(func)) | 4826 else if (PyCFunction_Check(func)) |
|
| 4719 PCALL(PCALL_CFUNCTION); | 4827 PCALL(PCALL_CFUNCTION); |
|
| 4720 else | 4828 else |
|
| 4721 PCALL(PCALL_OTHER); | 4829 PCALL(PCALL_OTHER); |
|
| 4722 #endif | 4830 #endif |
|
| 4723 » if (PyCFunction_Check(func)) { | 4831 » if (ml) { |
|
| 4724 PyThreadState *tstate = PyThreadState_GET(); | 4832 PyThreadState *tstate = PyThreadState_GET(); |
|
| 4725 » » C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); | 4833 » » C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict)); |
|
| 4726 } | 4834 } |
|
| 4727 else | 4835 else |
|
| 4728 result = PyObject_Call(func, callargs, kwdict); | 4836 result = PyObject_Call(func, callargs, kwdict); |
|
| 4729 call_fail: | 4837 call_fail: |
|
| 4730 Py_XDECREF(callargs); | 4838 Py_XDECREF(callargs); |
|
| 4731 Py_XDECREF(kwdict); | 4839 Py_XDECREF(kwdict); |
|
| 4732 return result; | 4840 return result; |
|
| 4733 } | 4841 } |
|
| 4734 | 4842 |
|
| 4735 static PyObject * | 4843 static PyObject * |
|
| 4736 ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) | 4844 ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) |
|
| 4737 { | 4845 { |
|
| 4738 int nstar = 0; | 4846 int nstar = 0; |
|
| 4739 PyObject *callargs = NULL; | 4847 PyObject *callargs = NULL; |
|
| 4740 PyObject *stararg = NULL; | 4848 PyObject *stararg = NULL; |
|
| 4741 PyObject *kwdict = NULL; | 4849 PyObject *kwdict = NULL; |
|
| 4742 PyObject *result = NULL; | 4850 PyObject *result = NULL; |
|
| 4851 PyMethodDef *ml = NULL; |
|
| 4852 PyObject *self = NULL; |
|
| 4853 |
|
| 4854 if (PyCFunction_Check(func)) { |
|
| 4855 ml = PyCFunction_GET_METHODDEF(func); |
|
| 4856 self = PyCFunction_GET_SELF(func); |
|
| 4857 } |
|
| 4858 else if (PyMethodDescr_Check(func)) { |
|
| 4859 /* Only apply C calling optimization if self is on the stack |
|
| 4860 * and not the first element of callargs. */ |
|
| 4861 if (na > 0) { |
|
| 4862 ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4863 self = (*pp_stack)[-(na + 2 * nk)]; |
|
| 4864 na--; |
|
| 4865 } |
|
| 4866 } |
|
| 4743 | 4867 |
|
| 4744 if (flags & CALL_FLAG_KW) { | 4868 if (flags & CALL_FLAG_KW) { |
|
| 4745 kwdict = EXT_POP(*pp_stack); | 4869 kwdict = EXT_POP(*pp_stack); |
|
| 4746 if (!PyDict_Check(kwdict)) { | 4870 if (!PyDict_Check(kwdict)) { |
|
| 4747 PyObject *d; | 4871 PyObject *d; |
|
| 4748 d = PyDict_New(); | 4872 d = PyDict_New(); |
|
| 4749 if (d == NULL) | 4873 if (d == NULL) |
|
| 4750 goto ext_call_fail; | 4874 goto ext_call_fail; |
|
| 4751 if (PyDict_Update(d, kwdict) != 0) { | 4875 if (PyDict_Update(d, kwdict) != 0) { |
|
| 4752 Py_DECREF(d); | 4876 Py_DECREF(d); |
|
| (...skipping 55 matching lines...) | | Loading... |
| 4808 PCALL(PCALL_FUNCTION); | 4932 PCALL(PCALL_FUNCTION); |
|
| 4809 else if (PyMethod_Check(func)) | 4933 else if (PyMethod_Check(func)) |
|
| 4810 PCALL(PCALL_METHOD); | 4934 PCALL(PCALL_METHOD); |
|
| 4811 else if (PyType_Check(func)) | 4935 else if (PyType_Check(func)) |
|
| 4812 PCALL(PCALL_TYPE); | 4936 PCALL(PCALL_TYPE); |
|
| 4813 else if (PyCFunction_Check(func)) | 4937 else if (PyCFunction_Check(func)) |
|
| 4814 PCALL(PCALL_CFUNCTION); | 4938 PCALL(PCALL_CFUNCTION); |
|
| 4815 else | 4939 else |
|
| 4816 PCALL(PCALL_OTHER); | 4940 PCALL(PCALL_OTHER); |
|
| 4817 #endif | 4941 #endif |
|
| 4818 » if (PyCFunction_Check(func)) { | 4942 » if (ml) { |
|
| 4819 PyThreadState *tstate = PyThreadState_GET(); | 4943 PyThreadState *tstate = PyThreadState_GET(); |
|
| 4820 » » C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); | 4944 » » C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict)); |
|
| 4821 } | 4945 } |
|
| 4822 else | 4946 else |
|
| 4823 result = PyObject_Call(func, callargs, kwdict); | 4947 result = PyObject_Call(func, callargs, kwdict); |
|
| 4824 ext_call_fail: | 4948 ext_call_fail: |
|
| 4825 Py_XDECREF(callargs); | 4949 Py_XDECREF(callargs); |
|
| 4826 Py_XDECREF(kwdict); | 4950 Py_XDECREF(kwdict); |
|
| 4827 Py_XDECREF(stararg); | 4951 Py_XDECREF(stararg); |
|
| 4828 return result; | 4952 return result; |
|
| 4829 } | 4953 } |
|
| 4830 | 4954 |
|
| 4831 #ifdef WITH_LLVM | 4955 #ifdef WITH_LLVM |
|
| 4832 void inc_feedback_counter(PyCodeObject *co, int expected_opcode, | 4956 void inc_feedback_counter(PyCodeObject *co, int expected_opcode, |
|
| 4833 » » » int opcode_index, int counter_id) | 4957 » » » int opcode_index, int arg_index, int counter_id) |
|
| 4834 { | 4958 { |
|
| 4835 #ifndef NDEBUG | 4959 #ifndef NDEBUG |
|
| 4836 unsigned char actual_opcode = | 4960 unsigned char actual_opcode = |
|
| 4837 PyString_AS_STRING(co->co_code)[opcode_index]; | 4961 PyString_AS_STRING(co->co_code)[opcode_index]; |
|
| 4838 assert((actual_opcode == expected_opcode || | 4962 assert((actual_opcode == expected_opcode || |
|
| 4839 actual_opcode == EXTENDED_ARG) && | 4963 actual_opcode == EXTENDED_ARG) && |
|
| 4840 "Mismatch between feedback and opcode array."); | 4964 "Mismatch between feedback and opcode array."); |
|
| 4841 #endif /* NDEBUG */ | 4965 #endif /* NDEBUG */ |
|
| 4842 PyRuntimeFeedback &feedback = | 4966 PyRuntimeFeedback &feedback = |
|
| 4843 co->co_runtime_feedback->GetOrCreateFeedbackEntry( | 4967 co->co_runtime_feedback->GetOrCreateFeedbackEntry( |
|
| 4844 » » » opcode_index, 0); | 4968 » » » opcode_index, arg_index); |
|
| 4845 feedback.IncCounter(counter_id); | 4969 feedback.IncCounter(counter_id); |
|
| 4846 } | 4970 } |
|
| 4847 | 4971 |
|
| 4848 // Records func into the feedback array. | 4972 // Records func into the feedback array. |
|
| 4849 void record_func(PyCodeObject *co, int expected_opcode, | 4973 void record_func(PyCodeObject *co, int expected_opcode, |
|
| 4850 int opcode_index, int arg_index, PyObject *func) | 4974 int opcode_index, int arg_index, PyObject *func) |
|
| 4851 { | 4975 { |
|
| 4852 #ifndef NDEBUG | 4976 #ifndef NDEBUG |
|
| 4853 unsigned char actual_opcode = | 4977 unsigned char actual_opcode = |
|
| 4854 PyString_AS_STRING(co->co_code)[opcode_index]; | 4978 PyString_AS_STRING(co->co_code)[opcode_index]; |
|
| (...skipping 510 matching lines...) | | Loading... |
| 5365 Py_DECREF(l); | 5489 Py_DECREF(l); |
|
| 5366 return NULL; | 5490 return NULL; |
|
| 5367 } | 5491 } |
|
| 5368 PyList_SetItem(l, i, x); | 5492 PyList_SetItem(l, i, x); |
|
| 5369 } | 5493 } |
|
| 5370 return l; | 5494 return l; |
|
| 5371 #endif | 5495 #endif |
|
| 5372 } | 5496 } |
|
| 5373 | 5497 |
|
| 5374 #endif | 5498 #endif |
|
| OLD | NEW |
|---|