| LEFT | RIGHT |
|---|
| 1 | 1 |
|
| 2 /* Execute compiled code */ | 2 /* Execute compiled code */ |
|
| 3 | 3 |
|
| 4 /* XXX TO DO: | 4 /* XXX TO DO: |
|
| 5 XXX speed up searching for keywords by using a dictionary | 5 XXX speed up searching for keywords by using a dictionary |
|
| 6 XXX document it! | 6 XXX document it! |
|
| 7 */ | 7 */ |
|
| 8 | 8 |
|
| 9 /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it | 9 /* Note: this file will be compiled as C ifndef WITH_LLVM, so try to keep it |
|
| 10 generally C. */ | 10 generally C. */ |
|
| 11 | 11 |
|
| 12 /* enable more aggressive intra-module optimizations, where available */ | 12 /* enable more aggressive intra-module optimizations, where available */ |
|
| 13 #define PY_LOCAL_AGGRESSIVE | 13 #define PY_LOCAL_AGGRESSIVE |
|
| 14 | 14 |
|
| 15 #include "Python.h" | 15 #include "Python.h" |
|
| 16 | 16 |
|
| 17 #include "code.h" | 17 #include "code.h" |
|
| 18 #include "frameobject.h" | 18 #include "frameobject.h" |
|
| 19 #include "eval.h" | 19 #include "eval.h" |
|
| 20 #include "llvm_compile.h" |
|
| 21 #include "opcode.h" | 20 #include "opcode.h" |
|
| 22 #include "structmember.h" | 21 #include "structmember.h" |
|
| 23 | 22 |
|
| 23 #include "JIT/llvm_compile.h" |
|
| 24 #include "Util/EventTimer.h" | 24 #include "Util/EventTimer.h" |
|
| 25 #include <ctype.h> | 25 #include <ctype.h> |
|
| 26 | 26 |
|
| 27 #ifdef WITH_LLVM | 27 #ifdef WITH_LLVM |
|
| 28 #include "global_llvm_data.h" |
|
| 29 #include "_llvmfunctionobject.h" | 28 #include "_llvmfunctionobject.h" |
|
| 30 #include "llvm/Function.h" | 29 #include "llvm/Function.h" |
|
| 31 #include "llvm/Support/ManagedStatic.h" | 30 #include "llvm/Support/ManagedStatic.h" |
|
| 32 #include "llvm/Support/raw_ostream.h" | 31 #include "llvm/Support/raw_ostream.h" |
|
| 33 #include "Util/RuntimeFeedback.h" | 32 #include "JIT/global_llvm_data.h" |
|
| 33 #include "JIT/RuntimeFeedback.h" |
|
| 34 #include "Util/Stats.h" | 34 #include "Util/Stats.h" |
|
| 35 | 35 |
|
| 36 #include <set> | 36 #include <set> |
|
| 37 | 37 |
|
| 38 using llvm::errs; | 38 using llvm::errs; |
|
| 39 #endif | 39 #endif |
|
| 40 | 40 |
|
| 41 | 41 |
|
| 42 /* Make a call to stop the call overhead timer before going through to | 42 /* Make a call to stop the call overhead timer before going through to |
|
| 43 PyObject_Call. */ | 43 PyObject_Call. */ |
|
| 44 static inline PyObject * | 44 static inline PyObject * |
|
| 45 _PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) | 45 _PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) |
|
| 46 { | 46 { |
|
| 47 /* If we're calling a compiled C function with *args or **kwargs, then | 47 /* If we're calling a compiled C function with *args or **kwargs, then |
|
| 48 * this enum should be CALL_ENTER_C. However, most calls to C | 48 * this enum should be CALL_ENTER_C. However, most calls to C |
|
| 49 * functions are simple and are fast-tracked through the CALL_FUNCTION | 49 * functions are simple and are fast-tracked through the CALL_FUNCTION |
|
| 50 * opcode. */ | 50 * opcode. */ |
|
| 51 PY_LOG_TSC_EVENT(CALL_ENTER_PYOBJ_CALL); | 51 PY_LOG_TSC_EVENT(CALL_ENTER_PYOBJ_CALL); |
|
| 52 return PyObject_Call(func, arg, kw); | 52 return PyObject_Call(func, arg, kw); |
|
| 53 } | 53 } |
|
| 54 | 54 |
|
| 55 | 55 |
|
| 56 #ifdef Py_WITH_INSTRUMENTATION | 56 #ifdef Py_WITH_INSTRUMENTATION |
|
| 57 std::string |
|
| 58 _PyEval_GetCodeName(PyCodeObject *code) |
|
| 59 { |
|
| 60 std::string result; |
|
| 61 llvm::raw_string_ostream wrapper(result); |
|
| 62 |
|
| 63 wrapper << PyString_AsString(code->co_filename) |
|
| 64 << ":" << code->co_firstlineno << " " |
|
| 65 << "(" << PyString_AsString(code->co_name) << ")"; |
|
| 66 wrapper.flush(); |
|
| 67 return result; |
|
| 68 } |
|
| 69 |
|
| 70 // Collect statistics about how long we block for compilation to LLVM IR and to |
|
| 71 // machine code. |
|
| 72 class IrCompilationTimes : public DataVectorStats<int64_t> { |
|
| 73 public: |
|
| 74 IrCompilationTimes() |
|
| 75 : DataVectorStats<int64_t>("Time blocked for IR JIT in ns") {} |
|
| 76 }; |
|
| 77 class McCompilationTimes : public DataVectorStats<int64_t> { |
|
| 78 public: |
|
| 79 McCompilationTimes() |
|
| 80 : DataVectorStats<int64_t>("Time blocked for MC JIT in ns") {} |
|
| 81 }; |
|
| 82 |
|
| 83 static llvm::ManagedStatic<IrCompilationTimes> ir_compilation_times; |
|
| 84 static llvm::ManagedStatic<McCompilationTimes> mc_compilation_times; |
|
| 85 |
|
| 86 class FeedbackMapCounter { |
|
| 87 public: |
|
| 88 ~FeedbackMapCounter() { |
|
| 89 errs() << "\nFeedback maps created:\n"; |
|
| 90 errs() << "N: " << this->counter_ << "\n"; |
|
| 91 } |
|
| 92 |
|
| 93 void IncCounter() { |
|
| 94 this->counter_++; |
|
| 95 } |
|
| 96 |
|
| 97 private: |
|
| 98 unsigned counter_; |
|
| 99 }; |
|
| 100 |
|
| 101 static llvm::ManagedStatic<FeedbackMapCounter> feedback_map_counter; |
|
| 102 |
|
| 103 |
|
| 57 class HotnessTracker { | 104 class HotnessTracker { |
|
| 58 // llvm::DenseSet or llvm::SmallPtrSet may be better, but as of this | 105 // llvm::DenseSet or llvm::SmallPtrSet may be better, but as of this |
|
| 59 // writing, they don't seem to work with std::vector. | 106 // writing, they don't seem to work with std::vector. |
|
| 60 std::set<PyCodeObject*> hot_code_; | 107 std::set<PyCodeObject*> hot_code_; |
|
| 61 public: | 108 public: |
|
| 62 ~HotnessTracker(); | 109 ~HotnessTracker(); |
|
| 63 | 110 |
|
| 64 void AddHotCode(PyCodeObject *code_obj) { | 111 void AddHotCode(PyCodeObject *code_obj) { |
|
| 65 // This will prevent the code object from ever being | 112 // This will prevent the code object from ever being |
|
| 66 // deleted. | 113 // deleted. |
|
| 67 Py_INCREF(code_obj); | 114 Py_INCREF(code_obj); |
|
| 68 this->hot_code_.insert(code_obj); | 115 this->hot_code_.insert(code_obj); |
|
| 69 } | 116 } |
|
| 70 }; | 117 }; |
|
| 71 | 118 |
|
| 72 static bool | 119 static bool |
|
| 73 compare_hotness(const PyCodeObject *first, const PyCodeObject *second) | 120 compare_hotness(const PyCodeObject *first, const PyCodeObject *second) |
|
| 74 { | 121 { |
|
| 75 return first->co_hotness > second->co_hotness; | 122 return first->co_hotness > second->co_hotness; |
|
| 76 } | 123 } |
|
| 77 | 124 |
|
| 78 HotnessTracker::~HotnessTracker() | 125 HotnessTracker::~HotnessTracker() |
|
| 79 { | 126 { |
|
| 80 » errs() << "\nCode objects deemed hot (n=" << this->hot_code_.size() | 127 » errs() << "\nCode objects deemed hot:\n"; |
|
| 81 » << ")\n"; | 128 » errs() << "N: " << this->hot_code_.size() << "\n"; |
|
| 82 | 129 » errs() << "Function -> hotness score:\n"; |
|
| 83 » errs() << "Function -> hotness metric:\n"; |
|
| 84 std::vector<PyCodeObject*> to_sort(this->hot_code_.begin(), | 130 std::vector<PyCodeObject*> to_sort(this->hot_code_.begin(), |
|
| 85 this->hot_code_.end()); | 131 this->hot_code_.end()); |
|
| 86 std::sort(to_sort.begin(), to_sort.end(), compare_hotness); | 132 std::sort(to_sort.begin(), to_sort.end(), compare_hotness); |
|
| 87 for (std::vector<PyCodeObject*>::iterator co = to_sort.begin(); | 133 for (std::vector<PyCodeObject*>::iterator co = to_sort.begin(); |
|
| 88 co != to_sort.end(); ++co) { | 134 co != to_sort.end(); ++co) { |
|
| 89 » » errs() << PyString_AsString((*co)->co_filename) | 135 » » errs() << _PyEval_GetCodeName(*co) |
|
| 90 » » << ":" << (*co)->co_firstlineno << " " |
|
| 91 » » << "(" << PyString_AsString((*co)->co_name) << ")" |
|
| 92 << " -> " << (*co)->co_hotness << "\n"; | 136 << " -> " << (*co)->co_hotness << "\n"; |
|
| 93 } | 137 } |
|
| 94 } | 138 } |
|
| 95 | 139 |
|
| 96 static llvm::ManagedStatic<HotnessTracker> hot_code; | 140 static llvm::ManagedStatic<HotnessTracker> hot_code; |
|
| 97 | 141 |
|
| 98 | 142 |
|
| 99 // Keep track of which functions failed fatal guards, but kept being called. | 143 // Keep track of which functions failed fatal guards, but kept being called. |
|
| 100 // This can help gauge the efficacy of optimizations that involve fatal guards. | 144 // This can help gauge the efficacy of optimizations that involve fatal guards. |
|
| 101 class FatalBailTracker { | 145 class FatalBailTracker { |
|
| 102 public: | 146 public: |
|
| 103 ~FatalBailTracker() { | 147 ~FatalBailTracker() { |
|
| 104 errs() << "\nCode objects that failed fatal guards:\n"; | 148 errs() << "\nCode objects that failed fatal guards:\n"; |
|
| 105 errs() << "\tfile:line (funcname) bail hotness" | 149 errs() << "\tfile:line (funcname) bail hotness" |
|
| 106 << " -> final hotness\n"; | 150 << " -> final hotness\n"; |
|
| 107 | 151 |
|
| 108 for (TrackerData::const_iterator it = this->code_.begin(); | 152 for (TrackerData::const_iterator it = this->code_.begin(); |
|
| 109 it != this->code_.end(); ++it) { | 153 it != this->code_.end(); ++it) { |
|
| 110 PyCodeObject *code = it->first; | 154 PyCodeObject *code = it->first; |
|
| 111 if (code->co_hotness == it->second) | 155 if (code->co_hotness == it->second) |
|
| 112 continue; | 156 continue; |
|
| 113 » » » errs() << "\t" | 157 » » » errs() << "\t" << _PyEval_GetCodeName(code) |
|
| 114 » » » << PyString_AsString(code->co_name) << ":" |
|
| 115 » » » << code->co_firstlineno << " " |
|
| 116 » » » << "(" << PyString_AsString(code->co_name) << ")" |
|
| 117 << "\t" << it->second << " -> " | 158 << "\t" << it->second << " -> " |
|
| 118 << code->co_hotness << "\n"; | 159 << code->co_hotness << "\n"; |
|
| 119 } | 160 } |
|
| 120 } | 161 } |
|
| 121 | 162 |
|
| 122 void RecordFatalBail(PyCodeObject *code) { | 163 void RecordFatalBail(PyCodeObject *code) { |
|
| 123 Py_INCREF(code); | 164 Py_INCREF(code); |
|
| 124 this->code_.push_back(std::make_pair(code, code->co_hotness)); | 165 this->code_.push_back(std::make_pair(code, code->co_hotness)); |
|
| 125 } | 166 } |
|
| 126 | 167 |
|
| (...skipping 39 matching lines...) | | Loading... |
| 166 | 207 |
|
| 167 | 208 |
|
| 168 class BailCountStats { | 209 class BailCountStats { |
|
| 169 public: | 210 public: |
|
| 170 BailCountStats() : total_(0), trace_on_entry_(0), line_trace_(0), | 211 BailCountStats() : total_(0), trace_on_entry_(0), line_trace_(0), |
|
| 171 backedge_trace_(0), call_profile_(0), | 212 backedge_trace_(0), call_profile_(0), |
|
| 172 fatal_guard_fail_(0), guard_fail_(0) {}; | 213 fatal_guard_fail_(0), guard_fail_(0) {}; |
|
| 173 | 214 |
|
| 174 ~BailCountStats() { | 215 ~BailCountStats() { |
|
| 175 errs() << "\nBailed to the interpreter " << this->total_ | 216 errs() << "\nBailed to the interpreter " << this->total_ |
|
| 176 » » << " times\n"; | 217 » » << " times:\n"; |
|
| 177 errs() << "TRACE_ON_ENTRY: " << this->trace_on_entry_ << "\n"; | 218 errs() << "TRACE_ON_ENTRY: " << this->trace_on_entry_ << "\n"; |
|
| 178 errs() << "LINE_TRACE: " << this->line_trace_ << "\n"; | 219 errs() << "LINE_TRACE: " << this->line_trace_ << "\n"; |
|
| 179 errs() << "BACKEDGE_TRACE:" << this->backedge_trace_ << "\n"; | 220 errs() << "BACKEDGE_TRACE:" << this->backedge_trace_ << "\n"; |
|
| 180 errs() << "CALL_PROFILE: " << this->call_profile_ << "\n"; | 221 errs() << "CALL_PROFILE: " << this->call_profile_ << "\n"; |
|
| 181 errs() << "FATAL_GUARD_FAIL: " << this->fatal_guard_fail_ | 222 errs() << "FATAL_GUARD_FAIL: " << this->fatal_guard_fail_ |
|
| 182 << "\n"; | 223 << "\n"; |
|
| 183 errs() << "GUARD_FAIL: " << this->guard_fail_ << "\n"; | 224 errs() << "GUARD_FAIL: " << this->guard_fail_ << "\n"; |
|
| 184 | 225 |
|
| 185 errs() << "\n" << this->bail_site_freq_.size() | 226 errs() << "\n" << this->bail_site_freq_.size() |
|
| 186 << " bail sites:\n"; | 227 << " bail sites:\n"; |
|
| (...skipping 198 matching lines...) | | Loading... |
| 385 PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER | 426 PCALL_CFUNCTION + PCALL_TYPE + PCALL_GENERATOR + PCALL_OTHER |
|
| 386 PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION | 427 PCALL_FUNCTION > PCALL_FAST_FUNCTION > PCALL_FASTER_FUNCTION |
|
| 387 PCALL_METHOD > PCALL_BOUND_METHOD | 428 PCALL_METHOD > PCALL_BOUND_METHOD |
|
| 388 */ | 429 */ |
|
| 389 | 430 |
|
| 390 #define PCALL(POS) pcall[POS]++ | 431 #define PCALL(POS) pcall[POS]++ |
|
| 391 | 432 |
|
| 392 PyObject * | 433 PyObject * |
|
| 393 PyEval_GetCallStats(PyObject *self) | 434 PyEval_GetCallStats(PyObject *self) |
|
| 394 { | 435 { |
|
| 395 » return Py_BuildValue("iiiiiiiiiii", | 436 » return Py_BuildValue("iiiiiiiiiiiii", |
|
| 396 pcall[0], pcall[1], pcall[2], pcall[3], | 437 pcall[0], pcall[1], pcall[2], pcall[3], |
|
| 397 pcall[4], pcall[5], pcall[6], pcall[7], | 438 pcall[4], pcall[5], pcall[6], pcall[7], |
|
| 398 pcall[8], pcall[9], pcall[10]); | 439 pcall[8], pcall[9], pcall[10]); |
|
| 399 } | 440 } |
|
| 400 #else | 441 #else |
|
| 401 #define PCALL(O) | 442 #define PCALL(O) |
|
| 402 | 443 |
|
| 403 PyObject * | 444 PyObject * |
|
| 404 PyEval_GetCallStats(PyObject *self) | 445 PyEval_GetCallStats(PyObject *self) |
|
| 405 { | 446 { |
|
| (...skipping 657 matching lines...) | | Loading... |
| 1063 /* push frame */ | 1104 /* push frame */ |
|
| 1064 if (bail_reason == _PYFRAME_NO_BAIL && Py_EnterRecursiveCall("")) | 1105 if (bail_reason == _PYFRAME_NO_BAIL && Py_EnterRecursiveCall("")) |
|
| 1065 return NULL; | 1106 return NULL; |
|
| 1066 | 1107 |
|
| 1067 co = f->f_code; | 1108 co = f->f_code; |
|
| 1068 tstate->frame = f; | 1109 tstate->frame = f; |
|
| 1069 | 1110 |
|
| 1070 #ifdef WITH_LLVM | 1111 #ifdef WITH_LLVM |
|
| 1071 maybe_compile(co, f); | 1112 maybe_compile(co, f); |
|
| 1072 | 1113 |
|
| 1073 » if (f->f_use_llvm) { | 1114 » if (f->f_use_jit) { |
|
| 1074 assert(bail_reason == _PYFRAME_NO_BAIL); | 1115 assert(bail_reason == _PYFRAME_NO_BAIL); |
|
| 1075 assert(co->co_native_function != NULL && | 1116 assert(co->co_native_function != NULL && |
|
| 1076 "maybe_compile was supposed to ensure" | 1117 "maybe_compile was supposed to ensure" |
|
| 1077 " that co_native_function exists"); | 1118 " that co_native_function exists"); |
|
| 1078 » » if (!co->co_use_llvm) { | 1119 » » if (!co->co_use_jit) { |
|
| 1079 » » » /* A frame cannot use_llvm if the underlying code object | 1120 » » » // A frame cannot use_jit if the underlying code object |
|
| 1080 » » » * can't use_llvm. This comes up when a generator is | 1121 » » » // can't use_jit. This comes up when a generator is |
|
| 1081 » » » * invalidated while active. */ | 1122 » » » // invalidated while active. |
|
| 1082 » » » f->f_use_llvm = 0; | 1123 » » » f->f_use_jit = 0; |
|
| 1083 } | 1124 } |
|
| 1084 else { | 1125 else { |
|
| 1085 assert(co->co_fatalbailcount < PY_MAX_FATALBAILCOUNT); | 1126 assert(co->co_fatalbailcount < PY_MAX_FATALBAILCOUNT); |
|
| 1086 retval = co->co_native_function(f); | 1127 retval = co->co_native_function(f); |
|
| 1087 goto exit_eval_frame; | 1128 goto exit_eval_frame; |
|
| 1088 } | 1129 } |
|
| 1089 } | 1130 } |
|
| 1090 | 1131 |
|
| 1091 if (bail_reason != _PYFRAME_NO_BAIL) { | 1132 if (bail_reason != _PYFRAME_NO_BAIL) { |
|
| 1092 #ifdef Py_WITH_INSTRUMENTATION | 1133 #ifdef Py_WITH_INSTRUMENTATION |
|
| 1093 bail_count_stats->RecordBail(f, bail_reason); | 1134 bail_count_stats->RecordBail(f, bail_reason); |
|
| 1094 #endif | 1135 #endif |
|
| 1095 if (_Py_BailError) { | 1136 if (_Py_BailError) { |
|
| 1096 /* When we bail, we set f_lasti to the current opcode | 1137 /* When we bail, we set f_lasti to the current opcode |
|
| 1097 * minus 1, so we add one back. */ | 1138 * minus 1, so we add one back. */ |
|
| 1098 int lasti = f->f_lasti + 1; | 1139 int lasti = f->f_lasti + 1; |
|
| 1099 PyErr_Format(PyExc_RuntimeError, "bailed to the " | 1140 PyErr_Format(PyExc_RuntimeError, "bailed to the " |
|
| 1100 "interpreter at opcode index %d", lasti); | 1141 "interpreter at opcode index %d", lasti); |
|
| 1101 goto exit_eval_frame; | 1142 goto exit_eval_frame; |
|
| 1102 } | 1143 } |
|
| 1103 } | 1144 } |
|
| 1104 | 1145 |
|
| 1105 /* Create co_runtime_feedback now that we're about to use it. You | 1146 /* Create co_runtime_feedback now that we're about to use it. You |
|
| 1106 * might think this would cause a problem if the user flips | 1147 * might think this would cause a problem if the user flips |
|
| 1107 * Py_JitControl from "never" to "whenhot", but since the value of | 1148 * Py_JitControl from "never" to "whenhot", but since the value of |
|
| 1108 * rec_feedback is constant for the duration of this frame's execution, | 1149 * rec_feedback is constant for the duration of this frame's execution, |
|
| 1109 * we will not accidentally try to record feedback without initializing | 1150 * we will not accidentally try to record feedback without initializing |
|
| 1110 * co_runtime_feedback. */ | 1151 * co_runtime_feedback. */ |
|
| 1111 if (rec_feedback && co->co_runtime_feedback == NULL) { | 1152 if (rec_feedback && co->co_runtime_feedback == NULL) { |
|
| 1153 #if Py_WITH_INSTRUMENTATION |
|
| 1154 feedback_map_counter->IncCounter(); |
|
| 1155 #endif |
|
| 1112 co->co_runtime_feedback = PyFeedbackMap_New(); | 1156 co->co_runtime_feedback = PyFeedbackMap_New(); |
|
| 1113 } | 1157 } |
|
| 1114 #endif /* WITH_LLVM */ | 1158 #endif /* WITH_LLVM */ |
|
| 1115 | 1159 |
|
| 1116 switch (bail_reason) { | 1160 switch (bail_reason) { |
|
| 1117 case _PYFRAME_NO_BAIL: | 1161 case _PYFRAME_NO_BAIL: |
|
| 1118 case _PYFRAME_TRACE_ON_ENTRY: | 1162 case _PYFRAME_TRACE_ON_ENTRY: |
|
| 1119 if (tstate->use_tracing) { | 1163 if (tstate->use_tracing) { |
|
| 1120 if (_PyEval_TraceEnterFunction(tstate, f)) | 1164 if (_PyEval_TraceEnterFunction(tstate, f)) |
|
| 1121 /* Trace or profile function raised | 1165 /* Trace or profile function raised |
|
| (...skipping 1009 matching lines...) | | Loading... |
| 2131 while (STACK_LEVEL() > b->b_level) { | 2175 while (STACK_LEVEL() > b->b_level) { |
|
| 2132 v = POP(); | 2176 v = POP(); |
|
| 2133 Py_DECREF(v); | 2177 Py_DECREF(v); |
|
| 2134 } | 2178 } |
|
| 2135 } | 2179 } |
|
| 2136 DISPATCH(); | 2180 DISPATCH(); |
|
| 2137 | 2181 |
|
| 2138 PREDICTED(END_FINALLY); | 2182 PREDICTED(END_FINALLY); |
|
| 2139 TARGET(END_FINALLY) | 2183 TARGET(END_FINALLY) |
|
| 2140 v = POP(); | 2184 v = POP(); |
|
| 2185 w = POP(); |
|
| 2186 u = POP(); |
|
| 2141 if (PyInt_Check(v)) { | 2187 if (PyInt_Check(v)) { |
|
| 2142 why = (enum _PyUnwindReason) PyInt_AS_LONG(v); | 2188 why = (enum _PyUnwindReason) PyInt_AS_LONG(v); |
|
| 2143 assert(why != UNWIND_YIELD); | 2189 assert(why != UNWIND_YIELD); |
|
| 2144 if (why == UNWIND_RETURN || | 2190 if (why == UNWIND_RETURN || |
|
| 2145 why == UNWIND_CONTINUE) | 2191 why == UNWIND_CONTINUE) |
|
| 2146 » » » » » retval = POP(); | 2192 » » » » » retval = w; |
|
| 2193 » » » » else |
|
| 2194 » » » » » Py_DECREF(w); |
|
| 2147 } | 2195 } |
|
| 2148 else if (PyExceptionClass_Check(v) || | 2196 else if (PyExceptionClass_Check(v) || |
|
| 2149 PyString_Check(v)) { | 2197 PyString_Check(v)) { |
|
| 2150 w = POP(); |
|
| 2151 u = POP(); |
|
| 2152 PyErr_Restore(v, w, u); | 2198 PyErr_Restore(v, w, u); |
|
| 2153 why = UNWIND_RERAISE; | 2199 why = UNWIND_RERAISE; |
|
| 2154 break; | 2200 break; |
|
| 2155 } | 2201 } |
|
| 2156 else if (v != Py_None) { | 2202 else if (v != Py_None) { |
|
| 2157 PyErr_SetString(PyExc_SystemError, | 2203 PyErr_SetString(PyExc_SystemError, |
|
| 2158 "'finally' pops bad exception"); | 2204 "'finally' pops bad exception"); |
|
| 2159 why = UNWIND_EXCEPTION; | 2205 why = UNWIND_EXCEPTION; |
|
| 2206 Py_DECREF(w); |
|
| 2160 } | 2207 } |
|
| 2161 Py_DECREF(v); | 2208 Py_DECREF(v); |
|
| 2209 Py_DECREF(u); |
|
| 2162 break; | 2210 break; |
|
| 2163 | 2211 |
|
| 2164 TARGET(STORE_NAME) | 2212 TARGET(STORE_NAME) |
|
| 2165 x = POP(); | 2213 x = POP(); |
|
| 2166 err = _PyEval_StoreName(f, oparg, x); | 2214 err = _PyEval_StoreName(f, oparg, x); |
|
| 2167 if (err != 0) { | 2215 if (err != 0) { |
|
| 2168 why = UNWIND_EXCEPTION; | 2216 why = UNWIND_EXCEPTION; |
|
| 2169 break; | 2217 break; |
|
| 2170 } | 2218 } |
|
| 2171 DISPATCH(); | 2219 DISPATCH(); |
|
| (...skipping 545 matching lines...) | | Loading... |
| 2717 are not try/except/finally handlers, you may need | 2765 are not try/except/finally handlers, you may need |
|
| 2718 to update the PyGen_NeedsFinalizing() function. | 2766 to update the PyGen_NeedsFinalizing() function. |
|
| 2719 */ | 2767 */ |
|
| 2720 | 2768 |
|
| 2721 PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg, | 2769 PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg, |
|
| 2722 STACK_LEVEL()); | 2770 STACK_LEVEL()); |
|
| 2723 DISPATCH(); | 2771 DISPATCH(); |
|
| 2724 | 2772 |
|
| 2725 TARGET(WITH_CLEANUP) | 2773 TARGET(WITH_CLEANUP) |
|
| 2726 { | 2774 { |
|
| 2727 » » » /* At the top of the stack are 1-3 values indicating | 2775 » » » /* At the top of the stack are 3 values indicating |
|
| 2728 how/why we entered the finally clause: | 2776 how/why we entered the finally clause: |
|
| 2729 » » » - TOP = None | 2777 » » » - (TOP, SECOND, THIRD) = None, None, None |
|
| 2730 » » » - (TOP, SECOND) = (UNWIND_{RETURN,CONTINUE}), retval | 2778 » » » - (TOP, SECOND, THIRD) = (UNWIND_{RETURN,CONTINUE}), |
|
| 2731 » » » - TOP = UNWIND_*; no retval below it | 2779 » » » retval, None |
|
| 2780 » » » - (TOP, SECOND, THIRD) = UNWIND_*, None, None |
|
| 2732 - (TOP, SECOND, THIRD) = exc_info() | 2781 - (TOP, SECOND, THIRD) = exc_info() |
|
| 2733 Below them is EXIT, the context.__exit__ bound method
. | 2782 Below them is EXIT, the context.__exit__ bound method
. |
|
| 2734 In the last case, we must call | 2783 In the last case, we must call |
|
| 2735 EXIT(TOP, SECOND, THIRD) | 2784 EXIT(TOP, SECOND, THIRD) |
|
| 2736 otherwise we must call | 2785 otherwise we must call |
|
| 2737 EXIT(None, None, None) | 2786 EXIT(None, None, None) |
|
| 2738 | 2787 |
|
| 2739 In all cases, we remove EXIT from the stack, leaving | 2788 In all cases, we remove EXIT from the stack, leaving |
|
| 2740 the rest in the same order. | 2789 the rest in the same order. |
|
| 2741 | 2790 |
|
| 2742 In addition, if the stack represents an exception, | 2791 In addition, if the stack represents an exception, |
|
| 2743 *and* the function call returns a 'true' value, we | 2792 *and* the function call returns a 'true' value, we |
|
| 2744 "zap" this information, to prevent END_FINALLY from | 2793 "zap" this information, to prevent END_FINALLY from |
|
| 2745 re-raising the exception. (But non-local gotos | 2794 re-raising the exception. (But non-local gotos |
|
| 2746 should still be resumed.) | 2795 should still be resumed.) |
|
| 2747 */ | 2796 */ |
|
| 2748 | 2797 |
|
| 2749 PyObject *exit_func; | 2798 PyObject *exit_func; |
|
| 2750 | 2799 |
|
| 2751 u = POP(); | 2800 u = POP(); |
|
| 2752 » » » if (u == Py_None) { | 2801 » » » v = TOP(); |
|
| 2753 » » » »exit_func = TOP(); | 2802 » » » w = SECOND(); |
|
| 2754 » » » » SET_TOP(u); | 2803 » » » exit_func = THIRD(); |
|
| 2755 » » » » v = w = Py_None; | 2804 » » » SET_TOP(u); |
|
| 2756 » » » } | 2805 » » » SET_SECOND(v); |
|
| 2757 » » » else if (PyInt_Check(u)) { | 2806 » » » SET_THIRD(w); |
|
| 2758 » » » » switch(PyInt_AS_LONG(u)) { | 2807 » » » if (PyInt_Check(u)) |
|
| 2759 » » » » case UNWIND_RETURN: |
|
| 2760 » » » » case UNWIND_CONTINUE: |
|
| 2761 » » » » » /* Retval in TOP. */ |
|
| 2762 » » » » » exit_func = SECOND(); |
|
| 2763 » » » » » SET_SECOND(TOP()); |
|
| 2764 » » » » » SET_TOP(u); |
|
| 2765 » » » » » break; |
|
| 2766 » » » » default: |
|
| 2767 » » » » » exit_func = TOP(); |
|
| 2768 » » » » » SET_TOP(u); |
|
| 2769 » » » » » break; |
|
| 2770 » » » » } |
|
| 2771 u = v = w = Py_None; | 2808 u = v = w = Py_None; |
|
| 2772 } |
|
| 2773 else { |
|
| 2774 v = TOP(); |
|
| 2775 w = SECOND(); |
|
| 2776 exit_func = THIRD(); |
|
| 2777 SET_TOP(u); |
|
| 2778 SET_SECOND(v); |
|
| 2779 SET_THIRD(w); |
|
| 2780 } |
|
| 2781 /* XXX Not the fastest way to call it... */ | 2809 /* XXX Not the fastest way to call it... */ |
|
| 2782 x = PyObject_CallFunctionObjArgs(exit_func, u, v, w, | 2810 x = PyObject_CallFunctionObjArgs(exit_func, u, v, w, |
|
| 2783 NULL); | 2811 NULL); |
|
| 2784 Py_DECREF(exit_func); | 2812 Py_DECREF(exit_func); |
|
| 2785 if (x == NULL) { | 2813 if (x == NULL) { |
|
| 2786 why = UNWIND_EXCEPTION; | 2814 why = UNWIND_EXCEPTION; |
|
| 2787 break; /* Go to error exit */ | 2815 break; /* Go to error exit */ |
|
| 2788 } | 2816 } |
|
| 2789 | 2817 |
|
| 2790 if (u != Py_None) | 2818 if (u != Py_None) |
|
| 2791 err = PyObject_IsTrue(x); | 2819 err = PyObject_IsTrue(x); |
|
| 2792 else | 2820 else |
|
| 2793 err = 0; | 2821 err = 0; |
|
| 2794 Py_DECREF(x); | 2822 Py_DECREF(x); |
|
| 2795 | 2823 |
|
| 2796 if (err < 0) { | 2824 if (err < 0) { |
|
| 2797 why = UNWIND_EXCEPTION; | 2825 why = UNWIND_EXCEPTION; |
|
| 2798 break; /* Go to error exit */ | 2826 break; /* Go to error exit */ |
|
| 2799 } | 2827 } |
|
| 2800 else if (err > 0) { | 2828 else if (err > 0) { |
|
| 2801 /* There was an exception and a true return */ | 2829 /* There was an exception and a true return */ |
|
| 2802 STACKADJ(-2); |
|
| 2803 Py_INCREF(Py_None); | 2830 Py_INCREF(Py_None); |
|
| 2804 SET_TOP(Py_None); | 2831 SET_TOP(Py_None); |
|
| 2832 Py_INCREF(Py_None); |
|
| 2833 SET_SECOND(Py_None); |
|
| 2834 Py_INCREF(Py_None); |
|
| 2835 SET_THIRD(Py_None); |
|
| 2805 Py_DECREF(u); | 2836 Py_DECREF(u); |
|
| 2806 Py_DECREF(v); | 2837 Py_DECREF(v); |
|
| 2807 Py_DECREF(w); | 2838 Py_DECREF(w); |
|
| 2808 } else { | 2839 } else { |
|
| 2809 /* The stack was rearranged to remove EXIT | 2840 /* The stack was rearranged to remove EXIT |
|
| 2810 above. Let END_FINALLY do its thing */ | 2841 above. Let END_FINALLY do its thing */ |
|
| 2811 } | 2842 } |
|
| 2812 PREDICT(END_FINALLY); | 2843 PREDICT(END_FINALLY); |
|
| 2813 DISPATCH(); | 2844 DISPATCH(); |
|
| 2814 } | 2845 } |
|
| 2815 | 2846 |
|
| 2816 TARGET(CALL_FUNCTION) | 2847 TARGET(CALL_FUNCTION) |
|
| 2817 { | 2848 { |
|
| 2818 int num_args, num_kwargs, num_stack_slots; | 2849 int num_args, num_kwargs, num_stack_slots; |
|
| 2819 PY_LOG_TSC_EVENT(CALL_START_EVAL); | 2850 PY_LOG_TSC_EVENT(CALL_START_EVAL); |
|
| 2820 PCALL(PCALL_ALL); | 2851 PCALL(PCALL_ALL); |
|
| 2821 num_args = oparg & 0xff; | 2852 num_args = oparg & 0xff; |
|
| 2822 num_kwargs = (oparg>>8) & 0xff; | 2853 num_kwargs = (oparg>>8) & 0xff; |
|
| 2823 #ifdef WITH_LLVM | 2854 #ifdef WITH_LLVM |
|
| 2824 /* We'll focus on these simple calls with only | 2855 /* We'll focus on these simple calls with only |
|
| 2825 * positional args for now (since they're easy to | 2856 * positional args for now (since they're easy to |
|
| 2826 * implement). */ | 2857 * implement). */ |
|
| 2827 if (num_kwargs == 0) { | 2858 if (num_kwargs == 0) { |
|
| 2828 /* Duplicate this bit of logic from | 2859 /* Duplicate this bit of logic from |
|
| 2829 * _PyEval_CallFunction(). */ | 2860 * _PyEval_CallFunction(). */ |
|
| 2830 PyObject **func = stack_pointer - num_args - 1; | 2861 PyObject **func = stack_pointer - num_args - 1; |
|
| 2831 RECORD_FUNC(*func); | 2862 RECORD_FUNC(*func); |
|
| 2863 /* For C functions, record the types passed,· |
|
| 2864 * in order to do potential inlining. */ |
|
| 2865 if (PyCFunction_Check(*func) && |
|
| 2866 (PyCFunction_GET_FLAGS(*func) & |
|
| 2867 METH_ARG_RANGE)) { |
|
| 2868 for(int i = 0; i < num_args; i++) { |
|
| 2869 RECORD_TYPE(i + 1, |
|
| 2870 stack_pointer[-i-1]); |
|
| 2871 } |
|
| 2872 } |
|
| 2832 } | 2873 } |
|
| 2833 #endif | 2874 #endif |
|
| 2834 x = _PyEval_CallFunction(stack_pointer, | 2875 x = _PyEval_CallFunction(stack_pointer, |
|
| 2835 num_args, num_kwargs); | 2876 num_args, num_kwargs); |
|
| 2836 /* +1 for the actual function object. */ | 2877 /* +1 for the actual function object. */ |
|
| 2837 num_stack_slots = num_args + 2 * num_kwargs + 1; | 2878 num_stack_slots = num_args + 2 * num_kwargs + 1; |
|
| 2838 /* Clear the stack of the function object and | 2879 /* Clear the stack of the function object and |
|
| 2839 * arguments. */ | 2880 * arguments. */ |
|
| 2840 stack_pointer -= num_stack_slots; | 2881 stack_pointer -= num_stack_slots; |
|
| 2841 PUSH(x); | 2882 PUSH(x); |
|
| (...skipping 310 matching lines...) | | Loading... |
| 3152 PUSH(tb); | 3193 PUSH(tb); |
|
| 3153 PUSH(val); | 3194 PUSH(val); |
|
| 3154 PUSH(exc); | 3195 PUSH(exc); |
|
| 3155 /* Within the except or finally block, | 3196 /* Within the except or finally block, |
|
| 3156 PyErr_Occurred() should be false. | 3197 PyErr_Occurred() should be false. |
|
| 3157 END_FINALLY will restore the | 3198 END_FINALLY will restore the |
|
| 3158 exception if necessary. */ | 3199 exception if necessary. */ |
|
| 3159 PyErr_Clear(); | 3200 PyErr_Clear(); |
|
| 3160 } | 3201 } |
|
| 3161 else { | 3202 else { |
|
| 3203 Py_INCREF(Py_None); |
|
| 3204 PUSH(Py_None); |
|
| 3162 if (why & (UNWIND_RETURN | UNWIND_CONTIN
UE)) | 3205 if (why & (UNWIND_RETURN | UNWIND_CONTIN
UE)) |
|
| 3206 { |
|
| 3163 PUSH(retval); | 3207 PUSH(retval); |
|
| 3208 } |
|
| 3209 else { |
|
| 3210 Py_INCREF(Py_None); |
|
| 3211 PUSH(Py_None); |
|
| 3212 } |
|
| 3164 v = PyInt_FromLong((long)why); | 3213 v = PyInt_FromLong((long)why); |
|
| 3165 PUSH(v); | 3214 PUSH(v); |
|
| 3166 } | 3215 } |
|
| 3167 why = UNWIND_NOUNWIND; | 3216 why = UNWIND_NOUNWIND; |
|
| 3168 JUMPTO(b->b_handler); | 3217 JUMPTO(b->b_handler); |
|
| 3169 break; | 3218 break; |
|
| 3170 } | 3219 } |
|
| 3171 } /* unwind stack */ | 3220 } /* unwind stack */ |
|
| 3172 | 3221 |
|
| 3173 /* End the loop if we still have an error (or return) */ | 3222 /* End the loop if we still have an error (or return) */ |
|
| (...skipping 797 matching lines...) | | Loading... |
| 3971 _PyEval_TraceLeaveFunction(PyThreadState *tstate, PyFrameObject *f, | 4020 _PyEval_TraceLeaveFunction(PyThreadState *tstate, PyFrameObject *f, |
|
| 3972 PyObject *retval, | 4021 PyObject *retval, |
|
| 3973 char is_return_or_yield, char is_exception) | 4022 char is_return_or_yield, char is_exception) |
|
| 3974 { | 4023 { |
|
| 3975 int err = 0; | 4024 int err = 0; |
|
| 3976 if (tstate->c_tracefunc) { | 4025 if (tstate->c_tracefunc) { |
|
| 3977 if (is_return_or_yield) { | 4026 if (is_return_or_yield) { |
|
| 3978 if (_PyEval_CallTrace(tstate->c_tracefunc, | 4027 if (_PyEval_CallTrace(tstate->c_tracefunc, |
|
| 3979 tstate->c_traceobj, f, | 4028 tstate->c_traceobj, f, |
|
| 3980 PyTrace_RETURN, retval)) { | 4029 PyTrace_RETURN, retval)) { |
|
| 3981 » » » » is_exception = true; | 4030 » » » » is_exception = 1; |
|
| 3982 err = -1; | 4031 err = -1; |
|
| 3983 } | 4032 } |
|
| 3984 } | 4033 } |
|
| 3985 else if (is_exception) { | 4034 else if (is_exception) { |
|
| 3986 call_trace_protected(tstate->c_tracefunc, | 4035 call_trace_protected(tstate->c_tracefunc, |
|
| 3987 tstate->c_traceobj, f, | 4036 tstate->c_traceobj, f, |
|
| 3988 PyTrace_RETURN, NULL); | 4037 PyTrace_RETURN, NULL); |
|
| 3989 } | 4038 } |
|
| 3990 } | 4039 } |
|
| 3991 if (tstate->c_profilefunc) { | 4040 if (tstate->c_profilefunc) { |
|
| (...skipping 287 matching lines...) | | Loading... |
| 4279 co->co_hotness += 10; | 4328 co->co_hotness += 10; |
|
| 4280 } | 4329 } |
|
| 4281 | 4330 |
|
| 4282 // Decide whether to compile a code object's bytecode to native code based on | 4331 // Decide whether to compile a code object's bytecode to native code based on |
|
| 4283 // the current Py_JitControl setting and the code's hotness. We do the | 4332 // the current Py_JitControl setting and the code's hotness. We do the |
|
| 4284 // compilation if any of the following conditions are true: | 4333 // compilation if any of the following conditions are true: |
|
| 4285 // | 4334 // |
|
| 4286 // - We are running under PY_JIT_WHENHOT and co's hotness has passed the | 4335 // - We are running under PY_JIT_WHENHOT and co's hotness has passed the |
|
| 4287 // hotness threshold. | 4336 // hotness threshold. |
|
| 4288 // - The code object was marked as needing to be run through LLVM | 4337 // - The code object was marked as needing to be run through LLVM |
|
| 4289 // (co_use_llvm is true). | 4338 // (co_use_jit is true). |
|
| 4290 // - We are running under PY_JIT_ALWAYS. | 4339 // - We are running under PY_JIT_ALWAYS. |
|
| 4291 // | 4340 // |
|
| 4292 // Returns 0 on success or -1 on failure. | 4341 // Returns 0 on success or -1 on failure. |
|
| 4293 // | 4342 // |
|
| 4294 // If this code object has had too many fatal guard failures (see | 4343 // If this code object has had too many fatal guard failures (see |
|
| 4295 // PY_MAX_FATALBAILCOUNT), it is forced to use the eval loop forever. | 4344 // PY_MAX_FATALBAILCOUNT), it is forced to use the eval loop forever. |
|
| 4296 // | 4345 // |
|
| 4297 // This function is performance-critical. If you're changing this function, | 4346 // This function is performance-critical. If you're changing this function, |
|
| 4298 // you should keep a close eye on the benchmarks, particularly call_simple. | 4347 // you should keep a close eye on the benchmarks, particularly call_simple. |
|
| 4299 // In the past, seemingly-insignificant changes have produced 10-15% swings | 4348 // In the past, seemingly-insignificant changes have produced 10-15% swings |
|
| 4300 // in the macrobenchmarks. You've been warned. | 4349 // in the macrobenchmarks. You've been warned. |
|
| 4301 static inline int | 4350 static inline int |
|
| 4302 maybe_compile(PyCodeObject *co, PyFrameObject *f) | 4351 maybe_compile(PyCodeObject *co, PyFrameObject *f) |
|
| 4303 { | 4352 { |
|
| 4304 if (f->f_bailed_from_llvm != _PYFRAME_NO_BAIL) { | 4353 if (f->f_bailed_from_llvm != _PYFRAME_NO_BAIL) { |
|
| 4305 // Don't consider compiling code objects that we've already | 4354 // Don't consider compiling code objects that we've already |
|
| 4306 // bailed from. This avoids re-entering code that we just | 4355 // bailed from. This avoids re-entering code that we just |
|
| 4307 // bailed from. | 4356 // bailed from. |
|
| 4308 return 0; | 4357 return 0; |
|
| 4309 } | 4358 } |
|
| 4310 | 4359 |
|
| 4311 if (co->co_fatalbailcount >= PY_MAX_FATALBAILCOUNT) { | 4360 if (co->co_fatalbailcount >= PY_MAX_FATALBAILCOUNT) { |
|
| 4312 » » co->co_use_llvm = 0; | 4361 » » co->co_use_jit = 0; |
|
| 4313 return 0; | 4362 return 0; |
|
| 4314 } | 4363 } |
|
| 4315 | 4364 |
|
| 4316 » if (co->co_flags & CO_FDO_GLOBALS && | 4365 » if (co->co_watching && co->co_watching[WATCHING_GLOBALS] && |
|
| 4317 » (co->co_assumed_globals != f->f_globals || | 4366 » (co->co_watching[WATCHING_GLOBALS] != f->f_globals || |
|
| 4318 » co->co_assumed_builtins != f->f_builtins)) { | 4367 » co->co_watching[WATCHING_BUILTINS] != f->f_builtins)) { |
|
| 4319 // If there's no way a code object's assumptions about its | 4368 // If there's no way a code object's assumptions about its |
|
| 4320 // globals and/or builtins could be valid, don't even try the | 4369 // globals and/or builtins could be valid, don't even try the |
|
| 4321 // machine code. | 4370 // machine code. |
|
| 4322 return 0; | 4371 return 0; |
|
| 4323 } | 4372 } |
|
| 4324 | 4373 |
|
| 4325 bool is_hot = false; | 4374 bool is_hot = false; |
|
| 4326 if (co->co_hotness > PY_HOTNESS_THRESHOLD) { | 4375 if (co->co_hotness > PY_HOTNESS_THRESHOLD) { |
|
| 4327 is_hot = true; | 4376 is_hot = true; |
|
| 4328 #ifdef Py_WITH_INSTRUMENTATION | 4377 #ifdef Py_WITH_INSTRUMENTATION |
|
| 4329 hot_code->AddHotCode(co); | 4378 hot_code->AddHotCode(co); |
|
| 4330 #endif | 4379 #endif |
|
| 4331 } | 4380 } |
|
| 4332 switch (Py_JitControl) { | 4381 switch (Py_JitControl) { |
|
| 4333 default: | 4382 default: |
|
| 4334 PyErr_BadInternalCall(); | 4383 PyErr_BadInternalCall(); |
|
| 4335 return -1; | 4384 return -1; |
|
| 4336 case PY_JIT_WHENHOT: | 4385 case PY_JIT_WHENHOT: |
|
| 4337 if (is_hot) | 4386 if (is_hot) |
|
| 4338 » » » co->co_use_llvm = 1; | 4387 » » » co->co_use_jit = 1; |
|
| 4339 break; | 4388 break; |
|
| 4340 case PY_JIT_ALWAYS: | 4389 case PY_JIT_ALWAYS: |
|
| 4341 » » co->co_use_llvm = 1; | 4390 » » co->co_use_jit = 1; |
|
| 4342 break; | 4391 break; |
|
| 4343 case PY_JIT_NEVER: | 4392 case PY_JIT_NEVER: |
|
| 4344 break; | 4393 break; |
|
| 4345 } | 4394 } |
|
| 4346 | 4395 |
|
| 4347 » if (co->co_use_llvm) { | 4396 » if (co->co_use_jit) { |
|
| 4348 if (co->co_llvm_function == NULL) { | 4397 if (co->co_llvm_function == NULL) { |
|
| 4349 // Translate the bytecode to IR and optimize it if we | 4398 // Translate the bytecode to IR and optimize it if we |
|
| 4350 // haven't already done that. | 4399 // haven't already done that. |
|
| 4351 int target_optimization = | 4400 int target_optimization = |
|
| 4352 std::max(Py_DEFAULT_JIT_OPT_LEVEL, | 4401 std::max(Py_DEFAULT_JIT_OPT_LEVEL, |
|
| 4353 Py_OptimizeFlag); | 4402 Py_OptimizeFlag); |
|
| 4354 if (co->co_optimization < target_optimization) { | 4403 if (co->co_optimization < target_optimization) { |
|
| 4355 PY_LOG_TSC_EVENT(EVAL_COMPILE_START); | 4404 PY_LOG_TSC_EVENT(EVAL_COMPILE_START); |
|
| 4356 int r; | 4405 int r; |
|
| 4406 #if Py_WITH_INSTRUMENTATION |
|
| 4407 Timer timer(*ir_compilation_times); |
|
| 4408 #endif |
|
| 4357 PY_LOG_TSC_EVENT(LLVM_COMPILE_START); | 4409 PY_LOG_TSC_EVENT(LLVM_COMPILE_START); |
|
| 4358 » » » » if (_PyCode_WatchGlobals(co, f->f_globals, | 4410 » » » » if (_PyCode_WatchDict(co, |
|
| 4359 » » » » f->f_builtins)) { | 4411 » » » » WATCHING_GLOBALS, |
|
| 4412 » » » » f->f_globals)) |
|
| 4360 return -1; | 4413 return -1; |
|
| 4361 » » » » } | 4414 » » » » if (_PyCode_WatchDict(co, |
|
| 4415 » » » » WATCHING_BUILTINS, |
|
| 4416 » » » » f->f_builtins)) |
|
| 4417 » » » » » return -1; |
|
| 4362 r = _PyCode_ToOptimizedLlvmIr( | 4418 r = _PyCode_ToOptimizedLlvmIr( |
|
| 4363 co, target_optimization); | 4419 co, target_optimization); |
|
| 4364 PY_LOG_TSC_EVENT(LLVM_COMPILE_END); | 4420 PY_LOG_TSC_EVENT(LLVM_COMPILE_END); |
|
| 4365 if (r < 0) // Error | 4421 if (r < 0) // Error |
|
| 4366 return -1; | 4422 return -1; |
|
| 4367 if (r == 1) { // Codegen refused | 4423 if (r == 1) { // Codegen refused |
|
| 4368 » » » » » co->co_use_llvm = 0; | 4424 » » » » » co->co_use_jit = 0; |
|
| 4369 return 0; | 4425 return 0; |
|
| 4370 } | 4426 } |
|
| 4371 } | 4427 } |
|
| 4372 } | 4428 } |
|
| 4373 if (co->co_native_function == NULL) { | 4429 if (co->co_native_function == NULL) { |
|
| 4374 // Now try to JIT the IR function to machine code. | 4430 // Now try to JIT the IR function to machine code. |
|
| 4431 #if Py_WITH_INSTRUMENTATION |
|
| 4432 Timer timer(*mc_compilation_times); |
|
| 4433 #endif |
|
| 4375 PY_LOG_TSC_EVENT(JIT_START); | 4434 PY_LOG_TSC_EVENT(JIT_START); |
|
| 4376 co->co_native_function = | 4435 co->co_native_function = |
|
| 4377 _LlvmFunction_Jit(co->co_llvm_function); | 4436 _LlvmFunction_Jit(co->co_llvm_function); |
|
| 4378 PY_LOG_TSC_EVENT(JIT_END); | 4437 PY_LOG_TSC_EVENT(JIT_END); |
|
| 4379 if (co->co_native_function == NULL) { | 4438 if (co->co_native_function == NULL) { |
|
| 4380 return -1; | 4439 return -1; |
|
| 4381 } | 4440 } |
|
| 4382 } | 4441 } |
|
| 4383 PY_LOG_TSC_EVENT(EVAL_COMPILE_END); | 4442 PY_LOG_TSC_EVENT(EVAL_COMPILE_END); |
|
| 4384 } | 4443 } |
|
| 4385 | 4444 |
|
| 4386 » f->f_use_llvm = co->co_use_llvm; | 4445 » f->f_use_jit = co->co_use_jit; |
|
| 4387 return 0; | 4446 return 0; |
|
| 4388 } | 4447 } |
|
| 4389 #endif /* WITH_LLVM */ | 4448 #endif /* WITH_LLVM */ |
|
| 4390 | 4449 |
|
| 4391 #define C_TRACE(x, call) \ | 4450 #define C_TRACE(x, call) \ |
|
| 4392 if (_Py_ProfilingPossible && tstate->use_tracing && tstate->c_profilefunc) { \ | 4451 if (_Py_ProfilingPossible && tstate->use_tracing && tstate->c_profilefunc) { \ |
|
| 4393 if (_PyEval_CallTrace(tstate->c_profilefunc, \ | 4452 if (_PyEval_CallTrace(tstate->c_profilefunc, \ |
|
| 4394 tstate->c_profileobj, \ | 4453 tstate->c_profileobj, \ |
|
| 4395 tstate->frame, PyTrace_C_CALL, \ | 4454 tstate->frame, PyTrace_C_CALL, \ |
|
| 4396 func)) { \ | 4455 func)) { \ |
|
| (...skipping 34 matching lines...) | | Loading... |
| 4431 { | 4490 { |
|
| 4432 int n = na + 2 * nk; | 4491 int n = na + 2 * nk; |
|
| 4433 PyObject **pfunc = stack_pointer - n - 1; | 4492 PyObject **pfunc = stack_pointer - n - 1; |
|
| 4434 PyObject *func = *pfunc; | 4493 PyObject *func = *pfunc; |
|
| 4435 PyObject *x, *w; | 4494 PyObject *x, *w; |
|
| 4436 PyMethodDef *ml = NULL; | 4495 PyMethodDef *ml = NULL; |
|
| 4437 PyObject *self = NULL; | 4496 PyObject *self = NULL; |
|
| 4438 | 4497 |
|
| 4439 /* Always dispatch PyCFunction and PyMethodDescr first, because these | 4498 /* Always dispatch PyCFunction and PyMethodDescr first, because these |
|
| 4440 both represent methods written in C, and are presumed to be the most | 4499 both represent methods written in C, and are presumed to be the most |
|
| 4441 » frequent callable object. | 4500 » frequently called objects. |
|
| 4442 */ | 4501 */ |
|
| 4443 if (nk == 0) { | 4502 if (nk == 0) { |
|
| 4444 if (PyCFunction_Check(func)) { | 4503 if (PyCFunction_Check(func)) { |
|
| 4445 ml = PyCFunction_GET_METHODDEF(func); | 4504 ml = PyCFunction_GET_METHODDEF(func); |
|
| 4446 self = PyCFunction_GET_SELF(func); | 4505 self = PyCFunction_GET_SELF(func); |
|
| 4447 } | 4506 } |
|
| 4448 else if (PyMethodDescr_Check(func)) { | 4507 else if (PyMethodDescr_Check(func)) { |
|
| 4449 ml = ((PyMethodDescrObject*)func)->d_method; | 4508 ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4450 /* The first argument on the stack (the one immediately | 4509 /* The first argument on the stack (the one immediately |
|
| 4451 after func) is self. We borrow the reference from | 4510 after func) is self. We borrow the reference from |
|
| (...skipping 23 matching lines...) Loading... |
| 4475 case 2: args[1] = EXT_POP(stack_pointer); | 4534 case 2: args[1] = EXT_POP(stack_pointer); |
|
| 4476 case 1: args[0] = EXT_POP(stack_pointer); | 4535 case 1: args[0] = EXT_POP(stack_pointer); |
|
| 4477 case 0: break; | 4536 case 0: break; |
|
| 4478 } | 4537 } |
|
| 4479 | 4538 |
|
| 4480 /* But wait, you ask, what about {un,bin}ary functions? | 4539 /* But wait, you ask, what about {un,bin}ary functions? |
|
| 4481 Aren't we passing more arguments than it expects? | 4540 Aren't we passing more arguments than it expects? |
|
| 4482 Yes, but C allows this. Go C. */ | 4541 Yes, but C allows this. Go C. */ |
|
| 4483 if (min_arity <= na && na <= max_arity) { | 4542 if (min_arity <= na && na <= max_arity) { |
|
| 4484 C_TRACE(x, (*(PyCFunctionThreeArgs)meth) | 4543 C_TRACE(x, (*(PyCFunctionThreeArgs)meth) |
|
| 4485 » » » » » (self, args[0], args[1], args[2])); | 4544 » » » » (self, args[0], args[1], args[2])); |
|
| 4486 } | 4545 } |
|
| 4487 else { | 4546 else { |
|
| 4488 err_args(ml, flags, min_arity, max_arity, na); | 4547 err_args(ml, flags, min_arity, max_arity, na); |
|
| 4489 x = NULL; | 4548 x = NULL; |
|
| 4490 } | 4549 } |
|
| 4491 Py_XDECREF(args[0]); | 4550 Py_XDECREF(args[0]); |
|
| 4492 Py_XDECREF(args[1]); | 4551 Py_XDECREF(args[1]); |
|
| 4493 Py_XDECREF(args[2]); | 4552 Py_XDECREF(args[2]); |
|
| 4494 } | 4553 } |
|
| 4495 else { | 4554 else { |
|
| (...skipping 229 matching lines...) | | Loading... |
| 4725 } | 4784 } |
|
| 4726 return args; | 4785 return args; |
|
| 4727 } | 4786 } |
|
| 4728 | 4787 |
|
| 4729 static PyObject * | 4788 static PyObject * |
|
| 4730 do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) | 4789 do_call(PyObject *func, PyObject ***pp_stack, int na, int nk) |
|
| 4731 { | 4790 { |
|
| 4732 PyObject *callargs = NULL; | 4791 PyObject *callargs = NULL; |
|
| 4733 PyObject *kwdict = NULL; | 4792 PyObject *kwdict = NULL; |
|
| 4734 PyObject *result = NULL; | 4793 PyObject *result = NULL; |
|
| 4794 PyMethodDef *ml = NULL; |
|
| 4795 PyObject *self = NULL; |
|
| 4796 |
|
| 4797 if (PyCFunction_Check(func)) { |
|
| 4798 ml = PyCFunction_GET_METHODDEF(func); |
|
| 4799 self = PyCFunction_GET_SELF(func); |
|
| 4800 } |
|
| 4801 else if (PyMethodDescr_Check(func)) { |
|
| 4802 ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4803 self = (*pp_stack)[-(na + 2 * nk)]; |
|
| 4804 na--; |
|
| 4805 } |
|
| 4735 | 4806 |
|
| 4736 if (nk > 0) { | 4807 if (nk > 0) { |
|
| 4737 kwdict = update_keyword_args(NULL, nk, pp_stack, func); | 4808 kwdict = update_keyword_args(NULL, nk, pp_stack, func); |
|
| 4738 if (kwdict == NULL) | 4809 if (kwdict == NULL) |
|
| 4739 goto call_fail; | 4810 goto call_fail; |
|
| 4740 } | 4811 } |
|
| 4741 callargs = load_args(pp_stack, na); | 4812 callargs = load_args(pp_stack, na); |
|
| 4742 if (callargs == NULL) | 4813 if (callargs == NULL) |
|
| 4743 goto call_fail; | 4814 goto call_fail; |
|
| 4744 #ifdef CALL_PROFILE | 4815 #ifdef CALL_PROFILE |
|
| 4745 /* At this point, we have to look at the type of func to | 4816 /* At this point, we have to look at the type of func to |
|
| 4746 update the call stats properly. Do it here so as to avoid | 4817 update the call stats properly. Do it here so as to avoid |
|
| 4747 exposing the call stats machinery outside eval.cc. | 4818 exposing the call stats machinery outside eval.cc. |
|
| 4748 */ | 4819 */ |
|
| 4749 if (PyFunction_Check(func)) | 4820 if (PyFunction_Check(func)) |
|
| 4750 PCALL(PCALL_FUNCTION); | 4821 PCALL(PCALL_FUNCTION); |
|
| 4751 else if (PyMethod_Check(func)) | 4822 else if (PyMethod_Check(func)) |
|
| 4752 PCALL(PCALL_METHOD); | 4823 PCALL(PCALL_METHOD); |
|
| 4753 else if (PyType_Check(func)) | 4824 else if (PyType_Check(func)) |
|
| 4754 PCALL(PCALL_TYPE); | 4825 PCALL(PCALL_TYPE); |
|
| 4755 else if (PyCFunction_Check(func)) | 4826 else if (PyCFunction_Check(func)) |
|
| 4756 PCALL(PCALL_CFUNCTION); | 4827 PCALL(PCALL_CFUNCTION); |
|
| 4757 else | 4828 else |
|
| 4758 PCALL(PCALL_OTHER); | 4829 PCALL(PCALL_OTHER); |
|
| 4759 #endif | 4830 #endif |
|
| 4760 » if (PyCFunction_Check(func)) { | 4831 » if (ml) { |
|
| 4761 PyThreadState *tstate = PyThreadState_GET(); | 4832 PyThreadState *tstate = PyThreadState_GET(); |
|
| 4762 » » C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); | 4833 » » C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict)); |
|
| 4763 } | 4834 } |
|
| 4764 else | 4835 else |
|
| 4765 result = PyObject_Call(func, callargs, kwdict); | 4836 result = PyObject_Call(func, callargs, kwdict); |
|
| 4766 call_fail: | 4837 call_fail: |
|
| 4767 Py_XDECREF(callargs); | 4838 Py_XDECREF(callargs); |
|
| 4768 Py_XDECREF(kwdict); | 4839 Py_XDECREF(kwdict); |
|
| 4769 return result; | 4840 return result; |
|
| 4770 } | 4841 } |
|
| 4771 | 4842 |
|
| 4772 static PyObject * | 4843 static PyObject * |
|
| 4773 ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) | 4844 ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk) |
|
| 4774 { | 4845 { |
|
| 4775 int nstar = 0; | 4846 int nstar = 0; |
|
| 4776 PyObject *callargs = NULL; | 4847 PyObject *callargs = NULL; |
|
| 4777 PyObject *stararg = NULL; | 4848 PyObject *stararg = NULL; |
|
| 4778 PyObject *kwdict = NULL; | 4849 PyObject *kwdict = NULL; |
|
| 4779 PyObject *result = NULL; | 4850 PyObject *result = NULL; |
|
| 4851 PyMethodDef *ml = NULL; |
|
| 4852 PyObject *self = NULL; |
|
| 4853 |
|
| 4854 if (PyCFunction_Check(func)) { |
|
| 4855 ml = PyCFunction_GET_METHODDEF(func); |
|
| 4856 self = PyCFunction_GET_SELF(func); |
|
| 4857 } |
|
| 4858 else if (PyMethodDescr_Check(func)) { |
|
| 4859 /* Only apply C calling optimization if self is on the stack |
|
| 4860 * and not the first element of callargs. */ |
|
| 4861 if (na > 0) { |
|
| 4862 ml = ((PyMethodDescrObject*)func)->d_method; |
|
| 4863 self = (*pp_stack)[-(na + 2 * nk)]; |
|
| 4864 na--; |
|
| 4865 } |
|
| 4866 } |
|
| 4780 | 4867 |
|
| 4781 if (flags & CALL_FLAG_KW) { | 4868 if (flags & CALL_FLAG_KW) { |
|
| 4782 kwdict = EXT_POP(*pp_stack); | 4869 kwdict = EXT_POP(*pp_stack); |
|
| 4783 if (!PyDict_Check(kwdict)) { | 4870 if (!PyDict_Check(kwdict)) { |
|
| 4784 PyObject *d; | 4871 PyObject *d; |
|
| 4785 d = PyDict_New(); | 4872 d = PyDict_New(); |
|
| 4786 if (d == NULL) | 4873 if (d == NULL) |
|
| 4787 goto ext_call_fail; | 4874 goto ext_call_fail; |
|
| 4788 if (PyDict_Update(d, kwdict) != 0) { | 4875 if (PyDict_Update(d, kwdict) != 0) { |
|
| 4789 Py_DECREF(d); | 4876 Py_DECREF(d); |
|
| (...skipping 55 matching lines...) | | Loading... |
| 4845 PCALL(PCALL_FUNCTION); | 4932 PCALL(PCALL_FUNCTION); |
|
| 4846 else if (PyMethod_Check(func)) | 4933 else if (PyMethod_Check(func)) |
|
| 4847 PCALL(PCALL_METHOD); | 4934 PCALL(PCALL_METHOD); |
|
| 4848 else if (PyType_Check(func)) | 4935 else if (PyType_Check(func)) |
|
| 4849 PCALL(PCALL_TYPE); | 4936 PCALL(PCALL_TYPE); |
|
| 4850 else if (PyCFunction_Check(func)) | 4937 else if (PyCFunction_Check(func)) |
|
| 4851 PCALL(PCALL_CFUNCTION); | 4938 PCALL(PCALL_CFUNCTION); |
|
| 4852 else | 4939 else |
|
| 4853 PCALL(PCALL_OTHER); | 4940 PCALL(PCALL_OTHER); |
|
| 4854 #endif | 4941 #endif |
|
| 4855 » if (PyCFunction_Check(func)) { | 4942 » if (ml) { |
|
| 4856 PyThreadState *tstate = PyThreadState_GET(); | 4943 PyThreadState *tstate = PyThreadState_GET(); |
|
| 4857 » » C_TRACE(result, PyCFunction_Call(func, callargs, kwdict)); | 4944 » » C_TRACE(result, PyMethodDef_Call(ml, self, callargs, kwdict)); |
|
| 4858 } | 4945 } |
|
| 4859 else | 4946 else |
|
| 4860 result = PyObject_Call(func, callargs, kwdict); | 4947 result = PyObject_Call(func, callargs, kwdict); |
|
| 4861 ext_call_fail: | 4948 ext_call_fail: |
|
| 4862 Py_XDECREF(callargs); | 4949 Py_XDECREF(callargs); |
|
| 4863 Py_XDECREF(kwdict); | 4950 Py_XDECREF(kwdict); |
|
| 4864 Py_XDECREF(stararg); | 4951 Py_XDECREF(stararg); |
|
| 4865 return result; | 4952 return result; |
|
| 4866 } | 4953 } |
|
| 4867 | 4954 |
|
| (...skipping 534 matching lines...) | | Loading... |
| 5402 Py_DECREF(l); | 5489 Py_DECREF(l); |
|
| 5403 return NULL; | 5490 return NULL; |
|
| 5404 } | 5491 } |
|
| 5405 PyList_SetItem(l, i, x); | 5492 PyList_SetItem(l, i, x); |
|
| 5406 } | 5493 } |
|
| 5407 return l; | 5494 return l; |
|
| 5408 #endif | 5495 #endif |
|
| 5409 } | 5496 } |
|
| 5410 | 5497 |
|
| 5411 #endif | 5498 #endif |
|
| LEFT | RIGHT |
|---|