diff -r 96c88daac37a Python/ceval.c --- a/Python/ceval.c Wed Jan 27 11:52:13 2016 -0500 +++ b/Python/ceval.c Thu Feb 04 00:51:24 2016 -0500 @@ -147,6 +147,15 @@ PyFrameObject *, unsigned char *); static PyObject * special_lookup(PyObject *, _Py_Identifier *); +static int fast_add(PyObject *, PyObject *, PyObject **); +static int fast_sub(PyObject *, PyObject *, PyObject **); +static int fast_mul(PyObject *, PyObject *, PyObject **); +static int fast_floor_div(PyObject *, PyObject *, PyObject **); +static int fast_true_div(PyObject *, PyObject *, PyObject **); + +#define SINGLE_DIGIT_LONG_AS_LONG(op) \ + ((((PyLongObject*)(op))->ob_digit[0]) * Py_SIZE(op)) + #define NAME_ERROR_MSG \ "name '%.200s' is not defined" #define UNBOUNDLOCAL_ERROR_MSG \ @@ -1497,12 +1506,25 @@ TARGET(BINARY_MULTIPLY) { PyObject *right = POP(); PyObject *left = TOP(); - PyObject *res = PyNumber_Multiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) + PyObject *res; + + if (fast_mul(left, right, &res)) { + SET_TOP(NULL); goto error; + } + + if (res == NULL) { + res = PyNumber_Multiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + SET_TOP(res); + if (res == NULL) { + goto error; + } + } else { + SET_TOP(res); + } + DISPATCH(); } @@ -1521,24 +1543,52 @@ TARGET(BINARY_TRUE_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *quotient = PyNumber_TrueDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) + PyObject *quotient; + + if (fast_true_div(dividend, divisor, "ient)) { + SET_TOP(NULL); goto error; + } + + if (quotient == NULL) { + quotient = PyNumber_TrueDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + + SET_TOP(quotient); + if (quotient == NULL) { + goto error; + } + } else { + SET_TOP(quotient); + } + DISPATCH(); } TARGET(BINARY_FLOOR_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *quotient = PyNumber_FloorDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) + PyObject *quotient; + + if (fast_floor_div(dividend, divisor, "ient)) { + SET_TOP(NULL); goto error; + } + + if (quotient == NULL) { + quotient = PyNumber_FloorDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + + SET_TOP(quotient); + if (quotient == NULL) { + goto error; + } + } else { + SET_TOP(quotient); + } + DISPATCH(); } @@ -1560,31 +1610,59 @@ PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; - if (PyUnicode_CheckExact(left) && - PyUnicode_CheckExact(right)) { - sum = unicode_concatenate(left, right, f, next_instr); - /* unicode_concatenate consumed the ref to v */ + + if (fast_add(left, right, &sum)) { + SET_TOP(NULL); + goto error; } - else { - sum = PyNumber_Add(left, right); - Py_DECREF(left); + + if (sum == NULL) { + if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { + /* fast path for string concatenation */ + sum = unicode_concatenate(left, right, f, next_instr); + /* unicode_concatenate consumed the ref to left */ + Py_DECREF(right); + } + else { + sum = PyNumber_Add(left, right); + Py_DECREF(left); + Py_DECREF(right); + } + + SET_TOP(sum); + if (sum == NULL) { + goto error; + } + } else { + SET_TOP(sum); } - Py_DECREF(right); - SET_TOP(sum); - if (sum == NULL) - goto error; + DISPATCH(); } TARGET(BINARY_SUBTRACT) { PyObject *right = POP(); PyObject *left = TOP(); - PyObject *diff = PyNumber_Subtract(left, right); - Py_DECREF(right); - Py_DECREF(left); - SET_TOP(diff); - if (diff == NULL) + PyObject *diff; + + if (fast_sub(left, right, &diff)) { + SET_TOP(NULL); goto error; + } + + if (diff == NULL) { + diff = PyNumber_Subtract(left, right); + Py_DECREF(right); + Py_DECREF(left); + + SET_TOP(diff); + if (diff == NULL) { + goto error; + } + } else { + SET_TOP(diff); + } + DISPATCH(); } @@ -1699,12 +1777,26 @@ TARGET(INPLACE_MULTIPLY) { PyObject *right = POP(); PyObject *left = TOP(); - PyObject *res = PyNumber_InPlaceMultiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) + PyObject *res; + + if (fast_mul(left, right, &res)) { + SET_TOP(NULL); goto error; + } + + if (res == NULL) { + res = PyNumber_InPlaceMultiply(left, right); + Py_DECREF(left); + Py_DECREF(right); + + SET_TOP(res); + if (res == NULL) { + goto error; + } + } else { + SET_TOP(res); + } + DISPATCH(); } @@ -1723,24 +1815,52 @@ TARGET(INPLACE_TRUE_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *quotient = PyNumber_InPlaceTrueDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) + PyObject *quotient; + + if (fast_true_div(dividend, divisor, "ient)) { + SET_TOP(NULL); goto error; + } + + if (quotient == NULL) { + quotient = PyNumber_InPlaceTrueDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + + SET_TOP(quotient); + if (quotient == NULL) { + goto error; + } + } else { + SET_TOP(quotient); + } + DISPATCH(); } TARGET(INPLACE_FLOOR_DIVIDE) { PyObject *divisor = POP(); PyObject *dividend = TOP(); - PyObject *quotient = PyNumber_InPlaceFloorDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) + PyObject *quotient; + + if (fast_floor_div(dividend, divisor, "ient)) { + SET_TOP(NULL); goto error; + } + + if (quotient == NULL) { + quotient = PyNumber_InPlaceFloorDivide(dividend, divisor); + Py_DECREF(dividend); + Py_DECREF(divisor); + + SET_TOP(quotient); + if (quotient == NULL) { + goto error; + } + } else { + SET_TOP(quotient); + } + DISPATCH(); } @@ -1760,30 +1880,58 @@ PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; - if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { - sum = unicode_concatenate(left, right, f, next_instr); - /* unicode_concatenate consumed the ref to v */ + + if (fast_add(left, right, &sum)) { + SET_TOP(NULL); + goto error; } - else { - sum = PyNumber_InPlaceAdd(left, right); - Py_DECREF(left); + + if (sum == NULL) { + if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { + sum = unicode_concatenate(left, right, f, next_instr); + /* unicode_concatenate consumed the ref to 'left' */ + Py_DECREF(right); + } + else { + sum = PyNumber_InPlaceAdd(left, right); + Py_DECREF(left); + Py_DECREF(right); + } + + SET_TOP(sum); + if (sum == NULL) { + goto error; + } + } else { + SET_TOP(sum); } - Py_DECREF(right); - SET_TOP(sum); - if (sum == NULL) - goto error; + DISPATCH(); } TARGET(INPLACE_SUBTRACT) { PyObject *right = POP(); PyObject *left = TOP(); - PyObject *diff = PyNumber_InPlaceSubtract(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(diff); - if (diff == NULL) + PyObject *diff; + + if (fast_sub(left, right, &diff)) { + SET_TOP(NULL); goto error; + } + + if (diff == NULL) { + diff = PyNumber_InPlaceSubtract(left, right); + Py_DECREF(left); + Py_DECREF(right); + + SET_TOP(diff); + if (diff == NULL) { + goto error; + } + } else { + SET_TOP(diff); + } + DISPATCH(); } @@ -5318,6 +5466,360 @@ return res; } +static int +fast_add(PyObject *left, PyObject *right, PyObject **result) +{ + PyObject *sum; + double left_d, right_d; + int l_float, r_float; + int l_long = PyLong_CheckExact(left) && Py_ABS(Py_SIZE(left)) <= 1; + int r_long = PyLong_CheckExact(right) && Py_ABS(Py_SIZE(right)) <= 1; + + if (l_long && r_long) { + if (Py_SIZE(left) != 0) { + if (Py_SIZE(right) != 0) { + /* This will never overflow, as digits in + PyLong are 30 bits max */ + sum = PyLong_FromLong( + SINGLE_DIGIT_LONG_AS_LONG(left) + + SINGLE_DIGIT_LONG_AS_LONG(right)); + + Py_DECREF(left); + Py_DECREF(right); + + *result = sum; + return sum == NULL; + } + else { + /* right == 0 */ + Py_DECREF(right); + *result = left; + return 0; + } + } + else { + /* left == 0 */ + Py_DECREF(left); + *result = right; + return 0; + } + } + + r_float = PyFloat_CheckExact(right); + if (l_long && r_float) { + left_d = (double)SINGLE_DIGIT_LONG_AS_LONG(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + l_float = PyFloat_CheckExact(left); + if (l_float && r_long) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = (double)SINGLE_DIGIT_LONG_AS_LONG(right); + goto calc_float; + } + + if (l_float && r_float) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + *result = NULL; + return 0; + + calc_float: + Py_DECREF(left); + Py_DECREF(right); + + PyFPE_START_PROTECT("add", return -1) + left_d = left_d + right_d; + PyFPE_END_PROTECT(left_d) + *result = PyFloat_FromDouble(left_d); + return *result == NULL; +} + +static int +fast_sub(PyObject *left, PyObject *right, PyObject **result) +{ + PyObject *sub; + double left_d, right_d; + int l_float, r_float; + int l_long = PyLong_CheckExact(left) && Py_ABS(Py_SIZE(left)) <= 1; + int r_long = PyLong_CheckExact(right) && Py_ABS(Py_SIZE(right)) <= 1; + + if (l_long && r_long) { + if (Py_SIZE(left) != 0) { + if (Py_SIZE(right) != 0) { + /* This will never overflow, as digits in + PyLong are 30 bits max */ + sub = PyLong_FromLong( + SINGLE_DIGIT_LONG_AS_LONG(left) - + SINGLE_DIGIT_LONG_AS_LONG(right)); + + Py_DECREF(left); + Py_DECREF(right); + + *result = sub; + return sub == NULL; + } + else { + /* left != 0 && right == 0: + return left + */ + Py_DECREF(right); + *result = left; + return 0; + } + } + else { + /* left == 0 && right is something: + return (-right) + */ + Py_DECREF(left); + *result = PyLong_FromLong(-SINGLE_DIGIT_LONG_AS_LONG(right)); + return 0; + } + } + + r_float = PyFloat_CheckExact(right); + if (l_long && r_float) { + left_d = (double)SINGLE_DIGIT_LONG_AS_LONG(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + l_float = PyFloat_CheckExact(left); + if (l_float && r_long) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = (double)SINGLE_DIGIT_LONG_AS_LONG(right); + goto calc_float; + } + + if (l_float && r_float) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + *result = NULL; + return 0; + + calc_float: + Py_DECREF(left); + Py_DECREF(right); + + PyFPE_START_PROTECT("subtract", return -1) + left_d = left_d - right_d; + PyFPE_END_PROTECT(left_d) + *result = PyFloat_FromDouble(left_d); + return *result == NULL; +} + +static int +fast_mul(PyObject *left, PyObject *right, PyObject **result) +{ + PyObject *mul; + double left_d, right_d; + int l_float, r_float; + int l_long = PyLong_CheckExact(left) && Py_ABS(Py_SIZE(left)) <= 1; + int r_long = PyLong_CheckExact(right) && Py_ABS(Py_SIZE(right)) <= 1; + + if (l_long && r_long) { + if (Py_SIZE(left) != 0) { + if (Py_SIZE(right) != 0) { + +#ifdef HAVE_LONG_LONG + mul = PyLong_FromLongLong( + (long long)SINGLE_DIGIT_LONG_AS_LONG(left) * + SINGLE_DIGIT_LONG_AS_LONG(right)); +#else + mul = PyNumber_Multiply(left, right); +#endif + + Py_DECREF(left); + Py_DECREF(right); + + *result = mul; + return mul == NULL; + } + else { + /* left != 0 && right == 0: + return right + */ + Py_DECREF(left); + *result = right; + return 0; + } + } + else { + /* left == 0 && right == something: + return left + */ + Py_DECREF(right); + *result = left; + return 0; + } + } + + r_float = PyFloat_CheckExact(right); + if (l_long && r_float) { + left_d = (double)SINGLE_DIGIT_LONG_AS_LONG(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + l_float = PyFloat_CheckExact(left); + if (l_float && r_long) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = (double)SINGLE_DIGIT_LONG_AS_LONG(right); + goto calc_float; + } + + if (l_float && r_float) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + *result = NULL; + return 0; + + calc_float: + Py_DECREF(left); + Py_DECREF(right); + + PyFPE_START_PROTECT("multiply", return -1) + left_d = left_d * right_d; + PyFPE_END_PROTECT(left_d) + *result = PyFloat_FromDouble(left_d); + return *result == NULL; +} + +static int +fast_floor_div(PyObject *left, PyObject *right, PyObject **result) +{ + PyObject *res; + int l_long = PyLong_CheckExact(left) && Py_ABS(Py_SIZE(left)) <= 1; + int r_long = PyLong_CheckExact(right) && Py_ABS(Py_SIZE(right)) <= 1; + + if (l_long && r_long) { + if (Py_SIZE(left) != 0) { + if (Py_SIZE(right) != 0) { + long a, b, adivb; + + a = ((PyLongObject*)left)->ob_digit[0]; + b = ((PyLongObject*)right)->ob_digit[0]; + + if (Py_SIZE(left) != Py_SIZE(right)) { + a *= Py_SIZE(left); + b *= Py_SIZE(right); + + adivb = a / b; + if (a - adivb * b) { + /* we want floor */ + --adivb; + } + + res = PyLong_FromLong(adivb); + } else { + res = PyLong_FromLong(a / b); + } + + Py_DECREF(left); + Py_DECREF(right); + + *result = res; + return res == NULL; + } + else { + /* left != 0 && right == 0: + raise ZeroDivisionError + */ + Py_DECREF(left); + Py_DECREF(right); + + PyErr_SetString(PyExc_ZeroDivisionError, + "integer division or modulo by zero"); + *result = NULL; + return -1; + } + } + else { + /* left == 0 && right == something: + return left + */ + Py_DECREF(right); + *result = left; + return 0; + } + } + + *result = NULL; + return 0; +} + +static int +fast_true_div(PyObject *left, PyObject *right, PyObject **result) +{ + double left_d, right_d; + int l_float, r_float; + int l_long = PyLong_CheckExact(left) && Py_ABS(Py_SIZE(left)) <= 1; + int r_long = PyLong_CheckExact(right) && Py_ABS(Py_SIZE(right)) <= 1; + + if (l_long && r_long) { + if (!Py_SIZE(right)) { + Py_DECREF(left); + Py_DECREF(right); + PyErr_SetString(PyExc_ZeroDivisionError, + "division by zero"); + *result = NULL; + return -1; + } + + left_d = (double)SINGLE_DIGIT_LONG_AS_LONG(left); + right_d = (double)SINGLE_DIGIT_LONG_AS_LONG(right); + goto calc_float; + } + + r_float = PyFloat_CheckExact(right); + if (l_long && r_float) { + left_d = (double)SINGLE_DIGIT_LONG_AS_LONG(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + l_float = PyFloat_CheckExact(left); + if (l_float && r_long) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = (double)SINGLE_DIGIT_LONG_AS_LONG(right); + goto calc_float; + } + + if (l_float && r_float) { + left_d = PyFloat_AS_DOUBLE(left); + right_d = PyFloat_AS_DOUBLE(right); + goto calc_float; + } + + *result = NULL; + return 0; + + calc_float: + Py_DECREF(left); + Py_DECREF(right); + if (right_d == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, + "float division by zero"); + return -1; + } + PyFPE_START_PROTECT("divide", return -1) + left_d = left_d / right_d; + PyFPE_END_PROTECT(left_d) + *result = PyFloat_FromDouble(left_d); + return *result == NULL; +} + #ifdef DYNAMIC_EXECUTION_PROFILE static PyObject *

AltStyle によって変換されたページ (->オリジナル) /