Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 1de16c7

Browse files
Merge branch 'PHP-8.4'
* PHP-8.4: Snapshotted poly_func / poly_this may be spilled
2 parents 4122daa + 18276a8 commit 1de16c7

File tree

3 files changed

+139
-69
lines changed

3 files changed

+139
-69
lines changed

‎ext/opcache/jit/zend_jit_internal.h‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -437,16 +437,22 @@ struct _zend_jit_trace_rec {
437437

438438
#define ZEND_JIT_TRACE_START_REC_SIZE 2
439439

440+
typedef struct _zend_jit_ref_snapshot {
441+
union {
442+
int32_t ref; /* While generating code: The ir_ref to snapshot */
443+
int32_t offset; /* After compilation / during deopt: C stack offset if 'reg' is spilled */
444+
};
445+
int8_t reg; /* Set after compilation by zend_jit_snapshot_handler() */
446+
} zend_jit_ref_snapshot;
447+
440448
typedef struct _zend_jit_trace_exit_info {
441-
const zend_op *opline; /* opline where VM should continue execution */
442-
const zend_op_array *op_array;
443-
uint32_t flags; /* set of ZEND_JIT_EXIT_... */
444-
uint32_t stack_size;
445-
uint32_t stack_offset;
446-
int32_t poly_func_ref;
447-
int32_t poly_this_ref;
448-
int8_t poly_func_reg;
449-
int8_t poly_this_reg;
449+
const zend_op *opline; /* opline where VM should continue execution */
450+
const zend_op_array *op_array;
451+
uint32_t flags; /* set of ZEND_JIT_EXIT_... */
452+
uint32_t stack_size;
453+
uint32_t stack_offset;
454+
zend_jit_ref_snapshot poly_func;
455+
zend_jit_ref_snapshot poly_this;
450456
} zend_jit_trace_exit_info;
451457

452458
typedef struct _zend_jit_trace_stack {

‎ext/opcache/jit/zend_jit_ir.c‎

Lines changed: 73 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,8 @@ typedef struct _zend_jit_ctx {
277277
ir_ref tls;
278278
#endif
279279
ir_ref fp;
280+
ir_ref poly_func_ref; /* restored from parent trace snapshot */
281+
ir_ref poly_this_ref; /* restored from parent trace snapshot */
280282
ir_ref trace_loop_ref;
281283
ir_ref return_inputs;
282284
const zend_op_array *op_array;
@@ -624,12 +626,12 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
624626
uint32_t exit_point = 0, n = 0;
625627

626628
if (addr < 0) {
627-
if (t->exit_count > 0
628-
&& jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
629-
exit_point = t->exit_count - 1;
630-
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
631-
n = 2;
632-
}
629+
/* addr is not always the address of the *last* exit point,
630+
* so we can not optimize this to 'exit_point = t->exit_count-1' */
631+
exit_point = zend_jit_exit_point_by_addr(ptr);
632+
ZEND_ASSERT(exit_point != -1);
633+
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
634+
n = 2;
633635
}
634636
}
635637

@@ -660,8 +662,8 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
660662
ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
661663
}
662664
if (n) {
663-
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
664-
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
665+
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref);
666+
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref);
665667
}
666668
}
667669
}
@@ -710,6 +712,31 @@ uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint
710712
return new_exit_point;
711713
}
712714

715+
static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, int op)
716+
{
717+
int8_t *reg_ops = ctx->regs[snapshot_ref];
718+
ZEND_ASSERT(reg_ops[op] != ZREG_NONE);
719+
720+
int8_t reg = reg_ops[op];
721+
int32_t offset;
722+
723+
if (IR_REG_SPILLED(reg)) {
724+
reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD;
725+
offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op));
726+
} else {
727+
offset = 0;
728+
}
729+
730+
dest->reg = reg;
731+
dest->offset = offset;
732+
}
733+
734+
static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const zend_jit_ref_snapshot *b)
735+
{
736+
return a->reg == b->reg
737+
&& (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset));
738+
}
739+
713740
void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
714741
{
715742
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
@@ -722,18 +749,19 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps
722749
exit_flags = t->exit_info[exit_point].flags;
723750

724751
if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
725-
int8_t *reg_ops = ctx->regs[snapshot_ref];
752+
zend_jit_ref_snapshot func, this;
753+
zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1);
754+
zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n);
726755

727-
ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
728756
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
729-
&& (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
730-
|| t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
757+
&& (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func)
758+
|| !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) {
731759
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
732760
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
733761
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
734762
}
735-
t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
736-
t->exit_info[exit_point].poly_this_reg = reg_ops[n];
763+
t->exit_info[exit_point].poly_func = func;
764+
t->exit_info[exit_point].poly_this = this;
737765
n -= 2;
738766
}
739767

@@ -2709,6 +2737,8 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
27092737
jit->tls = IR_UNUSED;
27102738
#endif
27112739
jit->fp = IR_UNUSED;
2740+
jit->poly_func_ref = IR_UNUSED;
2741+
jit->poly_this_ref = IR_UNUSED;
27122742
jit->trace_loop_ref = IR_UNUSED;
27132743
jit->return_inputs = IR_UNUSED;
27142744
jit->bb_start_ref = NULL;
@@ -4386,6 +4416,18 @@ static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
43864416
return ir_RLOAD(type, reg);
43874417
}
43884418

4419+
/* Same as zend_jit_deopt_rload(), but 'reg' may be spilled on C stack */
4420+
static ir_ref zend_jit_deopt_rload_spilled(zend_jit_ctx *jit, ir_type type, int8_t reg, int32_t offset)
4421+
{
4422+
ZEND_ASSERT(reg >= 0);
4423+
4424+
if (IR_REG_SPILLED(reg)) {
4425+
return ir_LOAD(type, ir_ADD_OFFSET(zend_jit_deopt_rload(jit, type, IR_REG_NUM(reg)), offset));
4426+
} else {
4427+
return zend_jit_deopt_rload(jit, type, reg);
4428+
}
4429+
}
4430+
43894431
static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
43904432
{
43914433
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
@@ -8440,10 +8482,9 @@ static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32
84408482
return 1;
84418483
}
84428484

8443-
static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
8485+
static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func)
84448486
{
84458487
// JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
8446-
ir_ref func = ir_RLOAD_A(func_reg);
84478488
ir_ref if_trampoline = ir_IF(ir_AND_U32(
84488489
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
84498490
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
@@ -8537,8 +8578,8 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co
85378578
}
85388579

85398580
if (may_be_trampoline) {
8540-
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
8541-
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
8581+
jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
8582+
jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
85428583
}
85438584

85448585
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
@@ -8936,15 +8977,15 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
89368977
zend_class_entry *trace_ce,
89378978
zend_jit_trace_rec *trace,
89388979
int checked_stack,
8939-
int8_t func_reg,
8940-
int8_t this_reg,
8980+
ir_ref func_ref,
8981+
ir_ref this_ref,
89418982
bool polymorphic_side_trace)
89428983
{
89438984
zend_func_info *info = ZEND_FUNC_INFO(op_array);
89448985
zend_call_info *call_info = NULL;
89458986
zend_function *func = NULL;
89468987
zval *function_name;
8947-
ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
8988+
ir_ref if_static = IR_UNUSED, cold_path;
89488989

89498990
ZEND_ASSERT(opline->op2_type == IS_CONST);
89508991
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
@@ -8962,10 +9003,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
89629003
}
89639004

89649005
if (polymorphic_side_trace) {
8965-
/* function is passed in r0 from parent_trace */
8966-
ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
8967-
func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8968-
this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
9006+
/* function is passed from parent snapshot */
9007+
ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED);
89699008
} else {
89709009
ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
89719010

@@ -9111,8 +9150,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
91119150
return 0;
91129151
}
91139152

9114-
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
9115-
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
9153+
jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
9154+
jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
91169155

91179156
func = (zend_function*)trace->func;
91189157

@@ -17319,9 +17358,13 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
1731917358
}
1732017359

1732117360
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
17322-
ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
17323-
ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
17324-
ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
17361+
ZEND_ASSERT(parent->exit_info[exit_num].poly_func.reg >= 0 && parent->exit_info[exit_num].poly_this.reg >= 0);
17362+
if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func.reg)) {
17363+
ir_RLOAD_A(parent->exit_info[exit_num].poly_func.reg);
17364+
}
17365+
if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this.reg)) {
17366+
ir_RLOAD_A(parent->exit_info[exit_num].poly_this.reg);
17367+
}
1732517368
}
1732617369

1732717370
ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));

‎ext/opcache/jit/zend_jit_trace.c‎

Lines changed: 51 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,8 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t
199199
t->exit_info[exit_point].flags = flags;
200200
t->exit_info[exit_point].stack_size = stack_size;
201201
t->exit_info[exit_point].stack_offset = stack_offset;
202-
t->exit_info[exit_point].poly_func_ref = 0;
203-
t->exit_info[exit_point].poly_this_ref = 0;
204-
t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
205-
t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
202+
t->exit_info[exit_point].poly_func = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
203+
t->exit_info[exit_point].poly_this = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
206204
}
207205

208206
return exit_point;
@@ -3514,17 +3512,18 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t
35143512
}
35153513

35163514
static int zend_jit_trace_deoptimization(
3517-
zend_jit_ctx *jit,
3518-
uint32_t flags,
3519-
const zend_op *opline,
3520-
zend_jit_trace_stack *parent_stack,
3521-
int parent_vars_count,
3522-
zend_ssa *ssa,
3523-
zend_jit_trace_stack *stack,
3524-
zend_jit_exit_const *constants,
3525-
int8_t func_reg,
3526-
bool polymorphic_side_trace)
3515+
zend_jit_ctx *jit,
3516+
const zend_jit_trace_exit_info *exit_info,
3517+
zend_jit_trace_stack *parent_stack,
3518+
int parent_vars_count,
3519+
zend_ssa *ssa,
3520+
zend_jit_trace_stack *stack,
3521+
zend_jit_exit_const *constants,
3522+
bool polymorphic_side_trace)
35273523
{
3524+
uint32_t flags = exit_info->flags;
3525+
const zend_op *opline = exit_info->opline;
3526+
35283527
int i;
35293528
int check2 = -1;
35303529

@@ -3668,9 +3667,16 @@ static int zend_jit_trace_deoptimization(
36683667
zend_jit_check_exception(jit);
36693668
}
36703669

3671-
if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3672-
if (!zend_jit_free_trampoline(jit, func_reg)) {
3673-
return 0;
3670+
if (flags & ZEND_JIT_EXIT_METHOD_CALL) {
3671+
jit->poly_func_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
3672+
exit_info->poly_func.reg, exit_info->poly_func.offset);
3673+
jit->poly_this_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
3674+
exit_info->poly_this.reg, exit_info->poly_this.offset);
3675+
3676+
if (!polymorphic_side_trace) {
3677+
if (!zend_jit_free_trampoline(jit, jit->poly_func_ref)) {
3678+
return 0;
3679+
}
36743680
}
36753681
}
36763682

@@ -4265,11 +4271,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
42654271
if (parent_trace) {
42664272
/* Deoptimization */
42674273
if (!zend_jit_trace_deoptimization(&ctx,
4268-
zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4269-
zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4274+
&zend_jit_traces[parent_trace].exit_info[exit_num],
42704275
parent_stack, parent_vars_count, ssa, stack,
42714276
zend_jit_traces[parent_trace].constants,
4272-
zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
42734277
polymorphic_side_trace)) {
42744278
goto jit_failure;
42754279
}
@@ -6381,8 +6385,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
63816385
op_array, ssa, ssa_op, frame->call_level,
63826386
op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
63836387
p + 1, peek_checked_stack - checked_stack,
6384-
polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6385-
polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6388+
polymorphic_side_trace ? jit->poly_func_ref : -1,
6389+
polymorphic_side_trace ? jit->poly_this_ref : -1,
63866390
polymorphic_side_trace)) {
63876391
goto jit_failure;
63886392
}
@@ -7419,11 +7423,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n
74197423
NULL;
74207424

74217425
if (!zend_jit_trace_deoptimization(&ctx,
7422-
zend_jit_traces[trace_num].exit_info[exit_num].flags,
7423-
zend_jit_traces[trace_num].exit_info[exit_num].opline,
7426+
&zend_jit_traces[trace_num].exit_info[exit_num],
74247427
stack, stack_size, NULL, NULL,
74257428
zend_jit_traces[trace_num].constants,
7426-
zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
74277429
0)) {
74287430
goto jit_failure;
74297431
}
@@ -7973,6 +7975,17 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
79737975
}
79747976
}
79757977

7978+
static void zend_jit_dump_ref_snapshot(zend_jit_ref_snapshot *rs)
7979+
{
7980+
if (rs->reg == ZREG_NONE) {
7981+
fprintf(stderr, "?");
7982+
} else if (!IR_REG_SPILLED(rs->reg)) {
7983+
fprintf(stderr, "%s", zend_reg_name(rs->reg));
7984+
} else {
7985+
fprintf(stderr, "0x%x(%s)", rs->offset, zend_reg_name(IR_REG_NUM(rs->reg)));
7986+
}
7987+
}
7988+
79767989
static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
79777990
{
79787991
int i, j;
@@ -8003,9 +8016,11 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
80038016
if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
80048017
fprintf(stderr, "/POLY");
80058018
if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8006-
fprintf(stderr, "(%s, %s)",
8007-
t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
8008-
t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
8019+
fprintf(stderr, "(");
8020+
zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_func);
8021+
fprintf(stderr, ", ");
8022+
zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_this);
8023+
fprintf(stderr, ")");
80098024
}
80108025
}
80118026
if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
@@ -8737,9 +8752,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
87378752
}
87388753
}
87398754
if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8740-
ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8741-
zend_function*func= (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8755+
zend_jit_ref_snapshot*func_snapshot=&t->exit_info[exit_num].poly_func;
8756+
ZEND_ASSERT(func_snapshot->reg >= 0);
87428757

8758+
zend_function *func;
8759+
if (IR_REG_SPILLED(func_snapshot->reg)) {
8760+
func = *(zend_function**)(regs->gpr[IR_REG_NUM(func_snapshot->reg)] + func_snapshot->offset);
8761+
} else {
8762+
func = (zend_function*)regs->gpr[func_snapshot->reg];
8763+
}
87438764
if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
87448765
zend_string_release_ex(func->common.function_name, 0);
87458766
zend_free_trampoline(func);

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /