Implement stackless internal function calls

Co-authored-by: Dmitry Stogov <dmitry@zend.com>

Closes GH-12461

Author: iluuu1994

UPGRADING.INTERNALS

@@ -114,6 +114,12 @@ PHP 8.4 INTERNALS UPGRADE NOTES
 4. OpCode changes
 ========================
 
+* New FRAMELESS_ICALL_[0,3] opcodes for faster internal function calls have been
+  added. These opcodes don't create a stack frame, but pass arguments via opcode
+  operands. They only work for functions that are known at compile-time, and
+  that provide a frameless handler (search for usages of the
+  ZEND_FRAMELESS_FUNCTION macro).
+
 ========================
 5. SAPI changes
 ========================

Zend/Optimizer/block_pass.c

@@ -1017,6 +1017,7 @@ static void assemble_code_blocks(zend_cfg *cfg, zend_op_array *op_array, zend_op
 			case ZEND_ASSERT_CHECK:
 			case ZEND_JMP_NULL:
 			case ZEND_BIND_INIT_STATIC_OR_JMP:
+			case ZEND_JMP_FRAMELESS:
 				ZEND_SET_OP_JMP_ADDR(opline, opline->op2, new_opcodes + blocks[b->successors[0]].start);
 				break;
 			case ZEND_CATCH:

Zend/Optimizer/compact_literals.c

@@ -770,6 +770,7 @@ void zend_optimizer_compact_literals(zend_op_array *op_array, zend_optimizer_ctx
 					break;
 				case ZEND_DECLARE_ANON_CLASS:
 				case ZEND_DECLARE_CLASS_DELAYED:
+				case ZEND_JMP_FRAMELESS:
 					opline->extended_value = cache_size;
 					cache_size += sizeof(void *);
 					break;

Zend/Optimizer/dce.c

@@ -146,6 +146,7 @@ static inline bool may_have_side_effects(
 		case ZEND_ASSERT_CHECK:
 		case ZEND_JMP_NULL:
 		case ZEND_BIND_INIT_STATIC_OR_JMP:
+		case ZEND_JMP_FRAMELESS:
 			/* For our purposes a jumps and branches are side effects. */
 			return 1;
 		case ZEND_BEGIN_SILENCE:
@@ -167,6 +168,10 @@ static inline bool may_have_side_effects(
 		case ZEND_DO_FCALL_BY_NAME:
 		case ZEND_DO_ICALL:
 		case ZEND_DO_UCALL:
+		case ZEND_FRAMELESS_ICALL_0:
+		case ZEND_FRAMELESS_ICALL_1:
+		case ZEND_FRAMELESS_ICALL_2:
+		case ZEND_FRAMELESS_ICALL_3:
 			/* For now assume all calls have side effects */
 			return 1;
 		case ZEND_RECV:

Zend/Optimizer/dfa_pass.c

@@ -653,6 +653,7 @@ static void zend_ssa_replace_control_link(zend_op_array *op_array, zend_ssa *ssa
 			case ZEND_ASSERT_CHECK:
 			case ZEND_JMP_NULL:
 			case ZEND_BIND_INIT_STATIC_OR_JMP:
+			case ZEND_JMP_FRAMELESS:
 				if (ZEND_OP2_JMP_ADDR(opline) == op_array->opcodes + old->start) {
 					ZEND_SET_OP_JMP_ADDR(opline, opline->op2, op_array->opcodes + dst->start);
 				}

Zend/Optimizer/pass1.c

@@ -355,6 +355,7 @@ void zend_optimizer_pass1(zend_op_array *op_array, zend_optimizer_ctx *ctx)
 		case ZEND_JMP_NULL:
 		case ZEND_VERIFY_NEVER_TYPE:
 		case ZEND_BIND_INIT_STATIC_OR_JMP:
+		case ZEND_JMP_FRAMELESS:
 			collect_constants = 0;
 			break;
 		}

Zend/Optimizer/sccp.c

@@ -789,14 +789,10 @@ static bool can_ct_eval_func_call(zend_function *func, zend_string *name, uint32
 /* The functions chosen here are simple to implement and either likely to affect a branch,
  * or just happened to be commonly used with constant operands in WP (need to test other
  * applications as well, of course). */
-static inline zend_result ct_eval_func_call(
-		zend_op_array *op_array, zval *result, zend_string *name, uint32_t num_args, zval **args) {
+static inline zend_result ct_eval_func_call_ex(
+		zend_op_array *op_array, zval *result, zend_function *func, uint32_t num_args, zval **args) {
 	uint32_t i;
-	zend_function *func = zend_hash_find_ptr(CG(function_table), name);
-	if (!func || func->type != ZEND_INTERNAL_FUNCTION) {
-		return FAILURE;
-	}
-
+	zend_string *name = func->common.function_name;
 	if (num_args == 1 && Z_TYPE_P(args[0]) == IS_STRING &&
 			zend_optimizer_eval_special_func_call(result, name, Z_STR_P(args[0])) == SUCCESS) {
 		return SUCCESS;
@@ -855,6 +851,15 @@ static inline zend_result ct_eval_func_call(
 	return retval;
 }
 
+static inline zend_result ct_eval_func_call(
+		zend_op_array *op_array, zval *result, zend_string *name, uint32_t num_args, zval **args) {
+	zend_function *func = zend_hash_find_ptr(CG(function_table), name);
+	if (!func || func->type != ZEND_INTERNAL_FUNCTION) {
+		return FAILURE;
+	}
+	return ct_eval_func_call_ex(op_array, result, func, num_args, args);
+}
+
 #define SET_RESULT(op, zv) do { \
 	if (ssa_op->op##_def >= 0) { \
 		set_value(scdf, ctx, ssa_op->op##_def, zv); \
@@ -1708,6 +1713,51 @@ static void sccp_visit_instr(scdf_ctx *scdf, zend_op *opline, zend_ssa_op *ssa_o
 			SET_RESULT_BOT(result);
 			break;
 		}
+		case ZEND_FRAMELESS_ICALL_0:
+		case ZEND_FRAMELESS_ICALL_1:
+		case ZEND_FRAMELESS_ICALL_2:
+		case ZEND_FRAMELESS_ICALL_3: {
+			/* We already know it can't be evaluated, don't bother checking again */
+			if (ssa_op->result_def < 0 || IS_BOT(&ctx->values[ssa_op->result_def])) {
+				break;
+			}
+
+			zval *args[3] = {NULL};
+			zend_function *func = ZEND_FLF_FUNC(opline);
+			uint32_t num_args = ZEND_FLF_NUM_ARGS(opline->opcode);
+
+			switch (num_args) {
+	 			case 3: {
+					zend_op *op_data = opline + 1;
+					args[2] = get_op1_value(ctx, op_data, &ctx->scdf.ssa->ops[op_data - ctx->scdf.op_array->opcodes]);
+					ZEND_FALLTHROUGH;
+				}
+				case 2:
+					args[1] = get_op2_value(ctx, opline, &ctx->scdf.ssa->ops[opline - ctx->scdf.op_array->opcodes]);
+					ZEND_FALLTHROUGH;
+				case 1:
+					args[0] = get_op1_value(ctx, opline, &ctx->scdf.ssa->ops[opline - ctx->scdf.op_array->opcodes]);
+					break;
+			}
+			for (uint32_t i = 0; i < num_args; i++) {
+				if (!args[i]) {
+					SET_RESULT_BOT(result);
+					return;
+				} else if (IS_BOT(args[i]) || IS_PARTIAL_ARRAY(args[i])) {
+					SET_RESULT_BOT(result);
+					return;
+				} else if (IS_TOP(args[i])) {
+					return;
+				}
+			}
+			if (ct_eval_func_call_ex(scdf->op_array, &zv, func, num_args, args) == SUCCESS) {
+				SET_RESULT(result, &zv);
+				zval_ptr_dtor_nogc(&zv);
+				break;
+			}
+			SET_RESULT_BOT(result);
+			break;
+		}
 		default:
 		{
 			/* If we have no explicit implementation return BOT */
@@ -2155,7 +2205,13 @@ static int try_remove_definition(sccp_ctx *ctx, int var_num, zend_ssa_var *var,
 					if (opline->opcode == ZEND_DO_ICALL) {
 						removed_ops = remove_call(ctx, opline, ssa_op) - 1;
 					} else {
+						bool has_op_data = opline->opcode == ZEND_FRAMELESS_ICALL_3;
 						zend_ssa_remove_instr(ssa, opline, ssa_op);
+						removed_ops++;
+						if (has_op_data) {
+							zend_ssa_remove_instr(ssa, opline + 1, ssa_op + 1);
+							removed_ops++;
+						}
 					}
 					ssa_op->result_def = var_num;
 					opline->opcode = ZEND_QM_ASSIGN;
@@ -2191,8 +2247,13 @@ static int try_remove_definition(sccp_ctx *ctx, int var_num, zend_ssa_var *var,
 				if (opline->opcode == ZEND_DO_ICALL) {
 					removed_ops = remove_call(ctx, opline, ssa_op);
 				} else {
+					bool has_op_data = opline->opcode == ZEND_FRAMELESS_ICALL_3;
 					zend_ssa_remove_instr(ssa, opline, ssa_op);
 					removed_ops++;
+					if (has_op_data) {
+						zend_ssa_remove_instr(ssa, opline + 1, ssa_op + 1);
+						removed_ops++;
+					}
 				}
 			}
 		} else if (ssa_op->op1_def == var_num) {

Zend/Optimizer/zend_call_graph.c

@@ -73,6 +73,7 @@ ZEND_API void zend_analyze_calls(zend_arena **arena, zend_script *script, uint32
 					call_info->num_args = opline->extended_value;
 					call_info->next_callee = func_info->callee_info;
 					call_info->is_prototype = is_prototype;
+					call_info->is_frameless = false;
 					func_info->callee_info = call_info;
 
 					if (build_flags & ZEND_CALL_TREE) {
@@ -102,6 +103,24 @@ ZEND_API void zend_analyze_calls(zend_arena **arena, zend_script *script, uint32
 				call_info = NULL;
 				call++;
 				break;
+			case ZEND_FRAMELESS_ICALL_0:
+			case ZEND_FRAMELESS_ICALL_1:
+			case ZEND_FRAMELESS_ICALL_2:
+			case ZEND_FRAMELESS_ICALL_3: {
+				func = ZEND_FLF_FUNC(opline);
+				zend_call_info *call_info = zend_arena_calloc(arena, 1, sizeof(zend_call_info));
+				call_info->caller_op_array = op_array;
+				call_info->caller_init_opline = opline;
+				call_info->caller_call_opline = NULL;
+				call_info->callee_func = func;
+				call_info->num_args = ZEND_FLF_NUM_ARGS(opline->opcode);
+				call_info->next_callee = func_info->callee_info;
+				call_info->is_prototype = false;
+				call_info->is_frameless = true;
+				call_info->next_caller = NULL;
+				func_info->callee_info = call_info;
+				break;
+			}
 			case ZEND_DO_FCALL:
 			case ZEND_DO_ICALL:
 			case ZEND_DO_UCALL:
@@ -260,9 +279,11 @@ ZEND_API zend_call_info **zend_build_call_map(zend_arena **arena, zend_func_info
 		if (call->caller_call_opline) {
 			map[call->caller_call_opline - op_array->opcodes] = call;
 		}
-		for (i = 0; i < call->num_args; i++) {
-			if (call->arg_info[i].opline) {
-				map[call->arg_info[i].opline - op_array->opcodes] = call;
+		if (!call->is_frameless) {
+			for (i = 0; i < call->num_args; i++) {
+				if (call->arg_info[i].opline) {
+					map[call->arg_info[i].opline - op_array->opcodes] = call;
+				}
 			}
 		}
 	}

Zend/Optimizer/zend_call_graph.h

@@ -38,6 +38,7 @@ struct _zend_call_info {
 	bool               send_unpack;  /* Parameters passed by SEND_UNPACK or SEND_ARRAY */
 	bool               named_args;   /* Function has named arguments */
 	bool               is_prototype; /* An overridden child method may be called */
+	bool               is_frameless; /* A frameless function sends arguments through operands */
 	int                     num_args;	/* Number of arguments, excluding named and variadic arguments */
 	zend_send_arg_info      arg_info[1];
 };

Zend/Optimizer/zend_cfg.c

@@ -370,6 +370,7 @@ ZEND_API void zend_build_cfg(zend_arena **arena, const zend_op_array *op_array,
 			case ZEND_ASSERT_CHECK:
 			case ZEND_JMP_NULL:
 			case ZEND_BIND_INIT_STATIC_OR_JMP:
+			case ZEND_JMP_FRAMELESS:
 				BB_START(OP_JMP_ADDR(opline, opline->op2) - op_array->opcodes);
 				BB_START(i + 1);
 				break;
@@ -524,6 +525,7 @@ ZEND_API void zend_build_cfg(zend_arena **arena, const zend_op_array *op_array,
 			case ZEND_ASSERT_CHECK:
 			case ZEND_JMP_NULL:
 			case ZEND_BIND_INIT_STATIC_OR_JMP:
+			case ZEND_JMP_FRAMELESS:
 				block->successors_count = 2;
 				block->successors[0] = block_map[OP_JMP_ADDR(opline, opline->op2) - op_array->opcodes];
 				block->successors[1] = j + 1;