From 9bc27ad0f7315b8f89635c666973e84568b2f6ab Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 20 Dec 2024 04:45:01 +0100 Subject: [PATCH] Add ABIOverride type for def field (#56555) Together with #54899, this PR is intending to replicate the functionality of #54373, which allowed particular specializations to have a different ABI signature than what would be suggested by the MethodInstance's `specTypes` field. This PR handles that by adding a special `ABIOverwrite` type, which, when placed in the `owner` field of a `CodeInstance` instructs the system to use the given signature instead. --- Compiler/src/Compiler.jl | 2 +- Compiler/src/abstractinterpretation.jl | 18 +++++- Compiler/src/ssair/show.jl | 8 ++- Compiler/src/stmtinfo.jl | 8 +-- Compiler/src/typeinfer.jl | 2 +- Compiler/test/inference.jl | 7 ++- base/arrayshow.jl | 2 +- base/boot.jl | 8 ++- base/show.jl | 8 ++- src/aotcompile.cpp | 16 +++--- src/builtins.c | 16 ++++-- src/clangsa/GCChecker.cpp | 1 + src/codegen.cpp | 77 +++++++++++++++----------- src/debuginfo.cpp | 4 +- src/engine.cpp | 2 +- src/gf.c | 17 +++--- src/interpreter.c | 4 +- src/ircode.c | 2 +- src/jitlayers.cpp | 14 ++--- src/jitlayers.h | 1 + src/jl_exported_data.inc | 1 + src/jltypes.c | 3 +- src/julia.h | 11 +++- src/julia_internal.h | 7 +++ src/precompile_utils.c | 2 +- src/staticdata.c | 16 +++--- src/staticdata_utils.c | 16 +++--- test/precompile.jl | 4 +- 28 files changed, 177 insertions(+), 100 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 06cdbe09e06fc..2f4efda51da9b 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -41,7 +41,7 @@ ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Compiler, using Core.Intrinsics, Core.IR -using Core: Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, +using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, MethodTable, PartialOpaque, SimpleVector, TypeofVararg, _apply_iterate, apply_type, compilerbarrier, donotdelete, memoryref_isassigned, memoryrefget, memoryrefnew, memoryrefoffset, memoryrefset!, print, println, show, svec, diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 59d382d8e4a34..f2d4461a9874a 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2212,6 +2212,18 @@ function abstract_call_unionall(interp::AbstractInterpreter, argtypes::Vector{An return CallMeta(ret, Any, Effects(EFFECTS_TOTAL; nothrow), call.info) end +function ci_abi(ci::CodeInstance) + def = ci.def + isa(def, ABIOverride) && return def.abi + (def::MethodInstance).specTypes +end + +function get_ci_mi(ci::CodeInstance) + def = ci.def + isa(def, ABIOverride) && return def.def + return def::MethodInstance +end + function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState) argtypes = arginfo.argtypes ft′ = argtype_by_index(argtypes, 2) @@ -2223,12 +2235,12 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt if isa(method_or_ci, CodeInstance) our_world = sv.world.this argtype = argtypes_to_type(pushfirst!(argtype_tail(argtypes, 4), ft)) - specsig = method_or_ci.def.specTypes - defdef = method_or_ci.def.def + specsig = ci_abi(method_or_ci) + defdef = get_ci_mi(method_or_ci).def exct = method_or_ci.exctype if !hasintersect(argtype, specsig) return Future(CallMeta(Bottom, TypeError, EFFECTS_THROWS, NoCallInfo())) - elseif !(argtype <: specsig) || (isa(defdef, Method) && !(argtype <: defdef.sig)) + elseif !(argtype <: specsig) || ((!isa(method_or_ci.def, ABIOverride) && isa(defdef, Method)) && !(argtype <: defdef.sig)) exct = Union{exct, TypeError} end callee_valid_range = WorldRange(method_or_ci.min_world, method_or_ci.max_world) diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index 7d7b182655db7..e63d7b5cf640e 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -100,11 +100,17 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co if !(mi isa Core.MethodInstance) mi = (mi::Core.CodeInstance).def end + if isa(mi, Core.ABIOverride) + abi = mi.abi + mi = mi.def + else + abi = mi.specTypes + end show_unquoted(io, stmt.args[2], indent) print(io, "(") # XXX: this is wrong if `sig` is not a concretetype method # more correct would be to use `fieldtype(sig, i)`, but that would obscure / discard Varargs information in show - sig = mi.specTypes == Tuple ? Core.svec() : Base.unwrap_unionall(mi.specTypes).parameters::Core.SimpleVector + sig = abi == Tuple ? Core.svec() : Base.unwrap_unionall(abi).parameters::Core.SimpleVector print_arg(i) = sprint(; context=io) do io show_unquoted(io, stmt.args[i], indent) if (i - 1) <= length(sig) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 4f55f068e9456..a42a9e47b328e 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -71,7 +71,7 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo mi = specialize_method(m) # don't allow `Method`-edge for this optimized format edge = mi else - mi = edge.def + mi = edge.def::MethodInstance end if mi.specTypes === m.spec_types add_one_edge!(edges, edge) @@ -103,7 +103,7 @@ function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) while i <= length(edges) edgeᵢ = edges[i] edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) - edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) + edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge && !(i > 1 && edges[i-1] isa Type) return # found existing covered edge @@ -118,7 +118,7 @@ function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) while i <= length(edges) edgeᵢ_orig = edgeᵢ = edges[i] edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) - edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) + edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge.def && !(i > 1 && edges[i-1] isa Type) if edgeᵢ_orig isa MethodInstance @@ -385,7 +385,7 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) i += 1 end # add_invoke_edge alone - push!(edges, (edge.def.def::Method).sig) + push!(edges, (get_ci_mi(edge).def::Method).sig) push!(edges, edge) nothing end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 20c0a5000bd39..2e9fcf497faf6 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -528,7 +528,7 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) i += 2 continue elseif isa(callee, CodeInstance) - callee = callee.def + callee = get_ci_mi(callee) end ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) i += 2 diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 7cd6989bd2423..9301a2809daf3 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -4110,7 +4110,12 @@ end == [Union{Some{Float64}, Some{Int}, Some{UInt8}}] mi = codeinst else codeinst::Core.CodeInstance - mi = codeinst.def + def = codeinst.def + if isa(def, Core.ABIOverride) + mi = def.def + else + mi = def::Core.MethodInstance + end end return mi end == Core.MethodInstance diff --git a/base/arrayshow.jl b/base/arrayshow.jl index 3bc69e563a967..623111ef0883d 100644 --- a/base/arrayshow.jl +++ b/base/arrayshow.jl @@ -41,7 +41,7 @@ Accept keyword args `c` for alternate single character marker. """ function replace_with_centered_mark(s::AbstractString;c::AbstractChar = '⋅') N = textwidth(ANSIIterator(s)) - return join(setindex!([" " for i=1:N],string(c),ceil(Int,N/2))) + return N == 0 ? string(c) : join(setindex!([" " for i=1:N],string(c),ceil(Int,N/2))) end const undef_ref_alignment = (3,3) diff --git a/base/boot.jl b/base/boot.jl index ed0a722c5e562..4652524530703 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -465,6 +465,12 @@ struct InitError <: WrappedException error end +struct ABIOverride + abi::Type + def::MethodInstance + ABIOverride(@nospecialize(abi::Type), def::MethodInstance) = new(abi, def) +end + struct PrecompilableError <: Exception end String(s::String) = s # no constructor yet @@ -552,7 +558,7 @@ end function CodeInstance( - mi::MethodInstance, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const), + mi::Union{MethodInstance, ABIOverride}, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const), @nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt, effects::UInt32, @nospecialize(analysis_results), relocatability::UInt8, di::Union{DebugInfo,Nothing}, edges::SimpleVector) diff --git a/base/show.jl b/base/show.jl index cb36488b92bc1..d2f43e1ae92cc 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1353,7 +1353,13 @@ end show(io::IO, mi::Core.MethodInstance) = show_mi(io, mi) function show(io::IO, codeinst::Core.CodeInstance) print(io, "CodeInstance for ") - show_mi(io, codeinst.def) + def = codeinst.def + if isa(def, Core.ABIOverride) + show_mi(io, def.def) + print(io, " (ABI Overridden)") + else + show_mi(io, def::MethodInstance) + end end function show_mi(io::IO, mi::Core.MethodInstance, from_stackframe::Bool=false) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 198d7490cb092..b238c44c52676 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -105,7 +105,7 @@ jl_get_llvm_mis_impl(void *native_code, size_t *num_elements, jl_method_instance assert(*num_elements == map.size()); size_t i = 0; for (auto &ci : map) { - data[i++] = ci.first->def; + data[i++] = jl_get_ci_mi(ci.first); } } @@ -455,14 +455,14 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root if ((policy != CompilationPolicy::Default || params.params->trim) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { // XXX: SOURCE_MODE_FORCE_SOURCE is wrong here (neither sufficient nor necessary) - codeinst = jl_type_infer(codeinst->def, jl_atomic_load_relaxed(&codeinst->max_world), SOURCE_MODE_FORCE_SOURCE); + codeinst = jl_type_infer(jl_get_ci_mi(codeinst), jl_atomic_load_relaxed(&codeinst->max_world), SOURCE_MODE_FORCE_SOURCE); } if (codeinst) { orc::ThreadSafeModule result_m = - jl_create_ts_module(name_from_method_instance(codeinst->def), + jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, params.DL, params.TargetTriple); auto decls = jl_emit_codeinst(result_m, codeinst, NULL, params); - record_method_roots(method_roots, codeinst->def); + record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) it = compiled_functions.insert(std::make_pair(codeinst, std::make_pair(std::move(result_m), std::move(decls)))).first; } @@ -501,7 +501,7 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root proto.decl->setLinkage(GlobalVariable::InternalLinkage); //protodecl->setAlwaysInline(); jl_init_function(proto.decl, params.TargetTriple); - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; // TODO: maybe this can be cached in codeinst->specfptr? @@ -641,12 +641,12 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm data->jl_fvar_map[codeinst] = std::make_tuple((uint32_t)-3, (uint32_t)-3); } else { - orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(codeinst->def), + orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, clone.getModuleUnlocked()->getDataLayout(), Triple(clone.getModuleUnlocked()->getTargetTriple())); jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, NULL, params); JL_GC_PROMISE_ROOTED(codeinst->def); // analyzer seems confused - record_method_roots(method_roots, codeinst->def); + record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; else if (jl_options.trim != JL_TRIM_NO) { @@ -2267,7 +2267,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, jl_ // To get correct names in the IR this needs to be at least 2 output.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); JL_GC_PUSH1(&output.temporary_roots); - auto decls = jl_emit_code(m, mi, src, output); + auto decls = jl_emit_code(m, mi, src, NULL, output); output.temporary_roots = nullptr; JL_GC_POP(); // GC the global_targets array contents now since reflection doesn't need it diff --git a/src/builtins.c b/src/builtins.c index 06dbd84aff6c0..c326e53ea968d 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1589,11 +1589,19 @@ JL_CALLABLE(jl_f_invoke) return jl_gf_invoke_by_method(m, args[0], &args[2], nargs - 1); } else if (jl_is_code_instance(argtypes)) { jl_code_instance_t *codeinst = (jl_code_instance_t*)args[1]; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); // N.B.: specTypes need not be a subtype of the method signature. We need to check both. - if (!jl_tuple1_isa(args[0], &args[2], nargs - 1, (jl_datatype_t*)codeinst->def->specTypes) || - (jl_is_method(codeinst->def->def.value) && !jl_tuple1_isa(args[0], &args[2], nargs - 1, (jl_datatype_t*)codeinst->def->def.method->sig))) { - jl_type_error("invoke: argument type error", codeinst->def->specTypes, arg_tuple(args[0], &args[2], nargs - 1)); + if (jl_is_abioverride(codeinst->def)) { + jl_datatype_t *abi = (jl_datatype_t*)((jl_abi_override_t*)(codeinst->def))->abi; + if (!jl_tuple1_isa(args[0], &args[2], nargs - 1, abi)) { + jl_type_error("invoke: argument type error (ABI overwrite)", (jl_value_t*)abi, arg_tuple(args[0], &args[2], nargs - 1)); + } + } else { + if (!jl_tuple1_isa(args[0], &args[2], nargs - 1, (jl_datatype_t*)mi->specTypes) || + (jl_is_method(mi->def.value) && !jl_tuple1_isa(args[0], &args[2], nargs - 1, (jl_datatype_t*)mi->def.method->sig))) { + jl_type_error("invoke: argument type error", mi->specTypes, arg_tuple(args[0], &args[2], nargs - 1)); + } } if (jl_atomic_load_relaxed(&codeinst->min_world) > jl_current_task->world_age || jl_current_task->world_age > jl_atomic_load_relaxed(&codeinst->max_world)) { @@ -1609,7 +1617,7 @@ JL_CALLABLE(jl_f_invoke) if (codeinst->owner != jl_nothing) { jl_error("Failed to invoke or compile external codeinst"); } - return jl_invoke(args[0], &args[2], nargs - 1, codeinst->def); + return jl_invoke(args[0], &args[2], nargs - 1, mi); } } if (!jl_is_tuple_type(jl_unwrap_unionall(argtypes))) diff --git a/src/clangsa/GCChecker.cpp b/src/clangsa/GCChecker.cpp index 40093ca15859b..fdbe5ec9d9e29 100644 --- a/src/clangsa/GCChecker.cpp +++ b/src/clangsa/GCChecker.cpp @@ -847,6 +847,7 @@ bool GCChecker::isGCTrackedType(QualType QT) { Name.ends_with_insensitive("jl_vararg_t") || Name.ends_with_insensitive("jl_opaque_closure_t") || Name.ends_with_insensitive("jl_globalref_t") || + Name.ends_with_insensitive("jl_abi_override_t") || // Probably not technically true for these, but let's allow it as a root Name.ends_with_insensitive("jl_ircode_state") || Name.ends_with_insensitive("typemap_intersection_env") || diff --git a/src/codegen.cpp b/src/codegen.cpp index 86c9afd134d9a..9ae905a8aba32 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3021,9 +3021,8 @@ static bool uses_specsig(jl_value_t *sig, bool needsparams, jl_value_t *rettype, return false; // jlcall sig won't require any box allocations } -static std::pair uses_specsig(jl_method_instance_t *lam, jl_value_t *rettype, bool prefer_specsig) +static std::pair uses_specsig(jl_value_t *abi, jl_method_instance_t *lam, jl_value_t *rettype, bool prefer_specsig) { - jl_value_t *sig = lam->specTypes; bool needsparams = false; if (jl_is_method(lam->def.method)) { if ((size_t)jl_subtype_env_size(lam->def.method->sig) != jl_svec_len(lam->sparam_vals)) @@ -3033,7 +3032,7 @@ static std::pair uses_specsig(jl_method_instance_t *lam, jl_value_t needsparams = true; } } - return std::make_pair(uses_specsig(sig, needsparams, rettype, prefer_specsig), needsparams); + return std::make_pair(uses_specsig(abi, needsparams, rettype, prefer_specsig), needsparams); } @@ -4341,6 +4340,7 @@ static jl_llvm_functions_t orc::ThreadSafeModule &TSM, jl_method_instance_t *lam, jl_code_info_t *src, + jl_value_t *abi, jl_value_t *rettype, jl_codegen_params_t ¶ms); @@ -5487,6 +5487,13 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) return emit_invoke(ctx, lival, argv, nargs, rt, nullptr); } +static jl_value_t *get_ci_abi(jl_code_instance_t *ci) +{ + if (jl_typeof(ci->def) == (jl_value_t*)jl_abioverride_type) + return ((jl_abi_override_t*)ci->def)->abi; + return jl_get_ci_mi(ci)->specTypes; +} + static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok) { ++EmittedInvokes; @@ -5501,7 +5508,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR else { ci = lival.constant; assert(jl_is_code_instance(ci)); - mi = ((jl_code_instance_t*)ci)->def; + mi = jl_get_ci_mi((jl_code_instance_t*)ci); } assert(jl_is_method_instance(mi)); if (mi == ctx.linfo) { @@ -5530,7 +5537,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } else { bool specsig, needsparams; - std::tie(specsig, needsparams) = uses_specsig(mi, codeinst->rettype, ctx.params->prefer_specsig); + std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, ctx.params->prefer_specsig); if (needsparams) { if (trim_may_error(ctx.params->trim)) push_frames(ctx, ctx.linfo, mi); @@ -7256,7 +7263,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M name, M); jl_init_function(f, params.TargetTriple); if (trim_may_error(params.params->trim)) { - push_frames(ctx, ctx.linfo, codeinst->def); + push_frames(ctx, ctx.linfo, jl_get_ci_mi(codeinst)); } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); @@ -7273,7 +7280,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M } else { theFunc = prepare_call(jlinvoke_func); - theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst->def); + theFarg = literal_pointer_val(ctx, (jl_value_t*)jl_get_ci_mi(codeinst)); } theFarg = track_pjlvalue(ctx, theFarg); auto args = f->arg_begin(); @@ -8011,7 +8018,7 @@ const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value // generate a julia-callable function that calls f (AKA lam) // if is_opaque_closure, then generate the OC invoke, rather than a real invoke -static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, jl_returninfo_t &f, unsigned nargs, int retarg, bool is_opaque_closure, StringRef funcName, +static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_value_t *jlretty, jl_returninfo_t &f, unsigned nargs, int retarg, bool is_opaque_closure, StringRef funcName, Module *M, jl_codegen_params_t ¶ms) { ++GeneratedInvokeWrappers; @@ -8047,7 +8054,7 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, j argv[i] = mark_julia_slot(funcArg, oc_type, NULL, ctx.tbaa().tbaa_const); continue; } - jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *ty = jl_nth_slot_type(abi, i); Value *theArg; if (i == 0) { theArg = funcArg; @@ -8061,7 +8068,7 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, j } argv[i] = mark_julia_type(ctx, theArg, true, ty); } - jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, jlretty, f, argv, nargs); + jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, abi, jlretty, f, argv, nargs); if (retarg != -1) { Value *theArg; if (retarg == 0) @@ -8293,13 +8300,13 @@ get_specsig_di(jl_codectx_t &ctx, jl_debugcache_t &debuginfo, jl_value_t *rt, jl } /* aka Core.Compiler.tuple_tfunc */ -static jl_datatype_t *compute_va_type(jl_method_instance_t *lam, size_t nreq) +static jl_datatype_t *compute_va_type(jl_value_t *sig, size_t nreq) { - size_t nvargs = jl_nparams(lam->specTypes)-nreq; + size_t nvargs = jl_nparams(sig)-nreq; jl_svec_t *tupargs = jl_alloc_svec(nvargs); JL_GC_PUSH1(&tupargs); - for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + for (size_t i = nreq; i < jl_nparams(sig); ++i) { + jl_value_t *argType = jl_nth_slot_type(sig, i); // n.b. specTypes is required to be a datatype by construction for specsig if (is_uniquerep_Type(argType)) argType = jl_typeof(jl_tparam0(argType)); @@ -8339,6 +8346,7 @@ static jl_llvm_functions_t orc::ThreadSafeModule &TSM, jl_method_instance_t *lam, jl_code_info_t *src, + jl_value_t *abi, jl_value_t *jlrettype, jl_codegen_params_t ¶ms) { @@ -8417,7 +8425,7 @@ static jl_llvm_functions_t int n_ssavalues = jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_nrows(src->ssavaluetypes); size_t vinfoslen = jl_array_dim0(src->slotflags); ctx.slots.resize(vinfoslen, jl_varinfo_t(ctx.builder.getContext())); - assert(lam->specTypes); // the specTypes field should always be assigned + assert(abi); // the specTypes field should always be assigned // create SAvalue locations for SSAValue objects @@ -8426,7 +8434,7 @@ static jl_llvm_functions_t ctx.ssavalue_usecount.assign(n_ssavalues, 0); bool specsig, needsparams; - std::tie(specsig, needsparams) = uses_specsig(lam, jlrettype, params.params->prefer_specsig); + std::tie(specsig, needsparams) = uses_specsig(abi, lam, jlrettype, params.params->prefer_specsig); // step 3. some variable analysis size_t i; @@ -8436,7 +8444,7 @@ static jl_llvm_functions_t jl_sym_t *argname = slot_symbol(ctx, i); if (argname == jl_unused_sym) continue; - jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *ty = jl_nth_slot_type(abi, i); // TODO: jl_nth_slot_type should call jl_rewrap_unionall // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise // OpaqueClosure implicitly loads the env @@ -8454,7 +8462,7 @@ static jl_llvm_functions_t if (va && ctx.vaSlot != -1) { jl_varinfo_t &varinfo = ctx.slots[ctx.vaSlot]; varinfo.isArgument = true; - vatyp = specsig ? compute_va_type(lam, nreq) : (jl_tuple_type); + vatyp = specsig ? compute_va_type(abi, nreq) : (jl_tuple_type); varinfo.value = mark_julia_type(ctx, (Value*)NULL, false, vatyp); } @@ -8505,7 +8513,7 @@ static jl_llvm_functions_t ArgNames[i] = name; } } - returninfo = get_specsig_function(ctx, M, NULL, declarations.specFunctionObject, lam->specTypes, + returninfo = get_specsig_function(ctx, M, NULL, declarations.specFunctionObject, abi, jlrettype, ctx.is_opaque_closure, JL_FEAT_TEST(ctx,gcstack_arg), ArgNames, nreq); f = cast(returninfo.decl.getCallee()); @@ -8539,8 +8547,8 @@ static jl_llvm_functions_t std::string wrapName; raw_string_ostream(wrapName) << "jfptr_" << ctx.name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); declarations.functionObject = wrapName; - size_t nparams = jl_nparams(lam->specTypes); - gen_invoke_wrapper(lam, jlrettype, returninfo, nparams, retarg, ctx.is_opaque_closure, declarations.functionObject, M, ctx.emission_context); + size_t nparams = jl_nparams(abi); + gen_invoke_wrapper(lam, abi, jlrettype, returninfo, nparams, retarg, ctx.is_opaque_closure, declarations.functionObject, M, ctx.emission_context); // TODO: add attributes: maybe_mark_argument_dereferenceable(Arg, argType) // TODO: add attributes: dereferenceable // TODO: (if needsparams) add attributes: dereferenceable, readonly, nocapture @@ -8562,7 +8570,7 @@ static jl_llvm_functions_t if (!params.getContext().shouldDiscardValueNames() && ctx.emission_context.params->debug_info_level >= 2 && lam->def.method && jl_is_method(lam->def.method) && lam->specTypes != (jl_value_t*)jl_emptytuple_type) { ios_t sigbuf; ios_mem(&sigbuf, 0); - jl_static_show_func_sig((JL_STREAM*) &sigbuf, (jl_value_t*)lam->specTypes); + jl_static_show_func_sig((JL_STREAM*) &sigbuf, (jl_value_t*)abi); f->addFnAttr("julia.fsig", StringRef(sigbuf.buf, sigbuf.size)); ios_close(&sigbuf); } @@ -8619,7 +8627,7 @@ static jl_llvm_functions_t else if (!specsig) subrty = debugcache.jl_di_func_sig; else - subrty = get_specsig_di(ctx, debugcache, jlrettype, lam->specTypes, dbuilder); + subrty = get_specsig_di(ctx, debugcache, jlrettype, abi, dbuilder); SP = dbuilder.createFunction(nullptr ,dbgFuncName // Name ,f->getName() // LinkageName @@ -8951,7 +8959,7 @@ static jl_llvm_functions_t nullptr, nullptr, /*isboxed*/true, AtomicOrdering::NotAtomic, false, sizeof(void*)); } else { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *argType = jl_nth_slot_type(abi, i); // TODO: jl_nth_slot_type should call jl_rewrap_unionall? // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise bool isboxed = deserves_argbox(argType); @@ -9025,10 +9033,10 @@ static jl_llvm_functions_t assert(vi.boxroot == NULL); } else if (specsig) { - ctx.nvargs = jl_nparams(lam->specTypes) - nreq; + ctx.nvargs = jl_nparams(abi) - nreq; SmallVector vargs(ctx.nvargs); - for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + for (size_t i = nreq; i < jl_nparams(abi); ++i) { + jl_value_t *argType = jl_nth_slot_type(abi, i); // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = deserves_argbox(argType); Type *llvmArgType = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, argType); @@ -9954,6 +9962,7 @@ jl_llvm_functions_t jl_emit_code( orc::ThreadSafeModule &m, jl_method_instance_t *li, jl_code_info_t *src, + jl_value_t *abi, jl_codegen_params_t ¶ms) { JL_TIMING(CODEGEN, CODEGEN_LLVM); @@ -9962,8 +9971,10 @@ jl_llvm_functions_t jl_emit_code( assert((params.params == &jl_default_cgparams /* fast path */ || !params.cache || compare_cgparams(params.params, &jl_default_cgparams)) && "functions compiled with custom codegen params must not be cached"); + if (!abi) + abi = li->specTypes; JL_TRY { - decls = emit_function(m, li, src, src->rettype, params); + decls = emit_function(m, li, src, abi, src->rettype, params); auto stream = *jl_ExecutionEngine->get_dump_emitted_mi_name_stream(); if (stream) { jl_printf(stream, "%s\t", decls.specFunctionObject.c_str()); @@ -10030,11 +10041,11 @@ jl_llvm_functions_t jl_emit_codeinst( jl_codegen_params_t ¶ms) { JL_TIMING(CODEGEN, CODEGEN_Codeinst); - jl_timing_show_method_instance(codeinst->def, JL_TIMING_DEFAULT_BLOCK); + jl_timing_show_method_instance(jl_get_ci_mi(codeinst), JL_TIMING_DEFAULT_BLOCK); JL_GC_PUSH1(&src); if (!src) { src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_method_t *def = mi->def.method; // Check if this is the generic method for opaque closure wrappers - // if so, this must compile specptr such that it holds the specptr -> invoke wrapper @@ -10052,7 +10063,7 @@ jl_llvm_functions_t jl_emit_codeinst( } } assert(jl_egal((jl_value_t*)jl_atomic_load_relaxed(&codeinst->debuginfo), (jl_value_t*)src->debuginfo) && "trying to generate code for a codeinst for an incompatible src"); - jl_llvm_functions_t decls = jl_emit_code(m, codeinst->def, src, params); + jl_llvm_functions_t decls = jl_emit_code(m, jl_get_ci_mi(codeinst), src, get_ci_abi(codeinst), params); const std::string &specf = decls.specFunctionObject; const std::string &f = decls.functionObject; @@ -10060,7 +10071,7 @@ jl_llvm_functions_t jl_emit_codeinst( // Prepare debug info to receive this function // record that this function name came from this linfo, // so we can build a reverse mapping for debug-info. - bool toplevel = !jl_is_method(codeinst->def->def.method); + bool toplevel = !jl_is_method(jl_get_ci_mi(codeinst)->def.method); if (!toplevel) { //Safe b/c params holds context lock const DataLayout &DL = m.getModuleUnlocked()->getDataLayout(); @@ -10076,7 +10087,7 @@ jl_llvm_functions_t jl_emit_codeinst( jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); // don't change inferred state if (inferred) { - jl_method_t *def = codeinst->def->def.method; + jl_method_t *def = jl_get_ci_mi(codeinst)->def.method; if (// keep code when keeping everything !(JL_DELETE_NON_INLINEABLE) || // aggressively keep code when debugging level >= 2 diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 31f1ba8281a89..ce8406bf91490 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -167,7 +167,7 @@ void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const D // Non-opaque-closure MethodInstances are considered globally rooted // through their methods, but for OC, we need to create a global root // here. - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure) jl_as_global_root((jl_value_t*)mi, 1); getJITDebugRegistry().add_code_in_flight(name, codeinst, DL); @@ -374,7 +374,7 @@ void JITDebugInfoRegistry::registerJITObject(const object::ObjectFile &Object, jl_method_instance_t *mi = NULL; if (codeinst) { JL_GC_PROMISE_ROOTED(codeinst); - mi = codeinst->def; + mi = jl_get_ci_mi(codeinst); } jl_profile_atomic([&]() JL_NOTSAFEPOINT { if (mi) diff --git a/src/engine.cpp b/src/engine.cpp index 2b68de731c4dd..858f37b55e85e 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -139,7 +139,7 @@ void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) { jl_task_t *ct = jl_current_task; std::unique_lock lock(engine_lock); - auto record = Reservations.find(InferKey{ci->def, ci->owner}); + auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), ci->owner}); if (record == Reservations.end() || record->second.ci != ci) return; assert(jl_atomic_load_relaxed(&ct->tid) == record->second.tid); diff --git a/src/gf.c b/src/gf.c index a5ba309784557..d220ad057176a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -538,7 +538,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( jl_task_t *ct = jl_current_task; jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_gc_alloc(ct->ptls, sizeof(jl_code_instance_t), jl_code_instance_type); - codeinst->def = mi; + codeinst->def = (jl_value_t*)mi; codeinst->owner = owner; jl_atomic_store_relaxed(&codeinst->edges, edges); jl_atomic_store_relaxed(&codeinst->min_world, min_world); @@ -1772,17 +1772,18 @@ static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_wo JL_GC_POP(); } //jl_static_show(JL_STDERR, (jl_value_t*)replaced->def); - if (!jl_is_method(replaced->def->def.method)) + jl_method_instance_t *replaced_mi = jl_get_ci_mi(replaced); + if (!jl_is_method(replaced_mi->def.method)) return; // shouldn't happen, but better to be safe - JL_LOCK(&replaced->def->def.method->writelock); + JL_LOCK(&replaced_mi->def.method->writelock); if (jl_atomic_load_relaxed(&replaced->max_world) == ~(size_t)0) { assert(jl_atomic_load_relaxed(&replaced->min_world) - 1 <= max_world && "attempting to set illogical world constraints (probable race condition)"); jl_atomic_store_release(&replaced->max_world, max_world); } assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); // recurse to all backedges to update their valid range also - _invalidate_backedges(replaced->def, max_world, depth + 1); - JL_UNLOCK(&replaced->def->def.method->writelock); + _invalidate_backedges(replaced_mi, max_world, depth + 1); + JL_UNLOCK(&replaced_mi->def.method->writelock); } static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth) { @@ -2329,7 +2330,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) int replaced_edge; if (invokeTypes) { // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes - if (jl_egal(invokeTypes, caller->def->def.method->sig)) + if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself else replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); @@ -2907,7 +2908,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t jl_callptr_t ucache_invoke = jl_atomic_load_acquire(&ucache->invoke); if (ucache_invoke == NULL) { if ((!jl_is_method(def) || def->source == jl_nothing) && - !jl_cached_uninferred(jl_atomic_load_relaxed(&ucache->def->cache), world)) { + !jl_cached_uninferred(jl_atomic_load_relaxed(&jl_get_ci_mi(ucache)->cache), world)) { jl_throw(jl_new_struct(jl_missingcodeerror_type, (jl_value_t*)mi)); } jl_generate_fptr_for_unspecialized(ucache); @@ -2953,7 +2954,7 @@ jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_co jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { - jl_svec_t *sparams = m->def->sparam_vals; + jl_svec_t *sparams = jl_get_ci_mi(m)->sparam_vals; assert(sparams != jl_emptysvec); jl_fptr_sparam_t invoke = jl_atomic_load_relaxed(&m->specptr.fptr3); assert(invoke && "Forgot to set specptr for jl_fptr_sparam!"); diff --git a/src/interpreter.c b/src/interpreter.c index a465fe27f5084..b9bd1997caa17 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -154,7 +154,7 @@ static jl_value_t *do_invoke(jl_value_t **args, size_t nargs, interpreter_state if (codeinst->owner != jl_nothing) { jl_error("Failed to invoke or compile external codeinst"); } - result = jl_invoke(argv[0], nargs == 2 ? NULL : &argv[1], nargs - 2, codeinst->def); + result = jl_invoke(argv[0], nargs == 2 ? NULL : &argv[1], nargs - 2, jl_get_ci_mi(codeinst)); } } else { result = jl_invoke(argv[0], nargs == 2 ? NULL : &argv[1], nargs - 2, (jl_method_instance_t*)c); @@ -767,7 +767,7 @@ jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *mi, size_t world) jl_value_t *NOINLINE jl_fptr_interpret_call(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *codeinst) { interpreter_state *s; - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_task_t *ct = jl_current_task; size_t world = ct->world_age; jl_code_info_t *src = NULL; diff --git a/src/ircode.c b/src/ircode.c index 9e64e3fe2b574..99c5833ac3be7 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -1194,7 +1194,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t ios_close(s.s); JL_UNLOCK(&m->writelock); // Might GC if (metadata) { - code->parent = metadata->def; + code->parent = jl_get_ci_mi(metadata); jl_gc_wb(code, code->parent); code->rettype = metadata->rettype; jl_gc_wb(code, code->rettype); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index a1b01af716bb4..bdb8298df41a6 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -379,7 +379,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t jl_init_function(proto.decl, params.TargetTriple); // TODO: maybe this can be cached in codeinst->specfptr? int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); @@ -569,7 +569,7 @@ static void jl_compile_codeinst_now(jl_code_instance_t *codeinst) jl_ExecutionEngine->addModule(std::move(TSM)); // may safepoint // If logging of the compilation stream is enabled, // then dump the method-instance specialization type to the stream - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (jl_is_method(mi->def.method)) { auto stream = *jl_ExecutionEngine->get_dump_compiles_stream(); if (stream) { @@ -690,7 +690,7 @@ static void jl_emit_codeinst_to_jit( params.cache = true; params.imaging_mode = imaging_default(); orc::ThreadSafeModule result_m = - jl_create_ts_module(name_from_method_instance(codeinst->def), params.tsctx, params.DL, params.TargetTriple); + jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, params.DL, params.TargetTriple); params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); JL_GC_PUSH1(¶ms.temporary_roots); jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints @@ -698,7 +698,7 @@ static void jl_emit_codeinst_to_jit( JL_GC_POP(); return; } - jl_optimize_roots(params, codeinst->def, *result_m.getModuleUnlocked()); // contains safepoints + jl_optimize_roots(params, jl_get_ci_mi(codeinst), *result_m.getModuleUnlocked()); // contains safepoints params.temporary_roots = nullptr; JL_GC_POP(); { // drop lock before acquiring engine_lock @@ -922,14 +922,14 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec) compiler_start_time = jl_hrtime(); jl_code_info_t *src = NULL; JL_GC_PUSH1(&src); - jl_method_t *def = unspec->def->def.method; + jl_method_t *def = jl_get_ci_mi(unspec)->def.method; if (jl_is_method(def)) { src = (jl_code_info_t*)def->source; if (src && (jl_value_t*)src != jl_nothing) src = jl_uncompress_ir(def, NULL, (jl_value_t*)src); } else { - jl_method_instance_t *mi = unspec->def; + jl_method_instance_t *mi = jl_get_ci_mi(unspec); jl_code_instance_t *uninferred = jl_cached_uninferred( jl_atomic_load_relaxed(&mi->cache), 1); assert(uninferred); @@ -2234,7 +2234,7 @@ StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_callptr_t invoke, jl else { stream_fname << "jlsys_"; } - const char* unadorned_name = jl_symbol_name(codeinst->def->def.method->name); + const char* unadorned_name = jl_symbol_name(jl_get_ci_mi(codeinst)->def.method->name); stream_fname << unadorned_name << "_" << RLST_inc++; *fname = std::move(stream_fname.str()); // store to ReverseLocalSymbolTable addGlobalMapping(*fname, Addr); diff --git a/src/jitlayers.h b/src/jitlayers.h index b6f1e9ec20c6b..342cb516debd7 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -279,6 +279,7 @@ jl_llvm_functions_t jl_emit_code( orc::ThreadSafeModule &M, jl_method_instance_t *mi, jl_code_info_t *src, + jl_value_t *abi, jl_codegen_params_t ¶ms); jl_llvm_functions_t jl_emit_codeinst( diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 8711c14514145..62acce6ce1d65 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -116,6 +116,7 @@ XX(jl_simplevector_type) \ XX(jl_slotnumber_type) \ XX(jl_ssavalue_type) \ + XX(jl_abioverride_type) \ XX(jl_stackovf_exception) \ XX(jl_string_type) \ XX(jl_symbol_type) \ diff --git a/src/jltypes.c b/src/jltypes.c index 52b993cd5aa4d..fbdfe497ea312 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3656,7 +3656,7 @@ void jl_init_types(void) JL_GC_DISABLED "specsigflags", "precompile", "relocatability", "invoke", "specptr"), // function object decls jl_svec(18, - jl_method_instance_type, + jl_any_type, jl_any_type, jl_any_type, jl_ulong_type, @@ -3952,6 +3952,7 @@ void post_boot_hooks(void) jl_weakref_type = (jl_datatype_t*)core("WeakRef"); jl_vecelement_typename = ((jl_datatype_t*)jl_unwrap_unionall(core("VecElement")))->name; jl_nulldebuginfo = (jl_debuginfo_t*)core("NullDebugInfo"); + jl_abioverride_type = (jl_datatype_t*)core("ABIOverride"); jl_init_box_caches(); diff --git a/src/julia.h b/src/julia.h index 049416a33336f..a71f9db030274 100644 --- a/src/julia.h +++ b/src/julia.h @@ -426,7 +426,7 @@ typedef struct _jl_opaque_closure_t { // This type represents an executable operation typedef struct _jl_code_instance_t { JL_DATA_TYPE - jl_method_instance_t *def; // method this is specialized from + jl_value_t *def; // MethodInstance or ABIOverride jl_value_t *owner; // Compiler token this belongs to, `jl_nothing` is reserved for native _Atomic(struct _jl_code_instance_t*) next; // pointer to the next cache entry @@ -479,6 +479,13 @@ typedef struct _jl_code_instance_t { } specptr; // private data for `jlcall entry point } jl_code_instance_t; +// May be used as the ->def field of a CodeInstance to override the ABI +typedef struct _jl_abi_override_t { + JL_DATA_TYPE + jl_value_t *abi; + jl_method_instance_t *def; +} jl_abi_override_t; + // all values are callable as Functions typedef jl_value_t jl_function_t; @@ -935,6 +942,7 @@ extern JL_DLLIMPORT jl_datatype_t *jl_fielderror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_atomicerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_missingcodeerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_lineinfonode_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_abioverride_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_stackovf_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memory_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_readonlymemory_exception JL_GLOBALLY_ROOTED; @@ -1570,6 +1578,7 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_llvmpointer(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_llvmpointer_typename) #define jl_is_intrinsic(v) jl_typetagis(v,jl_intrinsic_type) #define jl_is_addrspacecore(v) jl_typetagis(v,jl_addrspacecore_type) +#define jl_is_abioverride(v) jl_typetagis(v,jl_abioverride_type) #define jl_genericmemory_isbitsunion(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_isunion) JL_DLLEXPORT int jl_subtype(jl_value_t *a, jl_value_t *b); diff --git a/src/julia_internal.h b/src/julia_internal.h index e0e8158f8e0d9..e452ff315d527 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -693,6 +693,13 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( uint32_t effects, jl_value_t *analysis_results, uint8_t relocatability, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/); +STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + if (jl_is_abioverride(ci->def)) + return ((jl_abi_override_t*)ci->def)->def; + return (jl_method_instance_t*)ci->def; +} + JL_DLLEXPORT const char *jl_debuginfo_file(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; JL_DLLEXPORT const char *jl_debuginfo_file1(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_module_t *jl_debuginfo_module1(jl_value_t *debuginfo_def) JL_NOTSAFEPOINT; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 4717351d499b0..f2777455c4ed1 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -336,7 +336,7 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met n = jl_array_nrows(new_ext_cis); for (i = 0; i < n; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_ext_cis, i); - precompile_enq_specialization_(ci->def, m); + precompile_enq_specialization_(jl_get_ci_mi(ci), m); } } } diff --git a/src/staticdata.c b/src/staticdata.c index a0d56f3e38f54..ca335bd0fefee 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -101,7 +101,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 194 +#define NUM_TAGS 195 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -216,6 +216,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_addrspace_typename); INSERT_TAG(jl_addrspacecore_type); INSERT_TAG(jl_debuginfo_type); + INSERT_TAG(jl_abioverride_type); // special typenames INSERT_TAG(jl_tuple_typename); @@ -909,13 +910,14 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } if (s->incremental && jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; + jl_method_instance_t *mi = jl_get_ci_mi(ci); // make sure we don't serialize other reachable cache entries of foreign methods // Should this now be: // if (ci !in ci->defs->cache) // record_field_change((jl_value_t**)&ci->next, NULL); // Why are we checking that the method/module this originates from is in_image? // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)ci->def->def.value)) { + if (jl_object_in_image((jl_value_t*)mi->def.value)) { // TODO: if (ci in ci->defs->cache) record_field_change((jl_value_t**)&ci->next, NULL); } @@ -923,8 +925,8 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) if (!is_relocatable_ci(&relocatable_ext_cis, ci)) record_field_change((jl_value_t**)&ci->inferred, jl_nothing); - else if (jl_is_method(ci->def->def.method) && // don't delete toplevel code - ci->def->def.method->source) { // don't delete code from optimized opaque closures that can't be reconstructed (and builtins) + else if (jl_is_method(mi->def.method) && // don't delete toplevel code + mi->def.method->source) { // don't delete code from optimized opaque closures that can't be reconstructed (and builtins) if (jl_atomic_load_relaxed(&ci->max_world) != ~(size_t)0 || // delete all code that cannot run jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant record_field_change((jl_value_t**)&ci->inferred, jl_nothing); @@ -1809,7 +1811,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED fptr_id = JL_API_CONST; } else { - if (jl_is_method(ci->def->def.method)) { + if (jl_is_method(jl_get_ci_mi(ci)->def.method)) { builtin_id = jl_fptr_id(jl_atomic_load_relaxed(&ci->specptr.fptr)); if (builtin_id) { // found in the table of builtins assert(builtin_id >= 2); @@ -2352,9 +2354,9 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) offset = ~offset; } jl_code_instance_t *codeinst = (jl_code_instance_t*)(base + offset); - assert(jl_is_method(codeinst->def->def.method) && jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return); + assert(jl_is_method(jl_get_ci_mi(codeinst)->def.method) && jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return); assert(specfunc ? jl_atomic_load_relaxed(&codeinst->invoke) != NULL : jl_atomic_load_relaxed(&codeinst->invoke) == NULL); - linfos[i] = codeinst->def; // now it's a MethodInstance + linfos[i] = jl_get_ci_mi(codeinst); // now it's a MethodInstance void *fptr = fvars.ptrs[i]; for (; clone_idx < fvars.nclones; clone_idx++) { uint32_t idx = fvars.clone_idxs[clone_idx] & jl_sysimg_val_mask; diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index ea0b7216155bd..5ff7a7b1b1fc0 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -188,7 +188,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, jl_code_instance_t *be; i = get_next_edge(mi->backedges, i, NULL, &be); JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here - int child_found = has_backedge_to_worklist(be->def, visited, stack); + int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack); if (child_found == 1 || child_found == 2) { // found what we were looking for, so terminate early found = 1; @@ -219,7 +219,7 @@ static int is_relocatable_ci(htable_t *relocatable_ext_cis, jl_code_instance_t * { if (!ci->relocatability) return 0; - jl_method_instance_t *mi = ci->def; + jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (!ptrhash_has(relocatable_ext_cis, ci) && jl_object_in_image((jl_value_t*)m) && (!jl_is_method(m) || jl_object_in_image((jl_value_t*)m->module))) return 0; @@ -249,7 +249,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) assert(jl_is_code_instance(ci)); if (!ci->relocatability) continue; - jl_method_instance_t *mi = ci->def; + jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { int found = has_backedge_to_worklist(mi, &visited, &stack); @@ -283,7 +283,7 @@ static void jl_collect_new_roots(htable_t *relocatable_ext_cis, jl_array_t *root for (size_t i = 0; i < l; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_ext_cis, i); assert(jl_is_code_instance(ci)); - jl_method_t *m = ci->def->def.method; + jl_method_t *m = jl_get_ci_mi(ci)->def.method; assert(jl_is_method(m)); ptrhash_put(&mset, (void*)m, (void*)m); ptrhash_put(relocatable_ext_cis, (void*)ci, (void*)ci); @@ -864,7 +864,7 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t *minworld, size *minworld = 1; size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); *maxworld = current_world; - assert(jl_is_method_instance(codeinst->def) && jl_is_method(codeinst->def->def.method)); + assert(jl_is_method_instance(jl_get_ci_mi(codeinst)) && jl_is_method(jl_get_ci_mi(codeinst)->def.method)); void **bp = ptrhash_bp(visiting, codeinst); if (*bp != HT_NOTFOUND) return (char*)*bp - (char*)HT_NOTFOUND; // cycle idx @@ -893,7 +893,7 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t *minworld, size size_t max_valid2; assert(!jl_is_method(edge)); // `Method`-edge isn't allowed for the optimized one-edge format if (jl_is_code_instance(edge)) - edge = (jl_value_t*)((jl_code_instance_t*)edge)->def; + edge = (jl_value_t*)jl_get_ci_mi((jl_code_instance_t*)edge); if (jl_is_method_instance(edge)) { jl_method_instance_t *mi = (jl_method_instance_t*)edge; sig = jl_type_intersection(mi->def.method->sig, (jl_value_t*)mi->specTypes); // TODO: ?? @@ -917,7 +917,7 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t *minworld, size continue; } if (jl_is_code_instance(callee)) - callee = ((jl_code_instance_t*)callee)->def; + callee = jl_get_ci_mi((jl_code_instance_t*)callee); if (jl_is_method_instance(callee)) { meth = callee->def.method; } @@ -1048,7 +1048,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list) for (size_t i = 0; i < nedges; i++) { jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_array_ptr_ref(edges, i); jl_svec_t *callees = jl_atomic_load_relaxed(&codeinst->edges); - jl_method_instance_t *caller = codeinst->def; + jl_method_instance_t *caller = jl_get_ci_mi(codeinst); jl_verify_method_graph(codeinst, &stack, &visiting); size_t minvalid = jl_atomic_load_relaxed(&codeinst->min_world); size_t maxvalid = jl_atomic_load_relaxed(&codeinst->max_world); diff --git a/test/precompile.jl b/test/precompile.jl index 1607d4c6b502b..d9bbf524609c9 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -915,7 +915,7 @@ precompile_test_harness("code caching") do dir # external callers mods = Module[] for be in mi.backedges - push!(mods, (be.def.def::Method).module) # XXX + push!(mods, ((be.def::Core.MethodInstance).def::Method).module) # XXX end @test MA ∈ mods @test MB ∈ mods @@ -1837,7 +1837,7 @@ precompile_test_harness("PkgCacheInspector") do load_path m = only(external_methods).func::Method @test m.name == :repl_cmd && m.nargs < 2 @test new_ext_cis === nothing || any(new_ext_cis) do ci - mi = ci.def + mi = ci.def::Core.MethodInstance mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} end end