diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index b6805a77124ca..9ba268fe95be8 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1027,3 +1027,12 @@ for a in ((@noinline Ref{Int}(2)), @test ex === a end end + +# Make sure that code that has unbound sparams works +#https://github.com/JuliaLang/julia/issues/56739 + +f56739(a) where {T} = a + +@test f56739(1) == 1 +g56739(x) = @noinline f56739(x) +@test g56739(1) == 1 diff --git a/base/meta.jl b/base/meta.jl index bcf4fbf632ab2..0078e15bcd98b 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -363,10 +363,15 @@ function _partially_inline!(@nospecialize(x), slot_replacements::Vector{Any}, return x end if isa(x, Core.ReturnNode) + # Unreachable doesn't have val defined + if !isdefined(x, :val) + return x + else return Core.ReturnNode( _partially_inline!(x.val, slot_replacements, type_signature, static_param_values, slot_offset, statement_offset, boundscheck), ) + end end if isa(x, Core.GotoIfNot) return Core.GotoIfNot( diff --git a/base/strings/io.jl b/base/strings/io.jl index 116bcf71eeb7a..b4a3c7ad3e0c2 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -250,7 +250,9 @@ print(io::IO, s::Union{String,SubString{String}}) = (write(io, s); nothing) """ repr(x; context=nothing) -Create a string from any value using the 2-argument `show(io, x)` function. +Create a string representation of any value using the 2-argument `show(io, x)` function, +which aims to produce a string that is parseable Julia code, where possible. +i.e. `eval(Meta.parse(repr(x))) == x` should hold true. You should not add methods to `repr`; define a [`show`](@ref) method instead. The optional keyword argument `context` can be set to a `:key=>value` pair, a diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index ffbbfd620997f..b075223d9c7e4 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -136,60 +136,56 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter()) m = match.method delete!(push!(Set{Method}(), m), m) copy(Core.Compiler.retrieve_code_info(Core.Compiler.specialize_method(match), typemax(UInt))) - - empty!(Set()) - push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) - (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] - (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] - (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] - (setindex!(Dict{Union{GlobalRef,Symbol}, Vector{Int}}(), [1], :two))[:two] - (setindex!(IdDict{Type, Union{Missing, Vector{Tuple{LineNumberNode, Expr}}}}(), missing, Int))[Int] - Dict{Symbol, Union{Nothing, Bool, Symbol}}(:one => false)[:one] - Dict(Base => [:(1+1)])[Base] - Dict(:one => [1])[:one] - Dict("abc" => Set())["abc"] - pushfirst!([], sum) - get(Base.pkgorigins, Base.PkgId(Base), nothing) - sort!([1,2,3]) - unique!([1,2,3]) - cumsum([1,2,3]) - append!(Int[], BitSet()) - isempty(BitSet()) - delete!(BitSet([1,2]), 3) - deleteat!(Int32[1,2,3], [1,3]) - deleteat!(Any[1,2,3], [1,3]) - Core.svec(1, 2) == Core.svec(3, 4) - any(t->t[1].line > 1, [(LineNumberNode(2,:none), :(1+1))]) - - # Code loading uses this - sortperm(mtime.(readdir(".")), rev=true) - # JLLWrappers uses these - Dict{Base.UUID,Set{String}}()[Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}() - get!(Set{String}, Dict{Base.UUID,Set{String}}(), Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")) - eachindex(IndexLinear(), Expr[]) - push!(Expr[], Expr(:return, false)) - vcat(String[], String[]) - k, v = (:hello => nothing) - Base.print_time_imports_report(Base) - Base.print_time_imports_report_init(Base) - - # Preferences uses these - get(Dict{String,Any}(), "missing", nothing) - delete!(Dict{String,Any}(), "missing") - for (k, v) in Dict{String,Any}() - println(k) - end - - # interactive startup uses this - write(IOBuffer(), "") - - # Not critical, but helps hide unrelated compilation from @time when using --trace-compile. - f55729() = Base.Experimental.@force_compile - @time @eval f55729() - @time @eval f55729() - break # only actually need to do this once end +empty!(Set()) +push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) +(setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] +(setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] +(setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] +(setindex!(Dict{Union{GlobalRef,Symbol}, Vector{Int}}(), [1], :two))[:two] +(setindex!(IdDict{Type, Union{Missing, Vector{Tuple{LineNumberNode, Expr}}}}(), missing, Int))[Int] +Dict{Symbol, Union{Nothing, Bool, Symbol}}(:one => false)[:one] +Dict(Base => [:(1+1)])[Base] +Dict(:one => [1])[:one] +Dict("abc" => Set())["abc"] +pushfirst!([], sum) +get(Base.pkgorigins, Base.PkgId(Base), nothing) +sort!([1,2,3]) +unique!([1,2,3]) +cumsum([1,2,3]) +append!(Int[], BitSet()) +isempty(BitSet()) +delete!(BitSet([1,2]), 3) +deleteat!(Int32[1,2,3], [1,3]) +deleteat!(Any[1,2,3], [1,3]) +Core.svec(1, 2) == Core.svec(3, 4) +any(t->t[1].line > 1, [(LineNumberNode(2,:none), :(1+1))]) + +# Code loading uses this +sortperm(mtime.(readdir(".")), rev=true) +# JLLWrappers uses these +Dict{Base.UUID,Set{String}}()[Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}() +get!(Set{String}, Dict{Base.UUID,Set{String}}(), Base.UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")) +eachindex(IndexLinear(), Expr[]) +push!(Expr[], Expr(:return, false)) +vcat(String[], String[]) +k, v = (:hello => nothing) +Base.print_time_imports_report(Base) +Base.print_time_imports_report_init(Base) + +# Preferences uses these +get(Dict{String,Any}(), "missing", nothing) +delete!(Dict{String,Any}(), "missing") +for (k, v) in Dict{String,Any}() + println(k) +end + +# interactive startup uses this +write(IOBuffer(), "") + +# precompile @time report generation and printing +@time @eval Base.Experimental.@force_compile """ julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index ba9a3c05e41ff..198d7490cb092 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -505,7 +505,7 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; // TODO: maybe this can be cached in codeinst->specfptr? - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke, 0, 0); + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); preal_decl = ""; // no need to fixup the name } if (!preal_decl.empty()) { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 87ba51457dc07..6aabc459d2c91 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2284,7 +2284,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, const jl_cgval_t argv[3] = { cmp, lhs, rhs }; jl_cgval_t ret; if (modifyop) { - ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); } else { if (trim_may_error(ctx.params->trim)) { @@ -4023,7 +4023,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, emit_lockstate_value(ctx, needlock, false); const jl_cgval_t argv[3] = { cmp, oldval, rhs }; if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, nullptr); } else { if (trim_may_error(ctx.params->trim)) { diff --git a/src/codegen.cpp b/src/codegen.cpp index cd404360d8bfe..86c9afd134d9a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2104,7 +2104,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -2171,18 +2171,18 @@ static jl_value_t *StackFrame( return frame; } -static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee, int no_debug=false) +static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee) { CallFrames frames; auto it = ctx.emission_context.enqueuers.find(callee); if (it != ctx.emission_context.enqueuers.end()) return; - if (no_debug) { // Used in tojlinvoke + auto DL = ctx.builder.getCurrentDebugLocation(); + if (caller == nullptr || !DL) { // Used in various places frames.push_back({ctx.funcName, "", 0}); ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); return; } - auto DL = ctx.builder.getCurrentDebugLocation(); auto filename = std::string(DL->getFilename()); auto line = DL->getLine(); auto fname = std::string(DL->getScope()->getSubprogram()->getName()); @@ -5249,14 +5249,12 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, trampoline); } -static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs) +static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs) { ++EmittedSpecfunCalls; // emit specialized call site bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - FunctionType *cft = returninfo.decl.getFunctionType(); - size_t nfargs = cft->getNumParams(); + size_t nfargs = returninfo.decl.getFunctionType()->getNumParams(); SmallVector argvals(nfargs); unsigned idx = 0; AllocaInst *result = nullptr; @@ -5348,23 +5346,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos idx++; } assert(idx == nfargs); - Value *TheCallee = returninfo.decl.getCallee(); - if (fromexternal) { - std::string namep("p"); - namep += cast(returninfo.decl.getCallee())->getName(); - GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); - if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(TheCallee->getType()), - namep); - ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; - } - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - TheCallee = ai.decorateInst(ctx.builder.CreateAlignedLoad(TheCallee->getType(), GV, Align(sizeof(void*)))); - setName(ctx.emission_context, TheCallee, namep); - } - CallInst *call = ctx.builder.CreateCall(cft, TheCallee, argvals); + CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals); call->setAttributes(returninfo.attrs); if (gcstack_arg && ctx.emission_context.use_swiftcc) call->setCallingConv(CallingConv::Swift); @@ -5406,7 +5388,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, llvm::Value *callee, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *nreturn_roots, jl_value_t *inferred_retty, Value *age_ok) { ++EmittedSpecfunCalls; // emit specialized call site @@ -5414,21 +5396,52 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos jl_returninfo_t returninfo = get_specsig_function(ctx, jl_Module, callee, specFunctionObject, specTypes, jlretty, is_opaque_closure, gcstack_arg); *cc = returninfo.cc; *nreturn_roots = returninfo.return_roots; - jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, specTypes, jlretty, returninfo, fromexternal, argv, nargs); + if (fromexternal) { + std::string namep("p"); + Value *TheCallee = returninfo.decl.getCallee(); + namep += cast(TheCallee)->getName(); + GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); + if (GV == nullptr) { + GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, + GlobalVariable::ExternalLinkage, + Constant::getNullValue(TheCallee->getType()), + namep); + ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; + } + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + TheCallee = ai.decorateInst(ctx.builder.CreateAlignedLoad(TheCallee->getType(), GV, Align(sizeof(void*)))); + setName(ctx.emission_context, TheCallee, namep); + returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), TheCallee); + } + if (age_ok) { + std::string funcName(specFunctionObject); + funcName += "_gfthunk"; + Function *gf_thunk = Function::Create(returninfo.decl.getFunctionType(), + GlobalVariable::InternalLinkage, funcName, jl_Module); + jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); + gf_thunk->setAttributes(AttributeList::get(gf_thunk->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); + // build a specsig -> jl_apply_generic converter thunk + // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), + // but which has the signature of a specsig + emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, specTypes, jlretty, is_opaque_closure, nargs, ctx.emission_context, + prepare_call(jlapplygeneric_func)); + returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), ctx.builder.CreateSelect(age_ok, returninfo.decl.getCallee(), gf_thunk)); + } + jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, specTypes, jlretty, returninfo, argv, nargs); // see if inference has a different / better type for the call than the lambda return update_julia_type(ctx, retval, inferred_retty); } static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_t *mi, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_returninfo_t::CallingConv *cc, unsigned *return_roots, jl_value_t *inferred_retty, Value *age_ok) { bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; return emit_call_specfun_other(ctx, is_opaque_closure, mi->specTypes, jlretty, NULL, - specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty); + specFunctionObject, fromexternal, argv, nargs, cc, return_roots, inferred_retty, age_ok); } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, jl_value_t *inferred_retty, Value *age_ok) { Value *theFptr; if (fromexternal) { @@ -5451,6 +5464,8 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty theFptr = jl_Module->getOrInsertFunction(specFunctionObject, ctx.types().T_jlfunc).getCallee(); addRetAttr(cast(theFptr), Attribute::NonNull); } + if (age_ok) + theFptr = ctx.builder.CreateSelect(age_ok, theFptr, prepare_call(jlapplygeneric_func)); Value *ret = emit_jlcall(ctx, FunctionCallee(ctx.types().T_jlfunc, theFptr), nullptr, argv, nargs, julia_call); return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } @@ -5469,10 +5484,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv, nargs, rt); + return emit_invoke(ctx, lival, argv, nargs, rt, nullptr); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, Value *age_ok) { ++EmittedInvokes; bool handled = false; @@ -5495,12 +5510,12 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR FunctionType *ft = ctx.f->getFunctionType(); StringRef protoname = ctx.f->getName(); if (ft == ctx.types().T_jlfunc) { - result = emit_call_specfun_boxed(ctx, ctx.rettype, protoname, nullptr, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, ctx.rettype, protoname, nullptr, argv, nargs, rt, age_ok); handled = true; } else if (ft != ctx.types().T_jlfuncparams) { unsigned return_roots = 0; - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); handled = true; } } @@ -5513,62 +5528,70 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR result = mark_julia_const(ctx, codeinst->rettype_const); handled = true; } - else if (invoke != jl_fptr_sparam_addr) { + else { bool specsig, needsparams; std::tie(specsig, needsparams) = uses_specsig(mi, codeinst->rettype, ctx.params->prefer_specsig); - std::string name; - StringRef protoname; - bool need_to_emit = true; - bool cache_valid = ctx.use_cache || ctx.external_linkage; - bool external = false; - - // Check if we already queued this up - auto it = ctx.call_targets.find(codeinst); - if (need_to_emit && it != ctx.call_targets.end()) { - assert(it->second.specsig == specsig); - protoname = it->second.decl->getName(); - need_to_emit = cache_valid = false; - } + if (needsparams) { + if (trim_may_error(ctx.params->trim)) + push_frames(ctx, ctx.linfo, mi); + Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, julia_call2); + result = mark_julia_type(ctx, r, true, rt); + handled = true; + } else { + std::string name; + StringRef protoname; + bool need_to_emit = true; + bool cache_valid = ctx.use_cache || ctx.external_linkage; + bool external = false; + + // Check if we already queued this up + auto it = ctx.call_targets.find(codeinst); + if (need_to_emit && it != ctx.call_targets.end()) { + assert(it->second.specsig == specsig); + protoname = it->second.decl->getName(); + need_to_emit = cache_valid = false; + } - // Check if it is already compiled (either JIT or externally) - if (need_to_emit && cache_valid) { - // optimization: emit the correct name immediately, if we know it - // TODO: use `emitted` map here too to try to consolidate names? - uint8_t specsigflags; - jl_callptr_t invoke; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) { - protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - if (ctx.external_linkage) { - // TODO: Add !specsig support to aotcompile.cpp - // Check that the codeinst is containing native code - if (specsig && (specsigflags & 0b100)) { - external = true; + // Check if it is already compiled (either JIT or externally) + if (need_to_emit && cache_valid) { + // optimization: emit the correct name immediately, if we know it + // TODO: use `emitted` map here too to try to consolidate names? + uint8_t specsigflags; + jl_callptr_t invoke; + void *fptr; + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); + if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) { + protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + if (ctx.external_linkage) { + // TODO: Add !specsig support to aotcompile.cpp + // Check that the codeinst is containing native code + if (specsig && (specsigflags & 0b100)) { + external = true; + need_to_emit = false; + } + } + else { // ctx.use_cache need_to_emit = false; } } - else { // ctx.use_cache - need_to_emit = false; - } } - } - if (need_to_emit) { - raw_string_ostream(name) << (specsig ? "j_" : "j1_") << name_from_method_instance(mi) << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); - protoname = StringRef(name); - } - jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; - unsigned return_roots = 0; - if (specsig) - result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt); - else - result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); - handled = true; - if (need_to_emit) { - Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); - ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; - if (trim_may_error(ctx.params->trim)) - push_frames(ctx, ctx.linfo, mi); + if (need_to_emit) { + raw_string_ostream(name) << (specsig ? "j_" : "j1_") << name_from_method_instance(mi) << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + protoname = StringRef(name); + } + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; + unsigned return_roots = 0; + if (specsig) + result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt, age_ok); + else + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt, age_ok); + handled = true; + if (need_to_emit) { + Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); + ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; + if (trim_may_error(ctx.params->trim)) + push_frames(ctx, ctx.linfo, mi); + } } } } @@ -5586,8 +5609,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR print_stacktrace(ctx, ctx.params->trim); } } - Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); - result = mark_julia_type(ctx, r, true, rt); + Value *r = age_ok ? emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call) : emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); + result = mark_julia_type(ctx, r, true, age_ok ? (jl_value_t*)jl_any_type : rt); } if (result.typ == jl_bottom_type) { #ifndef JL_NDEBUG @@ -5680,7 +5703,7 @@ static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, j Value *specptr = emit_unbox(ctx, ctx.types().T_size, closure_specptr, (jl_value_t*)jl_long_type); JL_GC_PUSH1(&sigtype); jl_cgval_t r = emit_call_specfun_other(ctx, true, sigtype, oc_rett, specptr, "", NULL, argv, nargs, - &cc, &return_roots, oc_rett); + &cc, &return_roots, oc_rett, nullptr); JL_GC_POP(); return r; } @@ -7233,8 +7256,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M name, M); jl_init_function(f, params.TargetTriple); if (trim_may_error(params.params->trim)) { - // TODO: Debuginfo! - push_frames(ctx, ctx.linfo, codeinst->def, 1); + push_frames(ctx, ctx.linfo, codeinst->def); } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); @@ -7276,11 +7298,10 @@ void emit_specsig_to_fptr1( jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, size_t nargs, jl_codegen_params_t ¶ms, - Function *target, - size_t min_world, size_t max_world) + Function *target) { ++EmittedCFuncInvalidates; - jl_codectx_t ctx(gf_thunk->getParent()->getContext(), params, min_world, max_world); + jl_codectx_t ctx(gf_thunk->getParent()->getContext(), params, 0, 0); ctx.f = gf_thunk; BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", gf_thunk); @@ -7397,12 +7418,11 @@ void emit_specsig_to_fptr1( } } -static Function* gen_cfun_wrapper( +static Function *gen_cfun_wrapper( Module *into, jl_codegen_params_t ¶ms, const function_sig_t &sig, jl_value_t *ff, const char *aliasname, jl_value_t *declrt, jl_method_instance_t *lam, - jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types, - size_t min_world, size_t max_world) + jl_unionall_t *unionall_env, jl_svec_t *sparam_vals, jl_array_t **closure_types) { ++GeneratedCFuncWrappers; // Generate a c-callable wrapper @@ -7410,36 +7430,17 @@ static Function* gen_cfun_wrapper( size_t nargs = sig.nccallargs; const char *name = "cfunction"; size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_code_instance_t *codeinst = NULL; bool nest = (!ff || unionall_env); jl_value_t *astrt = (jl_value_t*)jl_any_type; - void *callptr = NULL; - jl_callptr_t invoke = NULL; - int calltype = 0; if (aliasname) name = aliasname; else if (lam) name = jl_symbol_name(lam->def.method->name); - if (lam && params.cache) { + + jl_code_instance_t *codeinst = NULL; + if (lam) { // TODO: this isn't ideal to be unconditionally calling type inference (and compile) from here - codeinst = jl_compile_method_internal(lam, world); - uint8_t specsigflags; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - assert(invoke); - if (invoke == jl_fptr_args_addr) { - callptr = fptr; - calltype = 1; - } - else if (invoke == jl_fptr_const_return_addr) { - // don't need the fptr - callptr = (void*)codeinst->rettype_const; - calltype = 2; - } - else if (specsigflags & 0b1) { - callptr = fptr; - calltype = 3; - } + codeinst = jl_type_infer(lam, world, SOURCE_MODE_NOT_REQUIRED); astrt = codeinst->rettype; if (astrt != (jl_value_t*)jl_bottom_type && jl_type_intersection(astrt, declrt) == jl_bottom_type) { @@ -7515,7 +7516,7 @@ static Function* gen_cfun_wrapper( jl_init_function(cw, params.TargetTriple); cw->setAttributes(AttributeList::get(M->getContext(), {attributes, cw->getAttributes()})); - jl_codectx_t ctx(M->getContext(), params, min_world, max_world); + jl_codectx_t ctx(M->getContext(), params, 0, 0); ctx.f = cw; ctx.name = name; ctx.funcName = name; @@ -7534,8 +7535,8 @@ static Function* gen_cfun_wrapper( prepare_global_in(jl_Module, jlgetworld_global), ctx.types().alignof_ptr); cast(world_v)->setOrdering(AtomicOrdering::Acquire); - Value *age_ok = NULL; - if (calltype) { + Value *age_ok = nullptr; + if (codeinst) { LoadInst *lam_max = ctx.builder.CreateAlignedLoad( ctx.types().T_size, emit_ptrgep(ctx, literal_pointer_val(ctx, (jl_value_t*)codeinst), offsetof(jl_code_instance_t, max_world)), @@ -7699,79 +7700,9 @@ static Function* gen_cfun_wrapper( // Create the call bool jlfunc_sret; jl_cgval_t retval; - if (calltype == 2) { - nargs = 0; // arguments not needed -- TODO: not really true, should emit an age_ok test and jlcall - (void)nargs; // silence unused variable warning - jlfunc_sret = false; - retval = mark_julia_const(ctx, (jl_value_t*)callptr); - } - else if (calltype == 0 || calltype == 1) { - // emit a jlcall - jlfunc_sret = false; - Function *theFptr = NULL; - if (calltype == 1) { - StringRef fname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, invoke, codeinst); - theFptr = cast_or_null(jl_Module->getNamedValue(fname)); - if (!theFptr) { - theFptr = Function::Create(ctx.types().T_jlfunc, GlobalVariable::ExternalLinkage, - fname, jl_Module); - jl_init_function(theFptr, ctx.emission_context.TargetTriple); - jl_name_jlfunc_args(ctx.emission_context, theFptr); - addRetAttr(theFptr, Attribute::NonNull); - } - else { - assert(theFptr->getFunctionType() == ctx.types().T_jlfunc); - } - } - BasicBlock *b_generic, *b_jlcall, *b_after; - Value *ret_jlcall; - if (age_ok) { - assert(theFptr); - b_generic = BasicBlock::Create(ctx.builder.getContext(), "generic", cw); - b_jlcall = BasicBlock::Create(ctx.builder.getContext(), "apply", cw); - b_after = BasicBlock::Create(ctx.builder.getContext(), "after", cw); - ctx.builder.CreateCondBr(age_ok, b_jlcall, b_generic); - ctx.builder.SetInsertPoint(b_jlcall); - // for jlcall, we need to pass the function object even if it is a ghost. - Value *theF = boxed(ctx, inputargs[0]); - assert(theF); - ret_jlcall = emit_jlcall(ctx, theFptr, theF, ArrayRef(inputargs).drop_front(), nargs, julia_call); - ctx.builder.CreateBr(b_after); - ctx.builder.SetInsertPoint(b_generic); - } - Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); - if (age_ok) { - ctx.builder.CreateBr(b_after); - ctx.builder.SetInsertPoint(b_after); - PHINode *retphi = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 2); - retphi->addIncoming(ret_jlcall, b_jlcall); - retphi->addIncoming(ret, b_generic); - ret = retphi; - } - retval = mark_julia_type(ctx, ret, true, astrt); - } - else { - bool is_opaque_closure = jl_is_method(lam->def.value) && lam->def.method->is_for_opaque_closure; - assert(calltype == 3); - // emit a specsig call - StringRef protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)callptr, invoke, codeinst); - bool gcstack_arg = JL_FEAT_TEST(ctx, gcstack_arg); - jl_returninfo_t returninfo = get_specsig_function(ctx, M, NULL, protoname, lam->specTypes, astrt, is_opaque_closure, gcstack_arg); - if (age_ok) { - funcName += "_gfthunk"; - Function *gf_thunk = Function::Create(returninfo.decl.getFunctionType(), - GlobalVariable::InternalLinkage, funcName, M); - jl_init_function(gf_thunk, ctx.emission_context.TargetTriple); - gf_thunk->setAttributes(AttributeList::get(M->getContext(), {returninfo.attrs, gf_thunk->getAttributes()})); - // build a specsig -> jl_apply_generic converter thunk - // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), - // but which has the signature of a specsig - emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, lam->specTypes, codeinst->rettype, is_opaque_closure, nargs + 1, ctx.emission_context, - prepare_call_in(gf_thunk->getParent(), jlapplygeneric_func), min_world, max_world); - returninfo.decl = FunctionCallee(returninfo.decl.getFunctionType(), ctx.builder.CreateSelect(age_ok, returninfo.decl.getCallee(), gf_thunk)); - } - retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, codeinst->rettype, returninfo, nullptr, inputargs, nargs + 1); - jlfunc_sret = (returninfo.cc == jl_returninfo_t::SRet); + if (codeinst) { + retval = emit_invoke(ctx, mark_julia_const(ctx, (jl_value_t*)codeinst), inputargs, nargs + 1, astrt, age_ok); + jlfunc_sret = retval.V && isa(retval.V) && !retval.TIndex && retval.inline_roots.empty(); if (jlfunc_sret && sig.sret) { // fuse the two sret together assert(retval.ispointer()); @@ -7781,6 +7712,12 @@ static Function* gen_cfun_wrapper( result->eraseFromParent(); } } + else { + // emit a dispatch + jlfunc_sret = false; + Value *ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, inputargs, nargs + 1, julia_call); + retval = mark_julia_type(ctx, ret, true, astrt); + } // inline a call to typeassert here, if required emit_typecheck(ctx, retval, declrt, "cfunction"); @@ -7953,15 +7890,13 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con } } size_t world = jl_atomic_load_acquire(&jl_world_counter); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; // try to look up this function for direct invoking - jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0) : NULL; + jl_method_instance_t *lam = sigt ? jl_get_specialization1((jl_tupletype_t*)sigt, world, 0) : NULL; Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, sig, fexpr_rt.constant, NULL, declrt, lam, - unionall_env, sparam_vals, &closure_types, min_valid, max_valid); + unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { // F is actually an init_trampoline function that returns the real address @@ -8021,7 +7956,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // do codegen to create a C-callable alias/wrapper, or if sysimg_handle is set, // restore one from a loaded system image. -const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) +const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) { ++GeneratedCCallables; jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt); @@ -8050,8 +7985,6 @@ const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysi argtypes, NULL, false, CallingConv::C, false, ¶ms); if (sig.err_msg.empty()) { size_t world = jl_atomic_load_acquire(&jl_world_counter); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; if (sysimg_handle) { // restore a ccallable from the system image void *addr; @@ -8064,9 +7997,9 @@ const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysi } } else { - jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, &min_valid, &max_valid, 0); + jl_method_instance_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt, world, 0); //Safe b/c params holds context lock - gen_cfun_wrapper(unwrap(llvmmod)->getModuleUnlocked(), params, sig, ff, name, declrt, lam, NULL, NULL, NULL, min_valid, max_valid); + gen_cfun_wrapper(llvmmod, params, sig, ff, name, declrt, lam, NULL, NULL, NULL); } JL_GC_POP(); return name; @@ -8128,7 +8061,7 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlretty, j } argv[i] = mark_julia_type(ctx, theArg, true, ty); } - jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, jlretty, f, nullptr, argv, nargs); + jl_cgval_t retval = emit_call_specfun_other(ctx, is_opaque_closure, lam->specTypes, jlretty, f, argv, nargs); if (retarg != -1) { Value *theArg; if (retarg == 0) @@ -10084,8 +10017,7 @@ static jl_llvm_functions_t jl_emit_oc_wrapper(orc::ThreadSafeModule &m, jl_codeg size_t nrealargs = jl_nparams(mi->specTypes); emit_specsig_to_fptr1(gf_thunk, returninfo.cc, returninfo.return_roots, mi->specTypes, rettype, true, nrealargs, ctx.emission_context, - prepare_call_in(gf_thunk->getParent(), jlopaque_closure_call_func), // TODO: this could call emit_oc_call directly - ctx.min_world, ctx.max_world); + prepare_call_in(gf_thunk->getParent(), jlopaque_closure_call_func)); // TODO: this could call emit_oc_call directly declarations.specFunctionObject = funcName; } return declarations; diff --git a/src/gc-common.c b/src/gc-common.c index c751b54f059f5..3d578b81578b1 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -557,17 +557,8 @@ size_t jl_genericmemory_nbytes(jl_genericmemory_t *m) JL_NOTSAFEPOINT // tracking Memorys with malloc'd storage void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned){ // This is **NOT** a GC safe point. - mallocmemory_t *ma; - if (ptls->gc_tls_common.heap.mafreelist == NULL) { - ma = (mallocmemory_t*)malloc_s(sizeof(mallocmemory_t)); - } - else { - ma = ptls->gc_tls_common.heap.mafreelist; - ptls->gc_tls_common.heap.mafreelist = ma->next; - } - ma->a = (jl_genericmemory_t*)((uintptr_t)m | !!isaligned); - ma->next = ptls->gc_tls_common.heap.mallocarrays; - ptls->gc_tls_common.heap.mallocarrays = ma; + void *a = (void*)((uintptr_t)m | !!isaligned); + small_arraylist_push(&ptls->gc_tls_common.heap.mallocarrays, a); } // =========================================================================== // diff --git a/src/gc-common.h b/src/gc-common.h index 3007151009f7d..1745df17950f9 100644 --- a/src/gc-common.h +++ b/src/gc-common.h @@ -61,12 +61,6 @@ extern jl_gc_callback_list_t *gc_cblist_notify_gc_pressure; // malloc wrappers, aligned allocation // =========================================================================== // -// data structure for tracking malloc'd genericmemory. -typedef struct _mallocmemory_t { - jl_genericmemory_t *a; // lowest bit is tagged if this is aligned memory - struct _mallocmemory_t *next; -} mallocmemory_t; - #if defined(_OS_WINDOWS_) STATIC_INLINE void *jl_malloc_aligned(size_t sz, size_t align) { diff --git a/src/gc-debug.c b/src/gc-debug.c index 7c479484cde45..6e51064035b7b 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -1025,12 +1025,11 @@ void gc_stats_big_obj(void) v = v->next; } - mallocmemory_t *ma = ptls2->gc_tls.heap.mallocarrays; - while (ma != NULL) { - uint8_t bits =jl_astaggedvalue(ma->a)->bits.gc; + void **lst = ptls2->gc_tls.heap.mallocarrays.items; + for (size_t i = 0, l = ptls2->gc_tls.heap.mallocarrays.len; i < l; i++) { + jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[i] & ~(uintptr_t)1); + uint8_t bits = jl_astaggedvalue(m)->bits.gc; if (gc_marked(bits)) { - jl_genericmemory_t *m = (jl_genericmemory_t*)ma->a; - m = (jl_genericmemory_t*)((uintptr_t)m & ~(uintptr_t)1); size_t sz = jl_genericmemory_nbytes(m); if (gc_old(bits)) { assert(bits == GC_OLD_MARKED); @@ -1042,7 +1041,6 @@ void gc_stats_big_obj(void) stat.nbytes_used += sz; } } - ma = ma->next; } } jl_safe_printf("%lld kB (%lld%% old) in %lld large objects (%lld%% old)\n", diff --git a/src/gc-stock.c b/src/gc-stock.c index 61a013f347975..c1bc1d64ae199 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -388,24 +388,18 @@ static void sweep_weak_refs(void) jl_ptls_t ptls2 = gc_all_tls_states[i]; if (ptls2 != NULL) { size_t n = 0; - size_t ndel = 0; + size_t i = 0; size_t l = ptls2->gc_tls_common.heap.weak_refs.len; void **lst = ptls2->gc_tls_common.heap.weak_refs.items; - if (l == 0) - continue; - while (1) { - jl_weakref_t *wr = (jl_weakref_t*)lst[n]; - if (gc_marked(jl_astaggedvalue(wr)->bits.gc)) + // filter with preserving order + for (i = 0; i < l; i++) { + jl_weakref_t *wr = (jl_weakref_t*)lst[i]; + if (gc_marked(jl_astaggedvalue(wr)->bits.gc)) { + lst[n] = wr; n++; - else - ndel++; - if (n >= l - ndel) - break; - void *tmp = lst[n]; - lst[n] = lst[n + ndel]; - lst[n + ndel] = tmp; + } } - ptls2->gc_tls_common.heap.weak_refs.len -= ndel; + ptls2->gc_tls_common.heap.weak_refs.len = n; } } } @@ -629,10 +623,9 @@ void jl_gc_reset_alloc_count(void) JL_NOTSAFEPOINT reset_thread_gc_counts(); } -static void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT +static void jl_gc_free_memory(jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT { - assert(jl_is_genericmemory(v)); - jl_genericmemory_t *m = (jl_genericmemory_t*)v; + assert(jl_is_genericmemory(m)); assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2); char *d = (char*)m->ptr; size_t freed_bytes = memory_block_usable_size(d, isaligned); @@ -654,25 +647,23 @@ static void sweep_malloced_memory(void) JL_NOTSAFEPOINT for (int t_i = 0; t_i < gc_n_threads; t_i++) { jl_ptls_t ptls2 = gc_all_tls_states[t_i]; if (ptls2 != NULL) { - mallocmemory_t *ma = ptls2->gc_tls_common.heap.mallocarrays; - mallocmemory_t **pma = &ptls2->gc_tls_common.heap.mallocarrays; - while (ma != NULL) { - mallocmemory_t *nxt = ma->next; - jl_value_t *a = (jl_value_t*)((uintptr_t)ma->a & ~1); - int bits = jl_astaggedvalue(a)->bits.gc; - if (gc_marked(bits)) { - pma = &ma->next; + size_t n = 0; + size_t l = ptls2->gc_tls_common.heap.mallocarrays.len; + void **lst = ptls2->gc_tls_common.heap.mallocarrays.items; + // filter without preserving order + while (n < l) { + jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[n] & ~1); + if (gc_marked(jl_astaggedvalue(m)->bits.gc)) { + n++; } else { - *pma = nxt; - int isaligned = (uintptr_t)ma->a & 1; - jl_gc_free_memory(a, isaligned); - ma->next = ptls2->gc_tls_common.heap.mafreelist; - ptls2->gc_tls_common.heap.mafreelist = ma; + int isaligned = (uintptr_t)lst[n] & 1; + jl_gc_free_memory(m, isaligned); + l--; + lst[n] = lst[l]; } - gc_time_count_mallocd_memory(bits); - ma = nxt; } + ptls2->gc_tls_common.heap.mallocarrays.len = l; } } gc_time_mallocd_memory_end(); @@ -3439,8 +3430,7 @@ void jl_init_thread_heap(jl_ptls_t ptls) small_arraylist_new(&common_heap->live_tasks, 0); for (int i = 0; i < JL_N_STACK_POOLS; i++) small_arraylist_new(&common_heap->free_stacks[i], 0); - common_heap->mallocarrays = NULL; - common_heap->mafreelist = NULL; + small_arraylist_new(&common_heap->mallocarrays, 0); heap->young_generation_of_bigvals = (bigval_t*)calloc_s(sizeof(bigval_t)); // sentinel assert(gc_bigval_sentinel_tag != 0); // make sure the sentinel is initialized heap->young_generation_of_bigvals->header = gc_bigval_sentinel_tag; diff --git a/src/gc-tls-common.h b/src/gc-tls-common.h index ba36f5c1c238e..473668d648294 100644 --- a/src/gc-tls-common.h +++ b/src/gc-tls-common.h @@ -21,9 +21,8 @@ typedef struct { // that are holding onto a stack from the pool small_arraylist_t live_tasks; - // variables for tracking malloc'd arrays - struct _mallocmemory_t *mallocarrays; - struct _mallocmemory_t *mafreelist; + // variable for tracking malloc'd arrays + small_arraylist_t mallocarrays; #define JL_N_STACK_POOLS 16 small_arraylist_t free_stacks[JL_N_STACK_POOLS]; diff --git a/src/gf.c b/src/gf.c index bbf065a4fac0d..a5ba309784557 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3084,7 +3084,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc } // compile-time method lookup -jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache) +jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache) { if (jl_has_free_typevars((jl_value_t*)types)) return NULL; // don't poison the cache due to a malformed query @@ -3096,10 +3096,6 @@ jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES size_t max_valid2 = ~(size_t)0; int ambig = 0; jl_value_t *matches = jl_matching_methods(types, jl_nothing, 1, 1, world, &min_valid2, &max_valid2, &ambig); - if (*min_valid < min_valid2) - *min_valid = min_valid2; - if (*max_valid > max_valid2) - *max_valid = max_valid2; if (matches == jl_nothing || jl_array_nrows(matches) != 1 || ambig) return NULL; JL_GC_PUSH1(&matches); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 80867daade267..25494ba57d257 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -382,7 +382,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t jl_method_instance_t *mi = codeinst->def; size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke, 0, 0); + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); jl_gc_unsafe_leave(ct->ptls, gc_state); preal_decl = ""; // no need to fixup the name } @@ -713,7 +713,7 @@ static void jl_emit_codeinst_to_jit( int waiting = jl_analyze_workqueue(codeinst, params); if (waiting) { auto release = std::move(params.tsctx_lock); // unlock again before moving from it - incompletemodules.insert(std::pair(codeinst, std::make_tuple(std::move(params), waiting))); + incompletemodules.try_emplace(codeinst, std::move(params), waiting); } else { finish_params(result_m.getModuleUnlocked(), params); @@ -760,7 +760,7 @@ static void _jl_compile_codeinst( } -const char *jl_generate_ccallable(LLVMOrcThreadSafeModuleRef llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); +const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); // compile a C-callable alias extern "C" JL_DLLEXPORT_CODEGEN @@ -774,45 +774,68 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled); if (measure_compile_time_enabled) compiler_start_time = jl_hrtime(); + jl_codegen_params_t *pparams = (jl_codegen_params_t*)p; + DataLayout DL = pparams ? pparams->DL : jl_ExecutionEngine->getDataLayout(); + Triple TargetTriple = pparams ? pparams->TargetTriple : jl_ExecutionEngine->getTargetTriple(); orc::ThreadSafeContext ctx; auto into = unwrap(llvmmod); - jl_codegen_params_t *pparams = (jl_codegen_params_t*)p; orc::ThreadSafeModule backing; + bool success = true; + const char *name = ""; + SmallVector dependencies; if (into == NULL) { - if (!pparams) { - ctx = jl_ExecutionEngine->makeContext(); - } - backing = jl_create_ts_module("cextern", pparams ? pparams->tsctx : ctx, pparams ? pparams->DL : jl_ExecutionEngine->getDataLayout(), pparams ? pparams->TargetTriple : jl_ExecutionEngine->getTargetTriple()); + ctx = pparams ? pparams->tsctx : jl_ExecutionEngine->makeContext(); + backing = jl_create_ts_module("cextern", ctx, DL, TargetTriple); into = &backing; } - bool success = true; - { - auto Lock = into->getContext().getLock(); - Module *M = into->getModuleUnlocked(); - jl_codegen_params_t params(into->getContext(), M->getDataLayout(), Triple(M->getTargetTriple())); - params.imaging_mode = imaging_default(); + { // params scope + jl_codegen_params_t params(into->getContext(), DL, TargetTriple); if (pparams == NULL) { - M->getContext().setDiscardValueNames(true); + params.cache = p == NULL; + params.imaging_mode = imaging_default(); + params.tsctx.getContext()->setDiscardValueNames(true); pparams = ¶ms; } - assert(pparams->tsctx.getContext() == into->getContext().getContext()); - const char *name = jl_generate_ccallable(wrap(into), sysimg, declrt, sigt, *pparams); - if (!sysimg) { - jl_unique_gcsafe_lock lock(extern_c_lock); - if (jl_ExecutionEngine->getGlobalValueAddress(name)) { - success = false; + Module &M = *into->getModuleUnlocked(); + assert(pparams->tsctx.getContext() == &M.getContext()); + name = jl_generate_ccallable(&M, sysimg, declrt, sigt, *pparams); + if (!sysimg && !p) { + { // drop lock to keep analyzer happy (since it doesn't know we have the only reference to it) + auto release = std::move(params.tsctx_lock); } - if (success && p == NULL) { - jl_jit_globals(params.global_targets); - assert(params.workqueue.empty()); - if (params._shared_module) { - jl_ExecutionEngine->optimizeDLSyms(*params._shared_module); // safepoint - jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); + { // lock scope + jl_unique_gcsafe_lock lock(extern_c_lock); + if (jl_ExecutionEngine->getGlobalValueAddress(name)) + success = false; + } + params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock + if (success && params.cache) { + for (auto &it : params.workqueue) { + jl_code_instance_t *codeinst = it.first; + JL_GC_PROMISE_ROOTED(codeinst); + dependencies.push_back(codeinst); + recursive_compile_graph(codeinst, nullptr); } + jl_analyze_workqueue(nullptr, params, true); + assert(params.workqueue.empty()); + finish_params(&M, params); } - if (success && llvmmod == NULL) { - jl_ExecutionEngine->optimizeDLSyms(*M); // safepoint - jl_ExecutionEngine->addModule(std::move(*into)); + } + pparams = nullptr; + } + if (!sysimg && success && llvmmod == NULL) { + { // lock scope + jl_unique_gcsafe_lock lock(extern_c_lock); + if (!jl_ExecutionEngine->getGlobalValueAddress(name)) { + for (auto dep : dependencies) + jl_compile_codeinst_now(dep); + { + auto Lock = backing.getContext().getLock(); + jl_ExecutionEngine->optimizeDLSyms(*backing.getModuleUnlocked()); // safepoint + } + jl_ExecutionEngine->addModule(std::move(backing)); + success = jl_ExecutionEngine->getGlobalValueAddress(name); + assert(success); } } } diff --git a/src/jitlayers.h b/src/jitlayers.h index 6665be6a33faa..b6f1e9ec20c6b 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -260,7 +260,7 @@ struct jl_codegen_params_t { bool external_linkage = false; bool imaging_mode; bool use_swiftcc = true; - jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) + jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER : tsctx(std::move(ctx)), tsctx_lock(tsctx.getLock()), DL(std::move(DL)), @@ -271,6 +271,8 @@ struct jl_codegen_params_t { if (TargetTriple.isRISCV()) use_swiftcc = false; } + jl_codegen_params_t(jl_codegen_params_t &&) JL_NOTSAFEPOINT = default; + ~jl_codegen_params_t() JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE = default; }; jl_llvm_functions_t jl_emit_code( @@ -299,8 +301,7 @@ void emit_specsig_to_fptr1( jl_value_t *calltype, jl_value_t *rettype, bool is_for_opaque_closure, size_t nargs, jl_codegen_params_t ¶ms, - Function *target, - size_t min_world, size_t max_world) JL_NOTSAFEPOINT; + Function *target) JL_NOTSAFEPOINT; Function *get_or_emit_fptr1(StringRef Name, Module *M) JL_NOTSAFEPOINT; void jl_init_function(Function *F, const Triple &TT) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 4741316093f95..e0e8158f8e0d9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1195,7 +1195,7 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache); +JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp); JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; diff --git a/src/mtarraylist.c b/src/mtarraylist.c index 7af265a86ab63..0a0f3fe867e39 100644 --- a/src/mtarraylist.c +++ b/src/mtarraylist.c @@ -14,8 +14,8 @@ extern "C" { // but there can be any number of observers typedef struct { - _Atomic(uint32_t) len; - uint32_t max; + _Atomic(size_t) len; + size_t max; _Atomic(_Atomic(void*)*) items; _Atomic(void*) _space[SMALL_AL_N_INLINE]; } small_mtarraylist_t; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 81c60ba70d29f..4717351d499b0 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -264,10 +264,8 @@ static void *jl_precompile_(jl_array_t *m, int external_linkage) jl_value_t *item = jl_array_ptr_ref(m, i); if (jl_is_method_instance(item)) { mi = (jl_method_instance_t*)item; - size_t min_world = 0; - size_t max_world = ~(size_t)0; if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) - mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), 0); if (mi) jl_array_ptr_1d_push(m2, (jl_value_t*)mi); } diff --git a/src/support/arraylist.h b/src/support/arraylist.h index a83bd2808756c..8d4ef61ba251c 100644 --- a/src/support/arraylist.h +++ b/src/support/arraylist.h @@ -5,7 +5,7 @@ #define AL_N_INLINE 29 -#define SMALL_AL_N_INLINE 6 +#define SMALL_AL_N_INLINE 5 #ifdef __cplusplus extern "C" { @@ -13,7 +13,7 @@ extern "C" { #include "analyzer_annotations.h" -typedef struct { +typedef struct { // 32 words size_t len; size_t max; void **items; @@ -27,9 +27,9 @@ JL_DLLEXPORT void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT; JL_DLLEXPORT void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT; JL_DLLEXPORT void arraylist_grow(arraylist_t *a, size_t n) JL_NOTSAFEPOINT; -typedef struct { - uint32_t len; - uint32_t max; +typedef struct { // 8 words + size_t len; + size_t max; void **items; void *_space[SMALL_AL_N_INLINE]; } small_arraylist_t; diff --git a/test/bitarray.jl b/test/bitarray.jl index 67d8fae0eda6d..fd5c1421a256f 100644 --- a/test/bitarray.jl +++ b/test/bitarray.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Base: findprevnot, findnextnot -using Random, LinearAlgebra, Test +using Random, Test, LinearAlgebra # Ideally, these tests should not depend on LinearAlgebra isdefined(Main, :SizedArrays) || @eval Main include("testhelpers/SizedArrays.jl") using .Main.SizedArrays @@ -15,7 +15,6 @@ tc(r1,r2) = false bitcheck(b::BitArray) = Test._check_bitarray_consistency(b) bitcheck(x) = true -bcast_setindex!(b, x, I...) = (b[I...] .= x; b) function check_bitop_call(ret_type, func, args...; kwargs...) r2 = func(map(x->(isa(x, BitArray) ? Array(x) : x), args)...; kwargs...) @@ -34,6 +33,9 @@ macro check_bit_operation(ex) Expr(:call, :check_bitop_call, nothing, map(esc, ex.args)...) end +bcast_setindex!(b, x, I...) = (b[I...] .= x; b) + + let t0 = time_ns() global timesofar function timesofar(str) @@ -1641,69 +1643,6 @@ end timesofar("cat") -@testset "Linear algebra" begin - b1 = bitrand(v1) - b2 = bitrand(v1) - @check_bit_operation dot(b1, b2) Int - - b1 = bitrand(n1, n2) - @test_throws ArgumentError tril(b1, -n1 - 2) - @test_throws ArgumentError tril(b1, n2) - @test_throws ArgumentError triu(b1, -n1) - @test_throws ArgumentError triu(b1, n2 + 2) - for k in (-n1 - 1):(n2 - 1) - @check_bit_operation tril(b1, k) BitMatrix - end - for k in (-n1 + 1):(n2 + 1) - @check_bit_operation triu(b1, k) BitMatrix - end - - for sz = [(n1,n1), (n1,n2), (n2,n1)], (f,isf) = [(tril,istril), (triu,istriu)] - b1 = bitrand(sz...) - @check_bit_operation isf(b1) Bool - b1 = f(bitrand(sz...)) - @check_bit_operation isf(b1) Bool - end - - b1 = bitrand(n1,n1) - b1 .|= copy(b1') - @check_bit_operation issymmetric(b1) Bool - @check_bit_operation ishermitian(b1) Bool - - b1 = bitrand(n1) - b2 = bitrand(n2) - @check_bit_operation kron(b1, b2) BitVector - - b1 = bitrand(s1, s2) - b2 = bitrand(s3, s4) - @check_bit_operation kron(b1, b2) BitMatrix - - b1 = bitrand(v1) - @check_bit_operation diff(b1) Vector{Int} - - b1 = bitrand(n1, n2) - @check_bit_operation diff(b1, dims=1) Matrix{Int} - @check_bit_operation diff(b1, dims=2) Matrix{Int} - - b1 = bitrand(n1, n1) - @test ((svdb1, svdb1A) = (svd(b1), svd(Array(b1))); - svdb1.U == svdb1A.U && svdb1.S == svdb1A.S && svdb1.V == svdb1A.V) - @test ((qrb1, qrb1A) = (qr(b1), qr(Array(b1))); - Matrix(qrb1.Q) == Matrix(qrb1A.Q) && qrb1.R == qrb1A.R) - - b1 = bitrand(v1) - @check_bit_operation diagm(0 => b1) BitMatrix - - b1 = bitrand(v1) - b2 = bitrand(v1) - @check_bit_operation diagm(-1 => b1, 1 => b2) BitMatrix - - b1 = bitrand(n1, n1) - @check_bit_operation diag(b1) -end - -timesofar("linalg") - @testset "findmax, findmin" begin b1 = trues(0) @test_throws ArgumentError findmax(b1) diff --git a/test/llvmcall.jl b/test/llvmcall.jl index c83ac05b1ec48..ddf66ca680d45 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -157,7 +157,7 @@ module ObjLoadTest nothing end @test_throws(ErrorException("@ccallable was already defined for this method name"), - @eval @ccallable Cvoid jl_the_callback(not_the_method::Int) = "other") + @eval @ccallable String jl_the_callback(not_the_method::Int) = "other") # Make sure everything up until here gets compiled @test jl_the_callback() === nothing @test jl_the_callback(1) == "other" diff --git a/test/meta.jl b/test/meta.jl index 5fb1ebc0d3647..2b235d3cc1b0d 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -276,6 +276,11 @@ ci = code_lowered(g, Tuple{Val{true}})[1] @test Meta.partially_inline!(copy(ci.code), Any[isdefined_globalref, 1], Tuple{typeof(isdefined_globalref), Int}, [], 0, 0, :propagate)[1] == Expr(:isdefined, GlobalRef(Base, :foo)) + withunreachable(s::String) = sin(s) + ci = code_lowered(withunreachable, Tuple{String})[1] + ci.code[end] = Core.ReturnNode() + @test Meta.partially_inline!(copy(ci.code), Any[withunreachable, "foo"], Tuple{typeof(withunreachable), String}, + [], 0, 0, :propagate)[end] == Core.ReturnNode() end @testset "Base.Meta docstrings" begin