diff --git a/src/Actions.jl b/src/Actions.jl index 9dd5aac..1673f84 100644 --- a/src/Actions.jl +++ b/src/Actions.jl @@ -71,8 +71,7 @@ function process( !is_glob_match(path, settings.SliderServer.exclude) skip_cache = keep_running || - is_glob_match(path, settings.Export.ignore_cache) || - path ∈ settings.Export.ignore_cache + is_glob_match(path, settings.Export.ignore_cache) cached_state = skip_cache ? nothing : try_fromcache(settings.Export.cache_dir, new_hash) @@ -136,6 +135,7 @@ function process( output_dir, ) # TODO shutdown + # TODO cache end @info "### ✓ $(progress) Ready" s.path new_hash diff --git a/src/Configuration.jl b/src/Configuration.jl index e86adf3..6016fe3 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -32,6 +32,11 @@ end "List of notebook files to skip precomputation. Provide paths relative to `start_dir`." exclude::Vector{String} = String[] max_filesize_per_group::Integer = 1_000_000 + + "Combines multiple bind state precomputations into a single file" + bundling_enabled::Bool = false + "Maximum filesize of a full bundle including all group bond updates. Since these files are sent in full over the internet, their size must be small" + max_full_bundle_size::Integer = 500_000 end @extract_docs @option struct ExportSettings diff --git a/src/precomputed/index.jl b/src/precomputed/index.jl index 5d5e3c9..735704e 100644 --- a/src/precomputed/index.jl +++ b/src/precomputed/index.jl @@ -130,10 +130,14 @@ function generate_precomputed_staterequests( connections = run.bond_connections current_hash = sesh.current_hash + should_bundle = settings.Precompute.bundling_enabled + @assert run isa RunningNotebook mkpath(joinpath(output_dir, "bondconnections")) + mkpath(joinpath(output_dir, "bundles")) mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash))) + mkpath(joinpath(output_dir, "staterequest-bundled", URIs.escapeuri(current_hash))) bondconnections_path = joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash)) @@ -156,8 +160,12 @@ function generate_precomputed_staterequests( @warn "Notebook cannot be (fully) precomputed" report end + bundle_index = String[] foreach(groups) do group::VariableGroupPossibilities if group.judgement.should_precompute_all + should_bundle_group = should_bundle && group.judgement.can_fully_bundle + + bundle = Dict{String,Any}() for (combination, bonds_dict) in combination_iterator(group) filename = Pluto.pack(bonds_dict) |> base64urlencode if length(filename) > 255 @@ -176,10 +184,32 @@ function generate_precomputed_staterequests( write(write_path, Pluto.pack(result)) + if should_bundle_group + bundle[filename] = result + end + @debug "Written state request to " write_path values = (; (zip(group.names, combination))...) end end + + if should_bundle_group + bundle_signature = Pluto.pack(sort(group.names)) |> base64urlencode + push!(bundle_index, bundle_signature) + bundle_path = joinpath( + output_dir, + "staterequest-bundled", + URIs.escapeuri(current_hash), + bundle_signature, + ) + write(bundle_path, Pluto.pack(bundle)) + + @debug "Written bundled states to " bundle_path bundle_signature + end end end + write( + joinpath(output_dir, "bundles", URIs.escapeuri(current_hash)), + Pluto.pack(bundle_index), + ) end diff --git a/src/precomputed/types.jl b/src/precomputed/types.jl index 05e6e39..f3e0e02 100644 --- a/src/precomputed/types.jl +++ b/src/precomputed/types.jl @@ -11,6 +11,7 @@ const Reason = Symbol Base.@kwdef struct Judgement should_precompute_all::Bool = false + can_fully_bundle::Bool = false not_available::Bool = false close_to_filesize_limit::Bool = false exceeds_filesize_limit::Bool = false @@ -42,17 +43,21 @@ function VariableGroupPossibilities(; @assert settings isa PlutoDeploySettings limit = settings.Precompute.max_filesize_per_group + bundle_limit = settings.Precompute.max_full_bundle_size current = mean(file_size_sample_distribution) exceeds_filesize_limit = current > limit + exceeds_bundle_size_limit = current > bundle_limit close_to_filesize_limit = current > limit * 0.7 else - exceeds_filesize_limit = close_to_filesize_limit = false + + exceeds_filesize_limit = exceeds_bundle_size_limit = close_to_filesize_limit = false + end j = Judgement(; should_precompute_all=!is_not_available && !exceeds_filesize_limit, - exceeds_filesize_limit, + can_fully_bundle=!exceeds_bundle_size_limit, close_to_filesize_limit, not_available=is_not_available, ) @@ -203,9 +208,15 @@ sum_distributions(ds; init=Normal(0, 0)) = any(isnothing, ds) ? nothing : reduce(convolve, ds; init=init) -pretty(j::Judgement) = - if j.should_precompute_all +pretty(j::Judgement) = begin + s = "" + s *= if j.should_precompute_all j.close_to_filesize_limit ? "⚠️" : "✓" else "❌" end + if j.can_fully_bundle + s *= " 📦" + end + s +end