Refactor (GLMakie) render order (experimental)

GLMakie/assets/shader/postprocessing/OIT_blend.frag

@@ -8,18 +8,18 @@ in vec2 frag_uv;
 // contains sum_i C_i * weight(depth_i, alpha_i)
 uniform sampler2D sum_color;
 // contains pod_i (1 - alpha_i)
-uniform sampler2D prod_alpha;
+uniform sampler2D transmittance;
 
 out vec4 fragment_color;
 
 void main(void)
 {
     vec4 summed_color_weight = texture(sum_color, frag_uv);
-    float transmittance = texture(prod_alpha, frag_uv).r;
+    float tr = texture(transmittance, frag_uv).r;
 
     vec3 weighted_transparent = summed_color_weight.rgb / max(summed_color_weight.a, 0.00001);
-    vec3 full_weighted_transparent = weighted_transparent * (1 - transmittance);
+    vec3 full_weighted_transparent = weighted_transparent * (1 - tr);
 
     fragment_color.rgb = full_weighted_transparent;
-    fragment_color.a = transmittance;
+    fragment_color.a = tr;
 }

GLMakie/assets/shader/postprocessing/copy.frag

@@ -6,7 +6,5 @@ out vec4 fragment_color;
 
 void main(void)
 {
-    vec4 color = texture(color_texture, frag_uv);
-    fragment_color.rgb = color.rgb;
-    fragment_color.a = 1.0;
+    fragment_color = texture(color_texture, frag_uv);
 }

GLMakie/assets/shader/postprocessing/fxaa.frag

@@ -1047,4 +1047,5 @@ void main(void)
         0.0f,                                    // FxaaFloat fxaaConsoleEdgeThresholdMin,
         FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f)        // FxaaFloat fxaaConsole360ConstDir,
     ).rgb;
+    fragment_color.a = 0.0;
 }

GLMakie/assets/shader/postprocessing/postprocess.frag

@@ -21,17 +21,15 @@ bool unpack_bool(uint id) {
 
 void main(void)
 {
+    // color.a is transmittance, i.e. 1 - alpha
     vec4 color = texture(color_texture, frag_uv).rgba;
-    if(color.a <= 0){
-        discard;
-    }
-
     uint id = texture(object_ids, frag_uv).x;
+
     // do tonemappings
     //opaque = linear_tone_mapping(color.rgb, 1.8);  // linear color output
     fragment_color.rgb = color.rgb;
     // we store fxaa = true/false in highbit of the object id
-    if (unpack_bool(id)) {
+    if (unpack_bool(id) && (color.a < 1.0)) {
         fragment_color.a = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // compute luma
     } else {
         // we disable fxaa by setting luma to 1

GLMakie/src/GLAbstraction/GLRender.jl

@@ -173,10 +173,24 @@ end
 function enabletransparency()
     glEnablei(GL_BLEND, 0)
     glDisablei(GL_BLEND, 1)
+    # We render to a potentially transparent scene here. The final blending 
+    # may look like this:
+    # (s)creen <- (b)ackground <- (r)ender
+    # with the screen being opaque. This should blend as:
+    #   r.a * r.rgb + (1-r.a) * (b.a * b.rgb + (1-b.a) * s.rgb), 1
+    # = r.a * r.rgb + (1-r.a) * b.a * b.rgb + (1-r.a) * (1-b.a) * s.rgb, 1
+    # If we want to run the screen blend at a later stage we can:
+    # - precalculate p = b.a * b.rgb, 1 - b.a in the background buffer
+    # - track (1 - r.a) * p.a as alpha in the buffer
+    # which gives us the following blending rule:
+    # r.a * r.rgb + (1-r.a) * p.rgb, 0 * r.a + (1-r.a) * p.a 
+    glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ZERO, GL_ONE_MINUS_SRC_ALPHA)
+
     # This does:
     # target.rgb = source.a * source.rgb + (1 - source.a) * target.rgb
     # target.a = 0 * source.a + 1 * target.a
     # the latter is required to keep target.a = 1 for the OIT pass
-    glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ZERO, GL_ONE)
+    # glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ZERO, GL_ONE)
+    # glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
     return
 end

GLMakie/src/GLScene.jl

@@ -0,0 +1,246 @@
+struct GLScene
+    scene::WeakRef # TODO phase this out?
+
+    # TODO: do these need to be reactive for render on demand?
+    viewport::Observable{Rect2f}
+    clear::Observable{Bool}
+    backgroundcolor::Observable{RGBAf}
+    visible::Observable{Bool}
+    # clear_depth::Bool
+    # stecil/scene_based_occlusion
+
+    # TODO:
+    # - move postprocessors here
+    # - make them controllable by Makie
+    # - move SSAO settings into postprocessor controls
+    ssao::Makie.SSAO
+    # postprocessors
+
+    renderobjects::Vector{RenderObject}
+
+    # TODO: WeakRef or not?
+    plot2robj::Dict{UInt64, RenderObject}
+    robj2plot::Dict{UInt32, Plot}
+end
+
+function GLScene(scene::Scene)
+    return GLScene(
+        WeakRef(scene), 
+
+        scene.viewport,
+        scene.clear,
+        scene.backgroundcolor,
+        scene.visible,
+        scene.ssao,
+
+        RenderObject[],
+        Dict{UInt64, RenderObject}(),
+        Dict{UInt32, Plot}()
+    ) 
+end
+
+# A bit dodgy...
+Base.haskey(glscene::GLScene, robj::RenderObject) = haskey(glscene, robj.id)
+Base.haskey(glscene::GLScene, plot::AbstractPlot) = haskey(glscene, objectid(plot))
+Base.haskey(glscene::GLScene, robj_id::UInt32) = haskey(glscene.robj2plot, robj_id)
+Base.haskey(glscene::GLScene, plot_id::UInt64) = haskey(glscene.plot2robj, plot_id)
+
+Base.getindex(glscene::GLScene, robj::RenderObject) = getindex(glscene, robj.id)
+Base.getindex(glscene::GLScene, plot::AbstractPlot) = getindex(glscene, objectid(plot))
+Base.getindex(glscene::GLScene, robj_id::UInt32) = getindex(glscene.robj2plot, robj_id)
+Base.getindex(glscene::GLScene, plot_id::UInt64) = getindex(glscene.plot2robj, plot_id)
+
+
+function Base.show(io::IO, glscene::GLScene)
+    println(io, 
+        "GLScene(", 
+        glscene === nothing ? "nothing" : "scene", ", ", 
+        "viewport = ", glscene.viewport[], ", ", 
+        "clear = ", glscene.clear[], ", ", 
+        "backgroundcolor = ", glscene.backgroundcolor[], ", ", 
+        "visible = ", glscene.visible[], ", ", 
+        "ssao = ..., ", 
+        length(glscene.renderobjects), " RenderObjects",
+        ")"
+    )
+end
+
+
+# TODO: name? should we keep this separate from Screen?
+struct GLSceneTree
+    scene2index::Dict{WeakRef, Int}
+    robj2plot::Dict{UInt32, AbstractPlot} # for picking
+
+    # flattened scene tree
+    # Order:
+    # Scene        1
+    #   Scene      2
+    #     Scene    3
+    #     Scene    4
+    #       Scene  5
+    #     Scene    6
+    #   Scene      7
+    #   Scene      8
+    #     Scene    9
+    scenes::Vector{GLScene}
+    depth::Vector{Int}
+end
+
+GLSceneTree() = GLSceneTree(Dict{WeakRef, Int}(), Dict{UInt32, AbstractPlot}(), GLScene[], Int[])
+
+function gc_cleanup(tree::GLSceneTree)
+    @debug "WeakRef cleanup"
+    # TODO: do we need this? Can this create orphaned child scenes?
+    for k in copy(keys(tree.scene2index))
+        if k === nothing
+            # Remove scene from map
+            index = pop!(tree.scene2index, WeakRef(scene))
+
+            # Clean up RenderObjects (maps should get deleted with GLScene struct)
+            foreach(destroy!, tree.scenes[index])
+
+            # Remove GLScene (TODO: rendering parameters should not need cleanup? )
+            deleteat!(tree.scenes, index)
+            deleteat!(tree.depth, index)
+
+            # Update mapping
+            for (k, v) in tree.scene2index
+                if v > index
+                    tree.scene2index[k] -= 1
+                end
+            end
+        end
+    end
+end
+
+Base.haskey(tree::GLSceneTree, scene::Scene) = haskey(tree.scene2index, WeakRef(scene))
+Base.getindex(tree::GLSceneTree, scene::Scene) = tree.scenes[tree.scene2index[WeakRef(scene)]]
+Base.isempty(tree::GLSceneTree) = isempty(tree.scenes)
+
+function Base.show(io::IO, tree::GLSceneTree)
+    for i in eachindex(tree.scenes)
+        println(io, "  " ^ tree.depth[i], "GLScene(", length(tree.scenes[i].renderobjects), ")")
+    end
+end
+
+function insert_scene!(tree::GLSceneTree, scene::Scene, parent::Nothing, index::Integer)
+    # a root scene can only be added if the screen does not already have a root scene
+    @debug "Inserting root scene $(objectid(scene))"
+    if isempty(tree.scenes)
+        tree.scene2index[WeakRef(scene)] = 1
+        push!(tree.scenes, GLScene(scene))
+        push!(tree.depth, 1)
+    else
+        error("Cannot insert a root scene into a tree that already contains one.")
+    end
+
+    return
+end
+
+function insert_scene!(tree::GLSceneTree, scene::Scene, parent::Scene, index::Integer)
+    @debug "Inserting scene $(objectid(scene))"
+    if isempty(tree.scenes) # allow non-root scenes to act as root scenes
+        tree.scene2index[WeakRef(scene)] = 1
+        push!(tree.scenes, GLScene(scene))
+        push!(tree.depth, 1)
+        return
+    elseif !haskey(tree.scene2index, WeakRef(parent))
+        error("Cannot add a scene whose parent is not part of the displayed scene tree.")
+    end
+
+    @assert !isempty(tree.scenes) "An empty scene tree should not be reachable here."
+
+    # Figure out where the scene should be inserted
+    parent_index = tree.scene2index[WeakRef(parent)]
+    insert_index = parent_index
+    # @debug insert_index
+    while (index > 0) && (insert_index < length(tree.scenes))
+        # @debug "loop"
+        insert_index += 1
+        if tree.depth[insert_index] == tree.depth[parent_index] + 1
+            # found a child of parent
+            index -= 1
+        elseif tree.depth[insert_index] == tree.depth[parent_index]
+            # found a sibling of parent
+            # we can insert here but no further down
+            if index != 1
+                error("Cannot insert scene because other children of its parent are missing.")
+            end
+            index -= 1
+            break
+        end
+    end
+    # @debug insert_index, tree.depth[insert_index], tree.depth[parent_index]
+    if index == 1 && insert_index == length(tree.scenes)
+        insert_index += 1
+    elseif index != 0
+        error("Failed to find scene insertion index.")
+    end
+    # @debug insert_index
+
+    tree.scene2index[WeakRef(scene)] = insert_index
+    insert!(tree.scenes, insert_index, GLScene(scene))
+    insert!(tree.depth,  insert_index, tree.depth[parent_index] + 1)
+
+    return
+end
+
+function delete_scene!(screen, tree::GLSceneTree, scene::Scene)
+    @debug "Deleting scene $(objectid(scene))"
+    if haskey(tree.scene2index, WeakRef(scene))
+        # Delete all child scenes
+        for child in scene.children
+            delete_scene!(screen, tree, child)
+        end
+
+        # Remove scene from map
+        index = pop!(tree.scene2index, WeakRef(scene))
+
+        # Clean up RenderObjects (maps should get deleted with GLScene struct)
+        glscene = tree.scenes[index]
+        foreach(glscene.renderobjects) do robj
+            delete!(tree.robj2plot, robj.id)
+            destroy!(robj)
+        end
+
+        # Remove GLScene (TODO: rendering parameters should not need cleanup? )
+        deleteat!(tree.scenes, index)
+        deleteat!(tree.depth, index)
+
+        # Update mapping
+        for (k, v) in tree.scene2index
+            if v > index
+                tree.scene2index[k] -= 1
+            end
+        end
+
+        # Remove screen from scene to avoid double-deletion
+        filter!(x -> x !== screen, scene.current_screens)
+    else
+        @debug("Cannot delete scene from tree - does not exist in tree.")
+    end
+    return
+end
+
+function delete_plot!(tree::GLSceneTree, scene::Scene, plot::AbstractPlot)
+    @debug("Deleting plot $(objectid(plot))")
+
+    if haskey(tree, scene)
+        glscene = tree[scene]
+        for atomic in Makie.collect_atomic_plots(plot)
+            if haskey(glscene.plot2robj, objectid(atomic))
+                @debug "deleting atomic $(objectid(atomic))"
+                robj = pop!(glscene.plot2robj, objectid(atomic))
+
+                delete!(tree.robj2plot, robj.id)
+                delete!(glscene.robj2plot, robj.id)
+
+                filter!(x -> x !== robj, glscene.renderobjects)
+                destroy!(robj)
+            end
+        end
+    else
+        @debug("Cannot delete plot if its parent scene is not being shown.")
+    end
+    return
+end

GLMakie/src/drawing_primitives.jl

@@ -231,7 +231,12 @@ const EXCLUDE_KEYS = Set([:transformation, :tickranges, :ticklabels, :raw, :SSAO
 function cached_robj!(robj_func, screen, scene, plot::AbstractPlot)
     # poll inside functions to make wait on compile less prominent
     pollevents(screen, Makie.BackendTick)
-    robj = get!(screen.cache, objectid(plot)) do
+
+    @assert haskey(screen.scene_tree, scene)
+    glscene = screen.scene_tree[scene]
+
+
+    robj = get!(glscene.plot2robj, objectid(plot)) do
 
         filtered = filter(plot.attributes) do (k, v)
             return !in(k, EXCLUDE_KEYS)
@@ -336,21 +341,28 @@ function cached_robj!(robj_func, screen, scene, plot::AbstractPlot)
         robj = robj_func(gl_attributes)
 
         get!(gl_attributes, :ssao, Observable(false))
-        screen.cache2plot[robj.id] = plot
+
+        glscene.robj2plot[robj.id] = plot
+        screen.scene_tree.robj2plot[robj.id] = plot
+
         return robj
     end
-    push!(screen, scene, robj)
+
+    push!(glscene.renderobjects, robj)
+    @debug "Inserting robj $(robj.id) / atomic $(objectid(plot))"
+
     return robj
 end
 
 function Base.insert!(screen::Screen, scene::Scene, @nospecialize(x::Plot))
     ShaderAbstractions.switch_context!(screen.glscreen)
-    add_scene!(screen, scene)
     # poll inside functions to make wait on compile less prominent
     pollevents(screen, Makie.BackendTick)
     if isempty(x.plots) # if no plots inserted, this truly is an atomic
         draw_atomic(screen, scene, x)
+
     else
+        @debug "Inserting plot $(objectid(x))"
         foreach(x.plots) do x
             # poll inside functions to make wait on compile less prominent
             pollevents(screen, Makie.BackendTick)

GLMakie/src/gl_backend.jl

@@ -62,6 +62,7 @@ end
 
 include("glwindow.jl")
 include("postprocessing.jl")
+include("GLScene.jl")
 include("screen.jl")
 include("glshaders/visualize_interface.jl")
 include("glshaders/lines.jl")