Direct communication link between user JS and JL with WebIO support

frontend/components/Editor.js

@@ -549,11 +549,54 @@ patch: ${JSON.stringify(
                 }
             })
 
+        // If you are a happy notebook maker/developer,
+        // and you see these __pluto_integrations_handlers__ and you're like WOW!
+        // Let me write an integration with other code!! Please don't. Sure, try it out as you wish,
+        // but I will 100% change this name and structure, so please come to the Zulip chat and connect with us.
+        // @ts-ignore
+        window.__pluto_integrations_handlers__ = {}
+        // @ts-ignore
+        window.__pluto_integrations_send__ = (module, message) => {
+            this.client.send(
+                "integrations",
+                {
+                    module_name: module,
+                    body: message,
+                },
+                { notebook_id: this.state.notebook.notebook_id },
+                false
+            )
+        }
+
+        // @ts-ignore
+        let WebIO = window.webio.default
+        const webIO = new WebIO()
+        // @ts-ignore
+        window.WebIO = webIO
+        webIO.setSendCallback((message) => {
+            // @ts-ignore
+            window.__pluto_integrations_send__("WebIO", message)
+        })
+
+        // @ts-ignore
+        window.__pluto_integrations_handlers__["WebIO"] = (message) => {
+            webIO.dispatch(message.body)
+        }
+
         // these are update message that are _not_ a response to a `send(*, *, {create_promise: true})`
         const on_update = (update, by_me) => {
             if (this.state.notebook.notebook_id === update.notebook_id) {
                 const message = update.message
                 switch (update.type) {
+                    case "integrations":
+                        // @ts-ignore
+                        let handler = window.__pluto_integrations_handlers__[message.module_name]
+                        if (handler != null) {
+                            handler(message)
+                        } else {
+                            console.warn(`Unknown integrations message "${message.module_name}"`)
+                        }
+                        break
                     case "notebook_diff":
                         if (message?.response?.from_reset) {
                             console.log("Trying to reset state after failure")

frontend/editor.html

@@ -20,6 +20,7 @@
 
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/lodash.min.js" defer></script>
     <script src="https://cdn.jsdelivr.net/npm/@observablehq/[email protected]/dist/stdlib.js" defer></script>
+    <script src="https://cdn.jsdelivr.net/npm/@webio/[email protected]/dist/webio.bundle.js" defer></script>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/js/iframeResizer.min.js" defer></script>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/codemirror.min.js" defer></script>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/mode/julia/julia.min.js" defer></script>

src/evaluation/WorkspaceManager.jl

@@ -56,13 +56,22 @@ function make_workspace((session, notebook)::SN; force_offline::Bool=false)::Wor
         pid
     end
 
+    Distributed.remotecall_eval(Main, [pid], quote 
+        Main.PlutoRunner.workspace_info.notebook_id = $(string(notebook.notebook_id))
+    end)
+
+    integrations_channel = Core.eval(Main, quote
+        $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.IntegrationsWithOtherPackages.message_channel)), $pid)
+    end)
+
     log_channel = Core.eval(Main, quote
         $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.log_channel)), $pid)
     end)
     module_name = create_emptyworkspacemodule(pid)
     workspace = Workspace(pid, log_channel, module_name, Token())
 
     @async start_relaying_logs((session, notebook), log_channel)
+    @async start_relaying_integrations((session, notebook), integrations_channel)
     cd_workspace(workspace, notebook.path)
 
     force_offline || (notebook.process_status = ProcessStatus.ready)
@@ -84,6 +93,21 @@ function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.R
     end
 end
 
+function start_relaying_integrations((session, notebook)::SN, channel::Distributed.RemoteChannel)
+    while true
+        try
+            next_message = take!(channel)
+            putnotebookupdates!(session, notebook, UpdateMessage(:integrations, next_message, notebook))
+        catch e
+            if !isopen(channel)
+                break
+            end
+            @error "Failed to relay integrations message" exception=(e, catch_backtrace())
+        end
+    end
+end
+
+
 "Call `cd(\$path)` inside the workspace. This is done when creating a workspace, and whenever the notebook changes path."
 function cd_workspace(workspace, path::AbstractString)
     eval_in_workspace(workspace, quote

src/runner/IntegrationsWithOtherPackages.jl

@@ -0,0 +1,143 @@
+module IntegrationsWithOtherPackages
+
+import ..workspace_info
+include("./Requires.jl/src/Requires.jl")
+
+"Attempts to find the MIME pair corresponding to the extension of a filename. Defaults to `text/plain`."
+function mime_fromfilename(filename)
+    # This bad boy is from: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+    mimepairs = Dict(".aac" => "audio/aac", ".bin" => "application/octet-stream", ".bmp" => "image/bmp", ".css" => "text/css", ".csv" => "text/csv", ".eot" => "application/vnd.ms-fontobject", ".gz" => "application/gzip", ".gif" => "image/gif", ".htm" => "text/html", ".html" => "text/html", ".ico" => "image/vnd.microsoft.icon", ".jpeg" => "image/jpeg", ".jpg" => "image/jpeg", ".js" => "text/javascript", ".json" => "application/json", ".jsonld" => "application/ld+json", ".mjs" => "text/javascript", ".mp3" => "audio/mpeg", ".mp4" => "video/mp4", ".mpeg" => "video/mpeg", ".oga" => "audio/ogg", ".ogv" => "video/ogg", ".ogx" => "application/ogg", ".opus" => "audio/opus", ".otf" => "font/otf", ".png" => "image/png", ".pdf" => "application/pdf", ".rtf" => "application/rtf", ".sh" => "application/x-sh", ".svg" => "image/svg+xml", ".tar" => "application/x-tar", ".tif" => "image/tiff", ".tiff" => "image/tiff", ".ttf" => "font/ttf", ".txt" => "text/plain", ".wav" => "audio/wav", ".weba" => "audio/webm", ".webm" => "video/webm", ".webp" => "image/webp", ".woff" => "font/woff", ".woff2" => "font/woff2", ".xhtml" => "application/xhtml+xml", ".xml" => "application/xml", ".xul" => "application/vnd.mozilla.xul+xml", ".zip" => "application/zip")
+    file_extension = getkey(mimepairs, '.' * split(filename, '.')[end], ".txt")
+    MIME(mimepairs[file_extension])
+end
+
+export handle_websocket_message, message_channel
+"Called directly (through Distributed) from the main Pluto process"
+function handle_websocket_message(message)
+    try
+        result = on_websocket_message(Val(Symbol(message[:module_name])), message[:body])
+        if result !== nothing
+            @warn "Integrations `on_websocket_message($(Symbol(message[:module_name])))` returned a value, but is expected to return `nothing`"
+        end
+    catch ex
+        bt = stacktrace(catch_backtrace())
+        @error "Dispatching integrations websocket message failed:" message=message exception=(ex, bt)
+    end
+    nothing
+end
+const message_channel = Channel{Any}(10)
+
+"""
+Do not call yourself! Or do, but don't say I didn't warn you.
+Integrations should implement this to capture incoming websocket messages.
+I force returning nothing, because returning might give you the idea that
+the result is sent back to the client, which (right now) it isn't.
+If you want to send something back to the client, use `IntegrationsWithOtherPackages.message_channel`
+
+    function on_websocket_message(::Val{:MyModule}, body)::Nothing
+        # ...
+    end
+"""
+function on_websocket_message(module_name, body)::Nothing
+    throw(error("No websocket message handler defined for '$(module_name)'"))
+end
+
+export handle_request
+function handle_request(request)
+    try
+        on_request(Val(Symbol(request[:module_name])), request)
+    catch ex
+        bt = stacktrace(catch_backtrace())
+        @error "Dispatching integrations HTTP request failed:" request=request exception=(ex, bt)
+    end
+end
+
+"""
+Do not call yourself! Or do, but don't say I didn't warn you.
+Integrations should implement this to capture incoming http requests.
+Expects to result in a Dict with at least `:status => Int`, but could include more,
+as seen in the following example
+
+    function on_request(::Val{:MyModule}, request)
+        Dict(
+            :status => 200,
+            :headers => ["Content-Type" => "text/html"],
+            :body => "<html>Hi</html>",
+        )
+    end
+"""
+function on_request(module_name, request)::Dict{Symbol,<:Any}
+    throw(error("No http request handler defined for '$(module_name)'"))
+end
+
+"Still a bit unsure about this, but it will for now give you the (relative) path for your module requests"
+function get_base_url(module_name::Union{AbstractString, Symbol})
+    "/integrations/$(string(module_name))/$(workspace_info.notebook_id)"
+end
+
+module AssetRegistryIntegrations
+    import ..Requires
+    import ..mime_fromfilename
+    import ..workspace_info
+    import ..get_base_url
+    import ..on_request
+
+    function __init__()
+        Requires.@require AssetRegistry="bf4720bc-e11a-5d0c-854e-bdca1663c893" begin
+            if workspace_info.notebook_id === nothing
+                throw(error("Couldn't load AssetRegistry integrations, notebook_id not set inside PlutoRunner"))
+            end
+
+            AssetRegistry.baseurl[] = get_base_url(:AssetRegistry)
+
+            function on_request(::Val{:AssetRegistry}, request)
+                # local full_path = AssetRegistry.baseurl[] * "/" * request[:target]
+                local full_path = request[:target]
+                if haskey(AssetRegistry.registry, full_path)
+                    local file_path = AssetRegistry.registry[full_path]
+                    if isfile(file_path)
+                        Dict(
+                            :status => 200,
+                            :headers => ["Content-Type" => mime_fromfilename(file_path)],
+                            :body => read(file_path),
+                        )
+                    else
+                        Dict(:status => 404)
+                    end
+                else
+                    Dict(:status => 404)
+                end
+            end
+        end
+    end
+end
+
+
+module WebIOIntegrations
+    import ..Requires
+    import ..on_websocket_message
+    import ..message_channel
+
+    function __init__()
+        Requires.@require WebIO="0f1e0344-ec1d-5b48-a673-e5cf874b6c29" begin
+            import Sockets
+            import .WebIO
+
+            struct WebIOConnection <: WebIO.AbstractConnection end
+            Sockets.send(::WebIOConnection, data) = begin
+                put!(message_channel, Dict(
+                    :module_name => "WebIO",
+                    :body => data,
+                ))
+            end
+            Base.isopen(::WebIOConnection) = Base.isopen(message_channel)
+
+            function on_websocket_message(::Val{:WebIO}, body)
+                WebIO.dispatch(WebIOConnection(), body)
+                nothing
+            end
+        end
+    end
+end
+
+end

src/runner/PlutoRunner.jl

@@ -1201,6 +1201,14 @@ function Logging.handle_message(::PlutoLogger, level, msg, _module, group, id, f
     end
 end
 
+
+Base.@kwdef mutable struct WorkspaceInfo
+    notebook_id::Union{String,Nothing} = nothing
+end
+const workspace_info = WorkspaceInfo()
+
+include("./IntegrationsWithOtherPackages.jl")
+
 # we put this in __init__ to fix a world age problem
 function __init__()
     if Distributed.myid() != 1

src/runner/Requires.jl/LICENSE.md

@@ -0,0 +1,22 @@
+The Requires.jl package is licensed under the MIT "Expat" License:
+
+> Copyright (c) 2014: Mike Innes, Julia Computing & contributors.
+>
+> Permission is hereby granted, free of charge, to any person obtaining
+> a copy of this software and associated documentation files (the
+> "Software"), to deal in the Software without restriction, including
+> without limitation the rights to use, copy, modify, merge, publish,
+> distribute, sublicense, and/or sell copies of the Software, and to
+> permit persons to whom the Software is furnished to do so, subject to
+> the following conditions:
+>
+> The above copyright notice and this permission notice shall be
+> included in all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/runner/Requires.jl/README.md

@@ -0,0 +1,5 @@
+Because we don't want to pin the version number for any package used in PlutoRunner,
+we just copy this whole thing here.
+
+All credits to
+https://github.com/JuliaPackaging/Requires.jl

src/runner/Requires.jl/src/Requires.jl

@@ -0,0 +1,71 @@
+module Requires
+
+if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@compiler_options"))
+    @eval Base.Experimental.@compiler_options compile=min optimize=0 infer=false
+end
+
+using UUIDs
+
+function _include_path(relpath::String)
+    # Reproduces include()'s runtime relative path logic
+    # See Base._include_dependency()
+    prev = Base.source_path(nothing)
+    if prev === nothing
+        path = abspath(relpath)
+    else
+        path = normpath(joinpath(dirname(prev), relpath))
+    end
+end
+
+"""
+    @include("somefile.jl")
+
+Behaves like `include`, but caches the target file content at macro expansion
+time, and uses this as a fallback when the file doesn't exist at runtime. This
+is useful when compiling a sysimg. The argument `"somefile.jl"` must be a
+string literal, not an expression.
+
+`@require` blocks insert this automatically when you use `include`.
+"""
+macro include(relpath::String)
+    compiletime_path = joinpath(dirname(String(__source__.file)), relpath)
+    s = String(read(compiletime_path))
+    quote
+        # NB: Runtime include path may differ from the compile-time macro
+        # expansion path if the source has been relocated.
+        runtime_path = _include_path($relpath)
+        if isfile(runtime_path)
+            # NB: For Revise compatibility, include($relpath) needs to be
+            # emitted where $relpath is a string *literal*.
+            $(esc(:(include($relpath))))
+        else
+            include_string($__module__, $s, $relpath)
+        end
+    end
+end
+
+include("init.jl")
+include("require.jl")
+
+function __init__()
+    push!(package_callbacks, loadpkg)
+end
+
+if isprecompiling()
+    precompile(loadpkg, (Base.PkgId,)) || @warn "Requires failed to precompile `loadpkg`"
+    precompile(withpath, (Any, String)) || @warn "Requires failed to precompile `withpath`"
+    precompile(err, (Any, Module, String)) || @warn "Requires failed to precompile `err`"
+    precompile(parsepkg, (Expr,)) || @warn "Requires failed to precompile `parsepkg`"
+    precompile(listenpkg, (Any, Base.PkgId)) || @warn "Requires failed to precompile `listenpkg`"
+    precompile(callbacks, (Base.PkgId,)) || @warn "Requires failed to precompile `callbacks`"
+    precompile(withnotifications, (String, Module, String, String, Expr)) || @warn "Requires failed to precompile `withnotifications`"
+    precompile(replace_include, (Expr, LineNumberNode)) || @warn "Requires failed to precompile `replace_include`"
+    precompile(getfield(Requires, Symbol("@require")), (LineNumberNode, Module, Expr, Any)) || @warn "Requires failed to precompile `@require`"
+
+    precompile(_include_path, (String,)) || @warn "Requires failed to precompile `_include_path`"
+    precompile(getfield(Requires, Symbol("@include")), (LineNumberNode, Module, String)) || @warn "Requires failed to precompile `@include`"
+
+    precompile(__init__, ()) || @warn "Requires failed to precompile `__init__`"
+end
+
+end # module

src/runner/Requires.jl/src/init.jl

@@ -0,0 +1,24 @@
+export @init
+
+function initm(ex)
+  quote
+    if !@isdefined __inits__
+      const __inits__ = []
+    end
+    if !@isdefined __init__
+      __init__() = @init
+    end
+    push!(__inits__, () -> $ex)
+    nothing
+  end |> esc
+end
+
+function initm()
+  :(for f in __inits__
+      f()
+    end) |> esc
+end
+
+macro init(args...)
+  initm(args...)
+end