Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Direct communication link between user JS and JL with WebIO support #991

Closed
wants to merge 22 commits into from
Closed
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions frontend/components/Editor.js
Original file line number Diff line number Diff line change
Expand Up @@ -549,11 +549,54 @@ patch: ${JSON.stringify(
}
})

// If you are a happy notebook maker/developer,
// and you see these __pluto_integrations_handlers__ and you're like WOW!
// Let me write an integration with other code!! Please don't. Sure, try it out as you wish,
// but I will 100% change this name and structure, so please come to the Zulip chat and connect with us.
// @ts-ignore
window.__pluto_integrations_handlers__ = {}
// @ts-ignore
window.__pluto_integrations_send__ = (module, message) => {
this.client.send(
"integrations",
{
module_name: module,
body: message,
},
{ notebook_id: this.state.notebook.notebook_id },
false
)
}

// @ts-ignore
let WebIO = window.webio.default
const webIO = new WebIO()
// @ts-ignore
window.WebIO = webIO
webIO.setSendCallback((message) => {
// @ts-ignore
window.__pluto_integrations_send__("WebIO", message)
})

// @ts-ignore
window.__pluto_integrations_handlers__["WebIO"] = (message) => {
webIO.dispatch(message.body)
}

// these are update message that are _not_ a response to a `send(*, *, {create_promise: true})`
const on_update = (update, by_me) => {
if (this.state.notebook.notebook_id === update.notebook_id) {
const message = update.message
switch (update.type) {
case "integrations":
// @ts-ignore
let handler = window.__pluto_integrations_handlers__[message.module_name]
if (handler != null) {
handler(message)
} else {
console.warn(`Unknown integrations message "${message.module_name}"`)
}
break
case "notebook_diff":
if (message?.response?.from_reset) {
console.log("Trying to reset state after failure")
Expand Down
1 change: 1 addition & 0 deletions frontend/editor.html
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

<script src="https://cdn.jsdelivr.net/npm/[email protected]/lodash.min.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/@observablehq/[email protected]/dist/stdlib.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/@webio/[email protected]/dist/webio.bundle.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/js/iframeResizer.min.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/lib/codemirror.min.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/mode/julia/julia.min.js" defer></script>
Expand Down
24 changes: 24 additions & 0 deletions src/evaluation/WorkspaceManager.jl
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,22 @@ function make_workspace((session, notebook)::SN; force_offline::Bool=false)::Wor
pid
end

Distributed.remotecall_eval(Main, [pid], quote
Main.PlutoRunner.workspace_info.notebook_id = $(string(notebook.notebook_id))
end)

integrations_channel = Core.eval(Main, quote
$(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.IntegrationsWithOtherPackages.message_channel)), $pid)
end)

log_channel = Core.eval(Main, quote
$(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.log_channel)), $pid)
end)
module_name = create_emptyworkspacemodule(pid)
workspace = Workspace(pid, log_channel, module_name, Token())

@async start_relaying_logs((session, notebook), log_channel)
@async start_relaying_integrations((session, notebook), integrations_channel)
cd_workspace(workspace, notebook.path)

force_offline || (notebook.process_status = ProcessStatus.ready)
Expand All @@ -84,6 +93,21 @@ function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.R
end
end

function start_relaying_integrations((session, notebook)::SN, channel::Distributed.RemoteChannel)
while true
try
next_message = take!(channel)
putnotebookupdates!(session, notebook, UpdateMessage(:integrations, next_message, notebook))
catch e
if !isopen(channel)
break
end
@error "Failed to relay integrations message" exception=(e, catch_backtrace())
end
end
end


"Call `cd(\$path)` inside the workspace. This is done when creating a workspace, and whenever the notebook changes path."
function cd_workspace(workspace, path::AbstractString)
eval_in_workspace(workspace, quote
Expand Down
143 changes: 143 additions & 0 deletions src/runner/IntegrationsWithOtherPackages.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
module IntegrationsWithOtherPackages

import ..workspace_info
include("./Requires.jl/src/Requires.jl")

"Attempts to find the MIME pair corresponding to the extension of a filename. Defaults to `text/plain`."
function mime_fromfilename(filename)
# This bad boy is from: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
mimepairs = Dict(".aac" => "audio/aac", ".bin" => "application/octet-stream", ".bmp" => "image/bmp", ".css" => "text/css", ".csv" => "text/csv", ".eot" => "application/vnd.ms-fontobject", ".gz" => "application/gzip", ".gif" => "image/gif", ".htm" => "text/html", ".html" => "text/html", ".ico" => "image/vnd.microsoft.icon", ".jpeg" => "image/jpeg", ".jpg" => "image/jpeg", ".js" => "text/javascript", ".json" => "application/json", ".jsonld" => "application/ld+json", ".mjs" => "text/javascript", ".mp3" => "audio/mpeg", ".mp4" => "video/mp4", ".mpeg" => "video/mpeg", ".oga" => "audio/ogg", ".ogv" => "video/ogg", ".ogx" => "application/ogg", ".opus" => "audio/opus", ".otf" => "font/otf", ".png" => "image/png", ".pdf" => "application/pdf", ".rtf" => "application/rtf", ".sh" => "application/x-sh", ".svg" => "image/svg+xml", ".tar" => "application/x-tar", ".tif" => "image/tiff", ".tiff" => "image/tiff", ".ttf" => "font/ttf", ".txt" => "text/plain", ".wav" => "audio/wav", ".weba" => "audio/webm", ".webm" => "video/webm", ".webp" => "image/webp", ".woff" => "font/woff", ".woff2" => "font/woff2", ".xhtml" => "application/xhtml+xml", ".xml" => "application/xml", ".xul" => "application/vnd.mozilla.xul+xml", ".zip" => "application/zip")
file_extension = getkey(mimepairs, '.' * split(filename, '.')[end], ".txt")
MIME(mimepairs[file_extension])
end

export handle_websocket_message, message_channel
"Called directly (through Distributed) from the main Pluto process"
function handle_websocket_message(message)
try
result = on_websocket_message(Val(Symbol(message[:module_name])), message[:body])
if result !== nothing
@warn "Integrations `on_websocket_message($(Symbol(message[:module_name])))` returned a value, but is expected to return `nothing`"
end
catch ex
bt = stacktrace(catch_backtrace())
@error "Dispatching integrations websocket message failed:" message=message exception=(ex, bt)
end
nothing
end
const message_channel = Channel{Any}(10)

"""
Do not call yourself! Or do, but don't say I didn't warn you.
Integrations should implement this to capture incoming websocket messages.
I force returning nothing, because returning might give you the idea that
fonsp marked this conversation as resolved.
Show resolved Hide resolved
the result is sent back to the client, which (right now) it isn't.
If you want to send something back to the client, use `IntegrationsWithOtherPackages.message_channel`

function on_websocket_message(::Val{:MyModule}, body)::Nothing
# ...
end
"""
function on_websocket_message(module_name, body)::Nothing
error("No websocket message handler defined for '$(module_name)'")
end

export handle_request
function handle_request(request)
try
on_request(Val(Symbol(request[:module_name])), request)
catch ex
bt = stacktrace(catch_backtrace())
@error "Dispatching integrations HTTP request failed:" request=request exception=(ex, bt)
end
end

"""
Do not call yourself! Or do, but don't say I didn't warn you.
Integrations should implement this to capture incoming http requests.
Expects to result in a Dict with at least `:status => Int`, but could include more,
as seen in the following example

function on_request(::Val{:MyModule}, request)
Dict(
:status => 200,
:headers => ["Content-Type" => "text/html"],
:body => "<html>Hi</html>",
)
end
"""
function on_request(module_name, request)::Dict{Symbol,<:Any}
error("No http request handler defined for '$(module_name)'")
end

"Still a bit unsure about this, but it will for now give you the (relative) path for your module requests"
function get_base_url(module_name::Union{AbstractString, Symbol})
"/integrations/$(string(module_name))/$(workspace_info.notebook_id)"
Copy link
Owner

@fonsp fonsp Mar 16, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we flip these two?

/integrations/$(workspace_info.notebook_id)/$(string(module_name))

end

module AssetRegistryIntegrations
import ..Requires
import ..mime_fromfilename
import ..workspace_info
import ..get_base_url
import ..on_request

function __init__()
Requires.@require AssetRegistry="bf4720bc-e11a-5d0c-854e-bdca1663c893" begin
if workspace_info.notebook_id === nothing
throw(error("Couldn't load AssetRegistry integrations, notebook_id not set inside PlutoRunner"))
end

AssetRegistry.baseurl[] = get_base_url(:AssetRegistry)

function on_request(::Val{:AssetRegistry}, request)
# local full_path = AssetRegistry.baseurl[] * "/" * request[:target]
local full_path = request[:target]
if haskey(AssetRegistry.registry, full_path)
local file_path = AssetRegistry.registry[full_path]
if isfile(file_path)
Dict(
:status => 200,
:headers => ["Content-Type" => mime_fromfilename(file_path)],
:body => read(file_path),
)
else
Dict(:status => 404)
end
else
Dict(:status => 404)
end
end
end
end
end


module WebIOIntegrations
import ..Requires
import ..on_websocket_message
import ..message_channel

function __init__()
Requires.@require WebIO="0f1e0344-ec1d-5b48-a673-e5cf874b6c29" begin
import Sockets
import .WebIO

struct WebIOConnection <: WebIO.AbstractConnection end
Sockets.send(::WebIOConnection, data) = begin
put!(message_channel, Dict(
:module_name => "WebIO",
:body => data,
))
end
Base.isopen(::WebIOConnection) = Base.isopen(message_channel)

function on_websocket_message(::Val{:WebIO}, body)
WebIO.dispatch(WebIOConnection(), body)
nothing
end
end
end
end

end
8 changes: 8 additions & 0 deletions src/runner/PlutoRunner.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1201,6 +1201,14 @@ function Logging.handle_message(::PlutoLogger, level, msg, _module, group, id, f
end
end


Base.@kwdef mutable struct WorkspaceInfo
notebook_id::Union{String,Nothing} = nothing
end
const workspace_info = WorkspaceInfo()

include("./IntegrationsWithOtherPackages.jl")

# we put this in __init__ to fix a world age problem
function __init__()
if Distributed.myid() != 1
Expand Down
22 changes: 22 additions & 0 deletions src/runner/Requires.jl/LICENSE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
The Requires.jl package is licensed under the MIT "Expat" License:

> Copyright (c) 2014: Mike Innes, Julia Computing & contributors.
>
> Permission is hereby granted, free of charge, to any person obtaining
> a copy of this software and associated documentation files (the
> "Software"), to deal in the Software without restriction, including
> without limitation the rights to use, copy, modify, merge, publish,
> distribute, sublicense, and/or sell copies of the Software, and to
> permit persons to whom the Software is furnished to do so, subject to
> the following conditions:
>
> The above copyright notice and this permission notice shall be
> included in all copies or substantial portions of the Software.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
5 changes: 5 additions & 0 deletions src/runner/Requires.jl/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Because we don't want to pin the version number for any package used in PlutoRunner,
we just copy this whole thing here.

All credits to
https://github.com/JuliaPackaging/Requires.jl
71 changes: 71 additions & 0 deletions src/runner/Requires.jl/src/Requires.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
module Requires

if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@compiler_options"))
@eval Base.Experimental.@compiler_options compile=min optimize=0 infer=false
end

using UUIDs

function _include_path(relpath::String)
# Reproduces include()'s runtime relative path logic
# See Base._include_dependency()
prev = Base.source_path(nothing)
if prev === nothing
path = abspath(relpath)
else
path = normpath(joinpath(dirname(prev), relpath))
end
end

"""
@include("somefile.jl")

Behaves like `include`, but caches the target file content at macro expansion
time, and uses this as a fallback when the file doesn't exist at runtime. This
is useful when compiling a sysimg. The argument `"somefile.jl"` must be a
string literal, not an expression.

`@require` blocks insert this automatically when you use `include`.
"""
macro include(relpath::String)
compiletime_path = joinpath(dirname(String(__source__.file)), relpath)
s = String(read(compiletime_path))
quote
# NB: Runtime include path may differ from the compile-time macro
# expansion path if the source has been relocated.
runtime_path = _include_path($relpath)
if isfile(runtime_path)
# NB: For Revise compatibility, include($relpath) needs to be
# emitted where $relpath is a string *literal*.
$(esc(:(include($relpath))))
else
include_string($__module__, $s, $relpath)
end
end
end

include("init.jl")
include("require.jl")

function __init__()
push!(package_callbacks, loadpkg)
end

if isprecompiling()
precompile(loadpkg, (Base.PkgId,)) || @warn "Requires failed to precompile `loadpkg`"
precompile(withpath, (Any, String)) || @warn "Requires failed to precompile `withpath`"
precompile(err, (Any, Module, String)) || @warn "Requires failed to precompile `err`"
precompile(parsepkg, (Expr,)) || @warn "Requires failed to precompile `parsepkg`"
precompile(listenpkg, (Any, Base.PkgId)) || @warn "Requires failed to precompile `listenpkg`"
precompile(callbacks, (Base.PkgId,)) || @warn "Requires failed to precompile `callbacks`"
precompile(withnotifications, (String, Module, String, String, Expr)) || @warn "Requires failed to precompile `withnotifications`"
precompile(replace_include, (Expr, LineNumberNode)) || @warn "Requires failed to precompile `replace_include`"
precompile(getfield(Requires, Symbol("@require")), (LineNumberNode, Module, Expr, Any)) || @warn "Requires failed to precompile `@require`"

precompile(_include_path, (String,)) || @warn "Requires failed to precompile `_include_path`"
precompile(getfield(Requires, Symbol("@include")), (LineNumberNode, Module, String)) || @warn "Requires failed to precompile `@include`"

precompile(__init__, ()) || @warn "Requires failed to precompile `__init__`"
end

end # module
24 changes: 24 additions & 0 deletions src/runner/Requires.jl/src/init.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
export @init

function initm(ex)
quote
if !@isdefined __inits__
const __inits__ = []
end
if !@isdefined __init__
__init__() = @init
end
push!(__inits__, () -> $ex)
nothing
end |> esc
end

function initm()
:(for f in __inits__
f()
end) |> esc
end

macro init(args...)
initm(args...)
end
Loading