From 279bc25747f038d7110c99d1903f2a8f482148b4 Mon Sep 17 00:00:00 2001 From: Chris Andreae Date: Sat, 22 Jul 2023 14:55:49 +0900 Subject: [PATCH] Compile service: Support passing RHS kconfig and building RHS Specifying RHS kconfig will cause both firmware images to be built, which will take twice as long as a LHS-only build. Clients should take care to only request a RHS build for kconfig options that affect the peripheral behaviour. --- .github/workflows/build-container.yml | 4 +- lambda/Gemfile | 4 - lambda/Gemfile.lock | 23 ------ lambda/app.rb | 77 +++++++++---------- lambda/compiler.rb | 35 +++++++-- lambda/gemset.nix | 76 ------------------- lambda/web_app.rb | 43 ----------- release.nix | 105 +++++++++++++++++--------- 8 files changed, 137 insertions(+), 230 deletions(-) delete mode 100644 lambda/web_app.rb diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index f460ffe9065..7062812693f 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -60,11 +60,11 @@ jobs: name: moergo-glove80-zmk-dev authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" - name: Build lambda image - run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A directLambdaImage -o directLambdaImage + run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A lambdaImage -o lambdaImage - name: Import OCI image into docker-daemon env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: skopeo --insecure-policy copy oci:directLambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$REVISION_TAG + run: skopeo --insecure-policy copy oci:lambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$REVISION_TAG - name: Push container image to Amazon ECR env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} diff --git a/lambda/Gemfile b/lambda/Gemfile index dd1dca602e2..6b6bbf753ec 100644 --- a/lambda/Gemfile +++ b/lambda/Gemfile @@ -1,7 +1,3 @@ source 'https://rubygems.org' gem 'aws_lambda_ric' -gem 'rack' -gem 'sinatra', '~> 2' -# The version on rubygems (1.0.7) is very out of date -gem 'serverless-rack', git: 'https://github.com/logandk/serverless-rack', branch: '7364305bc' diff --git a/lambda/Gemfile.lock b/lambda/Gemfile.lock index 0c2f9299aee..8b6c1f95c58 100644 --- a/lambda/Gemfile.lock +++ b/lambda/Gemfile.lock @@ -1,36 +1,13 @@ -GIT - remote: https://github.com/logandk/serverless-rack - revision: 7364305bcbbf7f6cc6851497069a5a4cb91936b1 - branch: 7364305bc - specs: - serverless-rack (1.0.7) - rack (~> 2.0) - GEM remote: https://rubygems.org/ specs: aws_lambda_ric (2.0.0) - mustermann (2.0.2) - ruby2_keywords (~> 0.0.1) - rack (2.2.4) - rack-protection (2.2.2) - rack - ruby2_keywords (0.0.5) - sinatra (2.2.2) - mustermann (~> 2.0) - rack (~> 2.2) - rack-protection (= 2.2.2) - tilt (~> 2.0) - tilt (2.0.11) PLATFORMS ruby DEPENDENCIES aws_lambda_ric - rack - serverless-rack! - sinatra (~> 2) BUNDLED WITH 2.1.4 diff --git a/lambda/app.rb b/lambda/app.rb index 05194be38f2..8c3eb554cca 100644 --- a/lambda/app.rb +++ b/lambda/app.rb @@ -1,54 +1,42 @@ # frozen_string_literal: true -require 'rack' -require 'serverless_rack' - -require './web_app' +require 'stringio' +require 'digest' +require 'json' require './compiler' -$app = Rack::Builder.new do - run WebApp -end.to_app - module LambdaFunction - # Handle a API Gateway/ALB-structured HTTP request using the Sinatra app - class HttpHandler - def self.process(event:, context:) - handle_request(app: $app, event: event, context: context) - end - end - - # Handle a non-HTTP proxied request, returning either the compiled result or - # an error as JSON. - class DirectHandler + # Handle a non-HTTP compile request, returning a JSON body of either the + # compiled result or an error. + class Handler REVISION = ENV.fetch('REVISION', 'unknown') def self.process(event:, context:) return { type: 'keep_alive' } if event.has_key?('keep_alive') - keymap_data = event.fetch('keymap') do - return error(status: 400, message: 'Missing required argument: keymap') + parse_base64_param = ->(param, required: true) do + if event.include?(param) + Base64.strict_decode64(event.fetch(param)) + elsif required + return error(status: 400, message: "Missing required argument: #{param}") + end + rescue ArgumentError + return error(status: 400, message: "Invalid Base64 in #{param} input") end - keymap_data = - begin - Base64.strict_decode64(keymap_data) - rescue ArgumentError - return error(status: 400, message: 'Invalid Base64 in keymap input') - end + keymap_data = parse_base64_param.('keymap') + kconfig_data = parse_base64_param.('kconfig', required: false) - if event.has_key?('kconfig') - kconfig_data = - begin - Base64.strict_decode64(event['kconfig']) - rescue ArgumentError - return error(status: 400, message: 'Invalid Base64 in kconfig input') - end - end + # Including kconfig settings that affect the RHS require building both + # firmware images, doubling compile time. Clients should omit rhs_kconfig + # where possible. + rhs_kconfig_data = parse_base64_param.('rhs_kconfig', required: false) result, log = begin - Compiler.new.compile(keymap_data, kconfig_data) + log_compile(keymap_data, kconfig_data, rhs_kconfig_data) + + Compiler.new.compile(keymap_data, kconfig_data, rhs_kconfig_data) rescue Compiler::CompileError => e return error(status: e.status, message: e.message, detail: e.log) end @@ -57,11 +45,24 @@ def self.process(event:, context:) { type: 'result', result: result, log: log, revision: REVISION } rescue StandardError => e - error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message]) + error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message], exception: e) end - def self.error(status:, message:, detail: nil) - { type: 'error', status: status, message: message, detail: detail, revision: REVISION } + def self.log_compile(keymap_data, kconfig_data, rhs_kconfig_data) + keymap = Digest::SHA1.base64digest(keymap_data) + kconfig = kconfig_data ? Digest::SHA1.base64digest(kconfig_data) : 'nil' + rhs_kconfig = rhs_kconfig_data ? Digest::SHA1.base64digest(rhs_kconfig_data) : 'nil' + puts("Compiling with keymap: #{keymap}; kconfig: #{kconfig}; rhs_kconfig: #{rhs_kconfig}") + end + + def self.error(status:, message:, detail: nil, exception: nil) + reported_error = { type: 'error', status:, message:, detail:, revision: REVISION } + + exception_detail = { class: exception.class, backtrace: exception.backtrace } if exception + logged_error = reported_error.merge(exception: exception_detail) + puts(JSON.dump(logged_error)) + + reported_error end end end diff --git a/lambda/compiler.rb b/lambda/compiler.rb index 068a4dcd47e..1dba233afac 100644 --- a/lambda/compiler.rb +++ b/lambda/compiler.rb @@ -15,20 +15,41 @@ def initialize(message, status: 400, log:) end end - def compile(keymap_data, kconfig_data) + def compile(keymap_data, lhs_kconfig_data, rhs_kconfig_data) + if rhs_kconfig_data && !rhs_kconfig_data.empty? + lhs_result, lhs_output = compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: false) + rhs_result, rhs_output = compile_board('glove80_rh', keymap_data: nil, kconfig_data: rhs_kconfig_data, include_static_rhs: false) + [ + lhs_result.concat(rhs_result), + ["LHS Output:", *lhs_output, "RHS Output:", *rhs_output], + ] + else + compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: true) + end + end + + def compile_board(board, keymap_data:, kconfig_data:, include_static_rhs: false) in_build_dir do - compile_command = ['compileZmk', './build.keymap'] + compile_command = ['compileZmk', '-b', board] - File.open('build.keymap', 'w') { |io| io.write(keymap_data) } + if keymap_data + File.open('build.keymap', 'w') { |io| io.write(keymap_data) } + compile_command << '-k' << './build.keymap' + end if kconfig_data File.open('build.conf', 'w') { |io| io.write(kconfig_data) } - compile_command << './build.conf' + compile_command << '-c' << './build.conf' + end + + if include_static_rhs + # Concatenate the pre-compiled glove80_rh image to the resulting uf2 + compile_command << '-m' end compile_output = nil - IO.popen(compile_command, err: [:child, :out]) do |io| + IO.popen(compile_command, 'rb', err: [:child, :out]) do |io| compile_output = io.read end @@ -39,11 +60,11 @@ def compile(keymap_data, kconfig_data) raise CompileError.new("Compile failed with exit status #{status}", log: compile_output) end - unless File.exist?('zephyr/combined.uf2') + unless File.exist?('zmk.uf2') raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output) end - result = File.read('zephyr/combined.uf2') + result = File.read('zmk.uf2') [result, compile_output] end diff --git a/lambda/gemset.nix b/lambda/gemset.nix index 11b36ff804c..6b2fd1a0207 100644 --- a/lambda/gemset.nix +++ b/lambda/gemset.nix @@ -9,80 +9,4 @@ }; version = "2.0.0"; }; - mustermann = { - dependencies = ["ruby2_keywords"]; - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "0m70qz27mlv2rhk4j1li6pw797gmiwwqg02vcgxcxr1rq2v53rnb"; - type = "gem"; - }; - version = "2.0.2"; - }; - rack = { - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "0axc6w0rs4yj0pksfll1hjgw1k6a5q0xi2lckh91knfb72v348pa"; - type = "gem"; - }; - version = "2.2.4"; - }; - rack-protection = { - dependencies = ["rack"]; - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "169jzzgvbjrqmz4q55wp9pg4ji2h90mggcdxy152gv5vp96l2hgx"; - type = "gem"; - }; - version = "2.2.2"; - }; - ruby2_keywords = { - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "1vz322p8n39hz3b4a9gkmz9y7a5jaz41zrm2ywf31dvkqm03glgz"; - type = "gem"; - }; - version = "0.0.5"; - }; - serverless-rack = { - dependencies = ["rack"]; - groups = ["default"]; - platforms = []; - source = { - fetchSubmodules = false; - rev = "7364305bcbbf7f6cc6851497069a5a4cb91936b1"; - sha256 = "0c7ch0s0nl70p6ijg7q0jnq8ca2rhp5wqfp91kai81dy7d71mq65"; - type = "git"; - url = "https://github.com/logandk/serverless-rack"; - }; - version = "1.0.7"; - }; - sinatra = { - dependencies = ["mustermann" "rack" "rack-protection" "tilt"]; - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "0mbjp75dy35q796iard8izsy7gk55g2c3q864r2p13my3yjmlcvz"; - type = "gem"; - }; - version = "2.2.2"; - }; - tilt = { - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "186nfbcsk0l4l86gvng1fw6jq6p6s7rc0caxr23b3pnbfb20y63v"; - type = "gem"; - }; - version = "2.0.11"; - }; } diff --git a/lambda/web_app.rb b/lambda/web_app.rb deleted file mode 100644 index 2439babfcc6..00000000000 --- a/lambda/web_app.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require 'sinatra/base' -require './compiler' - -class WebApp < Sinatra::Base - set :environment, :production - set :show_exceptions, false - set :logging, nil - set :default_content_type, 'application/json' - - def json_body(hash) - body(hash.to_json) - end - - post '/api/compile' do - request.body.rewind - keymap_data = request.body.read - result, log = Compiler.new.compile(keymap_data, nil) - - status 200 - content_type 'application/octet-stream' - headers 'X-Debug-Output': log.to_json - body result - end - - error Compiler::CompileError do - e = env['sinatra.error'] - status e.status - json_body(error: e.message, detail: e.log) - end - - error do - e = env['sinatra.error'] - status 500 - json_body(error: "Unexpected error: #{e.class}", detail: [e.message]) - end - - not_found do - status 404 - json_body(error: 'No such path', detail: nil) - end -end diff --git a/release.nix b/release.nix index a9dfdaee91c..3ed1cb5ef8d 100644 --- a/release.nix +++ b/release.nix @@ -55,26 +55,56 @@ let in pkgs.writeShellScriptBin "compileZmk" '' set -eo pipefail - if [ ! -f "$1" ]; then - echo "Error: Missing keymap file" >&2 - echo "Usage: compileZmk file.keymap [file.conf]" >&2 - exit 1 - fi - - KEYMAP="$(${realpath_coreutils}/bin/realpath $1)" - - if [ -z "''${2+x}" ]; then - KCONFIG= - else - if [ ! -f "$2" ]; then - echo "Error: Missing kconfig file" >&2 - echo "Usage: compileZmk file.keymap [file.conf]" >&2 + function usage() { + echo "Usage: compileZmk [-m] [-k keymap_file] [-c kconfig_file] [-b board]" + } + + function checkPath() { + if [ -z "$1" ]; then + return 0 + elif [ ! -f "$1" ]; then + echo "Error: Missing $2 file" >&2 + usage >&2 exit 1 fi - KCONFIG="$(${realpath_coreutils}/bin/realpath $2)" + ${realpath_coreutils}/bin/realpath "$1" + } + + keymap="${zmk.src}/app/boards/arm/glove80/glove80.keymap" + kconfig="" + board="glove80_lh" + merge_rhs="" + + while getopts "hk:c:d:b:m" opt; do + case "$opt" in + h|\?) + usage >&2 + exit 1 + ;; + k) + keymap="$OPTARG" + ;; + c) + kconfig="$OPTARG" + ;; + b) + board="$OPTARG" + ;; + m) + merge_rhs=t + ;; + esac + done + + if [ "$board" = "glove80_rh" -a -n "$merge_rhs" ]; then + echo "Cannot merge static RHS with built RHS" >&2 + exit 2 fi + keymap="$(checkPath "$keymap" keymap)" + kconfig="$(checkPath "$kconfig" Kconfig)" + export PATH=${lib.makeBinPath (with pkgs; zmk'.nativeBuildInputs ++ [ ccache ])}:$PATH export CMAKE_PREFIX_PATH=${zephyr} @@ -84,13 +114,17 @@ let if [ -n "$DEBUG" ]; then ccache -z; fi - cmake -G Ninja -S ${zmk'.src}/app ${lib.escapeShellArgs zmk'.cmakeFlags} "-DUSER_CACHE_DIR=/tmp/.cache" "-DKEYMAP_FILE=$KEYMAP" "-DCONF_FILE=$KCONFIG" -DBOARD=glove80_lh + cmake -G Ninja -S ${zmk'.src}/app ${lib.escapeShellArgs zmk'.cmakeFlags} "-DUSER_CACHE_DIR=/tmp/.cache" "-DKEYMAP_FILE=$keymap" "-DCONF_FILE=$kconfig" "-DBOARD=$board" "-DBUILD_VERSION=${revision}" ninja if [ -n "$DEBUG" ]; then ccache -s; fi - cat zephyr/zmk.uf2 ${zmk_glove80_rh}/zmk.uf2 > zephyr/combined.uf2 + if [ -n "$merge_rhs" ]; then + cat zephyr/zmk.uf2 ${zmk_glove80_rh}/zmk.uf2 > zmk.uf2 + else + mv zephyr/zmk.uf2 zmk.uf2 + fi ''; ccacheCache = pkgs.runCommandNoCC "ccache-cache" { @@ -101,7 +135,13 @@ let mkdir /tmp/build cd /tmp/build - compileZmk ${zmk.src}/app/boards/arm/glove80/glove80.keymap + compileZmk -b glove80_lh -k ${zmk.src}/app/boards/arm/glove80/glove80.keymap + + rm -fr /tmp/build + mkdir /tmp/build + cd /tmp/build + + compileZmk -b glove80_rh -k ${zmk.src}/app/boards/arm/glove80/glove80.keymap ''; entrypoint = pkgs.writeShellScriptBin "entrypoint" '' @@ -119,22 +159,22 @@ let exec "$@" ''; - startLambda = handler: pkgs.writeShellScriptBin "startLambda" '' + startLambda = pkgs.writeShellScriptBin "startLambda" '' set -euo pipefail export PATH=${lib.makeBinPath [ zmkCompileScript ]}:$PATH cd ${lambda.source} - ${lambda.bundleEnv}/bin/bundle exec aws_lambda_ric "app.LambdaFunction::${handler}.process" + ${lambda.bundleEnv}/bin/bundle exec aws_lambda_ric "app.LambdaFunction::Handler.process" ''; - simulateLambda = lambda: pkgs.writeShellScriptBin "simulateLambda" '' - ${pkgs.aws-lambda-rie}/bin/aws-lambda-rie ${lambda}/bin/startLambda + simulateLambda = pkgs.writeShellScriptBin "simulateLambda" '' + ${pkgs.aws-lambda-rie}/bin/aws-lambda-rie ${startLambda}/bin/startLambda ''; - lambdaImage = lambda: + lambdaImage = let appLayer = { name = "app-layer"; - path = [ lambda zmkCompileScript ]; + path = [ startLambda zmkCompileScript ]; }; in ociTools.makeSimpleImage { @@ -148,19 +188,10 @@ let Env = [ "CCACHE_DIR=/tmp/ccache" "REVISION=${revision}" ]; }; }; - - # There are two lambda handler functions, depending on whether the lambda is - # expected to handle Api Gateway/ELB HTTP requests itself. - startHttpLambda = startLambda "HttpHandler"; - startDirectLambda = startLambda "DirectHandler"; - httpLambdaImage = lambdaImage startHttpLambda; - directLambdaImage = lambdaImage startDirectLambda; - - simulateDirectLambda = simulateLambda startDirectLambda; - simulateHttpLambda = simulateLambda startHttpLambda; in { - inherit httpLambdaImage directLambdaImage zmkCompileScript ccacheCache; + inherit lambdaImage zmkCompileScript ccacheCache; + directLambdaImage = lambdaImage; - # nix shell -f release.nix simulateDirectLambda -c simulateLambda - inherit simulateHttpLambda simulateDirectLambda; + # nix shell -f release.nix simulateLambda -c simulateLambda + inherit simulateLambda; }