forked from feldera/feldera
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
142 lines (127 loc) · 5.77 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
# The base image contains tools to build the code given that
# we need a Java and Rust compiler to run alongside the pipeline manager
# as of now. This will change later.
FROM ubuntu:24.04 AS base
ENV DEBIAN_FRONTEND=noninteractive
# These two environment variables are used to make openssl-sys pick
# up libssl-dev and statically link it. Without it, our build defaults
# to building a vendored version of OpenSSL.
ENV OPENSSL_NO_VENDOR=1
ENV OPENSSL_STATIC=1
RUN apt update --fix-missing && apt install \
# pkg-config is required for cargo to find libssl
libssl-dev pkg-config \
# rdkafka dependency needs cmake and a CXX compiler
cmake build-essential \
# To install rust
curl \
# For running the SQL compiler
openjdk-21-jre-headless -y \
# Install locale-gen
locales \
# To add the nodesource debian repository
ca-certificates gnupg \
# Required by the `metrics-exporter-tcp` crate
protobuf-compiler
# Set UTF-8 locale. Needed for the Rust compiler to handle Unicode column names.
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
locale-gen
ENV LC_ALL=en_US.UTF-8
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
# Use cargo-chef to produce a recipe.json file
# to cache the requisite dependencies
FROM base AS chef
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
RUN /root/.cargo/bin/cargo install cargo-chef
WORKDIR app
# Build web-ui
FROM base AS web-ui-builder
## Install Bun.js
RUN apt install unzip -y
RUN curl -fsSL https://bun.sh/install | bash
ENV PATH="$HOME/.bun/bin:$PATH"
RUN ln -s $HOME/.bun/bin/bun /usr/bin/bun
# sveltekit
COPY web-console/package.json web-console/
COPY web-console/bun.lockb web-console/
RUN cd web-console && bun install
COPY web-console/static web-console/static
COPY web-console/src web-console/src
COPY web-console/.prettierignore web-console/
COPY web-console/.prettierrc web-console/
COPY web-console/eslint.config.js web-console/
COPY web-console/postcss.config.js web-console/
COPY web-console/svelte.config.js web-console/
COPY web-console/tailwind.config.ts web-console/
COPY web-console/tsconfig.json web-console/
COPY web-console/vite.config.ts web-console/
RUN cd web-console && bun run build
# Cache dependencies from rust
FROM chef AS planner
COPY Cargo.toml Cargo.toml
COPY Cargo.lock Cargo.lock
COPY crates crates
RUN mkdir sql-to-dbsp-compiler
COPY sql-to-dbsp-compiler/lib sql-to-dbsp-compiler/lib
RUN /root/.cargo/bin/cargo chef prepare --recipe-path recipe.json
# Use the recipe.json file to build dependencies first and cache that
# layer for faster incremental builds of source-code only changes
FROM chef AS builder
ENV CARGO_INCREMENTAL=0
COPY --from=planner /app/recipe.json recipe.json
RUN /root/.cargo/bin/cargo chef cook --release --recipe-path recipe.json --bin=pipeline-manager --features=pg-embed
COPY Cargo.toml Cargo.toml
COPY Cargo.lock Cargo.lock
COPY crates crates
RUN mkdir sql-to-dbsp-compiler || true
COPY sql-to-dbsp-compiler/lib sql-to-dbsp-compiler/lib
COPY --from=web-ui-builder /web-console/build web-console-build
ENV WEBUI_BUILD_DIR=/app/web-console-out
ENV WEBCONSOLE_BUILD_DIR=/app/web-console-build
RUN /root/.cargo/bin/cargo build --release --bin=pipeline-manager --features=pg-embed
# Java build can be performed in parallel
FROM base AS javabuild
RUN apt install maven git -y
RUN mkdir sql
COPY sql-to-dbsp-compiler /sql/sql-to-dbsp-compiler
RUN --mount=type=cache,target=/root/.m2 \
cd /sql/sql-to-dbsp-compiler && ./build.sh
# Minimal image for running the pipeline manager
FROM base AS release
# Pipeline manager binary
RUN useradd -ms /bin/bash feldera
USER feldera
WORKDIR /home/feldera
COPY --from=builder /app/target/release/pipeline-manager pipeline-manager
# SQL compiler uber jar
RUN mkdir -p lib/sql-to-dbsp-compiler/SQL-compiler/target
COPY --from=javabuild /sql/sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp-jar-with-dependencies.jar lib/sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp-jar-with-dependencies.jar
# Reuse `Cargo.lock` to ensure consistent crate versions.
RUN mkdir -p .feldera/cargo_workspace
COPY --chown=feldera Cargo.lock .feldera/cargo_workspace/Cargo.lock
# Copy over demos
RUN mkdir -p demos
COPY demo/packaged/sql demos
# The crates needed for the SQL compiler
COPY crates/ lib/crates/
COPY README.md lib/README.md
RUN mkdir -p lib/sql-to-dbsp-compiler/lib
# Copy over the rust code and sql-to-dbsp script
COPY sql-to-dbsp-compiler/lib lib/sql-to-dbsp-compiler/lib
COPY sql-to-dbsp-compiler/temp lib/sql-to-dbsp-compiler/temp
COPY sql-to-dbsp-compiler/SQL-compiler/sql-to-dbsp lib/sql-to-dbsp-compiler/SQL-compiler/sql-to-dbsp
# Install cargo and rust for this non-root user
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
# The download URL for mold uses x86_64/aarch64 whereas dpkg --print-architecture says amd64/arm64
RUN arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x86_64/g"`; \
curl -LO https://github.com/rui314/mold/releases/download/v2.32.1/mold-2.32.1-$arch-linux.tar.gz \
&& tar -xzvf mold-2.32.1-$arch-linux.tar.gz \
&& mv mold-2.32.1-$arch-linux /home/feldera/mold \
&& rm mold-2.32.1-$arch-linux.tar.gz
ENV PATH="$PATH:/home/feldera/.cargo/bin:/home/feldera/mold/bin"
ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold"
# Run the precompile phase to speed up Rust compilations during deployment
RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler --dbsp-override-path=/home/feldera/lib --precompile
ENV BANNER_ADDR=localhost
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/lib/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/lib", "--allowed-origins", "https://www.feldera.com", "--allowed-origins", "http://localhost:8080", "--demos-dir", "/home/feldera/demos"]