-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
261 lines (216 loc) · 9.12 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# Replication package for the BTW docker@rtems paper
FROM ubuntu:18.04
MAINTAINER Wolfgang Mauerer "[email protected]"
ENV DEBIAN_FRONTEND noninteractive
ENV LANG="C.UTF-8"
ENV LC_ALL="C.UTF-8"
RUN apt update && apt -y dist-upgrade
RUN apt install -y --no-install-recommends \
build-essential \
python \
python-dev \
python3 \
python3-dev \
unzip \
sudo \
joe \
bison \
less \
flex \
ca-certificates \
curl \
git \
openssh-client \
qemu-system-arm \
qemu-system-i386 \
sudo \
texinfo \
libz-dev \
pax \
u-boot-tools \
libncurses-dev \
default-jre \
stress-ng
# To build gcc with rsb, we need python 2.7 (otherwise, the build breaks)
# To use waf, we want python 3.6 (otherwise, each command invocations
# stalls for 30'' for some reason before doing anything)
RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 2
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1
RUN useradd -m -G sudo -s /bin/bash build && echo "build:build" | chpasswd
USER build
WORKDIR /home/build
# Prepare directory structure
## src/ - to store external source packages
## dbtoaster/ - the measurement code proper (dbtoaster app sources)
## rtems/ - binary toolchain and BSPs for RTEMS
## build/ - temporary directory for out-of-tree builds
## dbtoaster-dist/ - upstream DBToaster distribution including query sources
RUN mkdir -p $HOME/src $HOME/dbtoaster $HOME/rtems $HOME/build $HOME/dbtoaster-dist
WORKDIR /home/build/dbtoaster
RUN curl https://waf.io/waf-2.0.19 > waf
# TODO: Checkout a defined version here
RUN git clone git://git.rtems.org/rtems_waf.git rtems_waf
RUN chmod +x waf
# 1. Obtain RTEMS source builder and kernel sources
WORKDIR /home/build/src
RUN curl https://ftp.rtems.org/pub/rtems/releases/5/5.1/sources/rtems-source-builder-5.1.tar.xz | tar xJf -
RUN curl https://ftp.rtems.org/pub/rtems/releases/5/5.1/sources/rtems-5.1.tar.xz | tar xJf -
RUN mv rtems-source-builder-5.1 rsb
# ... and patch up some things
WORKDIR /home/build/src/rtems-5.1
ADD patches/rtems.diff /home/build/src
RUN cat ../rtems.diff | patch -p1
WORKDIR /home/build/src/rsb
ADD patches/rsb.diff /home/build/src
RUN cat ../rsb.diff | patch -p1
# 2. Check environment
WORKDIR /home/build/src/rsb/rtems
RUN ../source-builder/sb-check
# 3. Build toolchains for i386 and ARM
WORKDIR /home/build/src/rsb/rtems
RUN ../source-builder/sb-set-builder --prefix=$HOME/rtems/5 5/rtems-i386
RUN ../source-builder/sb-set-builder --prefix=$HOME/rtems/5 5/rtems-arm
# 4. Make toolchain binaries available via PATH
ENV PATH /home/build/rtems/5/bin:$PATH
# 5. Build rtems BSPs for virtual and real measurement systems
WORKDIR /home/build/build
RUN $HOME/src/rtems-5.1/configure --prefix=$HOME/rtems/5 --target=i386-rtems5 --enable-rtemsbsp=pc586 --enable-posix --disable-networking --enable-rtems-debug
RUN make -j && make install
RUN rm -rf /home/build/build/*
RUN $HOME/src/rtems-5.1/configure --prefix=$HOME/rtems/5 --target=arm-rtems5 --enable-rtemsbsp="realview_pbx_a9_qemu beagleboneblack" --enable-posix --disable-networking --enable-rtems-debug
RUN make -j && make install
RUN rm -rf /home/build/build/*
# 6. Install DBToaster
WORKDIR /home/build/dbtoaster-dist
RUN curl https://dbtoaster.github.io/dist/dbtoaster_2.3_linux.tgz | tar xzf -
ENV PATH /home/build/dbtoaster-dist/dbtoaster/bin:$PATH
# 7. Download and build DbGen
WORKDIR /home/build/src
ADD tpch/tpch-h-tool.zip .
RUN unzip tpch-h-tool.zip
WORKDIR /home/build/src/2.18.0_rc2/dbgen
RUN make CC=gcc DATABASE=DB2 MACHINE=LINUX WORKLOAD=TPCH -f makefile.suite
#########################################################################################################
USER root
RUN update-alternatives --set python /usr/bin/python3.6
USER build
# 8. Integrate measurement dispatch code
WORKDIR /home/build/dbtoaster
ADD app/*.cc app/wscript app/Makefile ./
RUN mkdir -p rootfs/data generated
# 9. Build the DBToaster backend (i.e., libdbtoaster.a)
WORKDIR /home/build/src
RUN git clone https://github.com/lfd/dbtoaster-backend.git
WORKDIR /home/build/src/dbtoaster-backend
RUN git checkout btw2021
WORKDIR /home/build/src/dbtoaster-backend/ddbtoaster/srccpp/lib
RUN make -j
WORKDIR /home/build/dbtoaster
RUN ln -s /home/build/src/dbtoaster-backend/ddbtoaster/srccpp/lib .
# TODO: These files should be included in the btw2021 repo and
# be copied into dbtoaster once they are in a mature state
RUN cp /home/build/src/dbtoaster-backend/ddbtoaster/srccpp/driver_sequential.cpp .
# 9. Build DBToaster header files for relevant TPCH and finance queries
# NOTE: We deliberately don't use -o Tpch<n>-V.hpp because then the class name
# in the generated code is adapted from query to Tpch<n>-V, which is an
# invalid C++ identifier
WORKDIR /home/build/dbtoaster-dist/dbtoaster/
RUN sed -i 's,examples/data/tpch/,data/tpch/,g' examples/queries/tpch/schemas.sql
RUN sed -i 's,examples/data/,data/,g' examples/queries/finance/*.sql
ADD queries/countbids.sql examples/queries/finance/
ADD queries/countone.sql examples/queries/finance/
ADD queries/avgbrokerprice.sql examples/queries/finance/
ARG TPCH_BIN="1 2 6 12 14 11a 18a"
ARG FINANCE_BIN="vwap axfinder pricespread brokerspread missedtrades brokervariance countbids countone avgbrokerprice"
RUN /bin/bash -c 'for i in ${TPCH_BIN}; do \
echo "Generating DBToaster code for TPCH query ${i}"; \
bin/dbtoaster -l cpp examples/queries/tpch/query${i}.sql > $HOME/dbtoaster/generated/Tpch${i}-V.hpp; \
done'
RUN /bin/bash -c 'for Q in ${FINANCE_BIN}; do \
echo "Generating DBToaster code for financial query ${Q}"; \
bin/dbtoaster -l cpp examples/queries/finance/${Q}.sql > $HOME/dbtoaster/generated/${Q}.hpp; \
done'
# 10. Build TPCH example data, and copy finance data
# (we delibertely only build a very small data set for
# to "smoke test" if compiled binaries work properly)
WORKDIR /home/build/src/2.18.0_rc2/dbgen
RUN rm -rf *.tbl
RUN ./dbgen -s 0.01
# On occasion, the dbgen dungpile seemingly randomly assigns permissions
# 101 to generated tbl files. Dude...
RUN chmod 644 *.tbl
RUN mkdir -p /home/build/dbtoaster/rootfs/data/tpch/
RUN /bin/bash -c 'for file in *.tbl; do \
f=`basename ${file} .tbl`; \
cp ${f}.tbl /home/build/dbtoaster/rootfs/data/tpch/${f}.csv; \
done'
RUN curl https://raw.githubusercontent.com/dbtoaster/dbtoaster-experiments-data/master/finance/standard/finance.csv > /home/build/dbtoaster/rootfs/data/finance.csv
# 11. Build the DBToaster RTEMS app for x86 and all TPCH queries, using
# the TSC for time measurements
WORKDIR /home/build/dbtoaster
RUN mkdir -p rtems
RUN rm lib/libdbtoaster.a # The distribution provided binary is for x86_64-linux
RUN ./waf configure --rtems=$HOME/rtems/5 --rtems-bsp=i386/pc586
RUN /bin/bash -c 'for i in ${TPCH_BIN}; do \
rm -f build/i386-rtems5-pc586/{StreamDriver,driver_sequential}.*.{o,d}; \
CXXFLAGS=-DUSE_RDTSC QUERY=Tpch${i}-V ./waf build; \
mv build/i386-rtems5-pc586/dbtoaster.exe rtems/tpch${i}.exe; \
done'
RUN /bin/bash -c 'for i in ${FINANCE_BIN}; do \
rm -f build/i386-rtems5-pc586/{StreamDriver,driver_sequential}.*.{o,d}; \
CXXFLAGS=-DUSE_RDTSC QUERY=${i} ./waf build; \
mv build/i386-rtems5-pc586/dbtoaster.exe rtems/finance${i}.exe; \
done'
# 12. Build Linux binaries for all TPCH and financial queries
WORKDIR /home/build/dbtoaster
RUN mkdir -p linux/
RUN /bin/bash -c 'for i in ${TPCH_BIN}; do \
TPCH=${i} make measure; \
done'
RUN /bin/bash -c 'for Q in ${FINANCE_BIN}; do \
FINANCE=${Q} make finance; \
done'
## 12.b Build Linux binaries with TSC based measurements
WORKDIR /home/build/src/dbtoaster-backend/ddbtoaster/srccpp/lib
RUN make clean
RUN CXXFLAGS="-DUSE_RDTSC" make -j
WORKDIR /home/build/dbtoaster
RUN mkdir -p linux/
RUN /bin/bash -c 'for i in ${TPCH_BIN}; do \
TPCH=${i} make measure_tsc; \
done'
RUN /bin/bash -c 'for Q in ${FINANCE_BIN}; do \
FINANCE=${Q} make finance_tsc; \
done'
# 13. Generate self-contained measurement package that can
# be deployed on Linux x86_64 targets
WORKDIR /home/build/dbtoaster
RUN ln -s rootfs/data .
ADD measure/dispatch.sh .
ADD measure/doall.sh .
ADD measure/caps.sh .
ADD measure/rename.sh .
ADD measure/lib.r .
ADD measure/collect.r .
ADD measure/gen_arguments.r .
WORKDIR /home/build/dbtoaster/data/tpch
ADD measure/caps.sh .
RUN cp /home/build/src/2.18.0_rc2/dbgen/dbgen .
RUN cp /home/build/src/2.18.0_rc2/dbgen/dists.dss .
WORKDIR /home/build/dbtoaster
RUN tar --transform 's,^,measure/,' -cjhf ~/measure.tar.bz2 *.r *.sh linux/ data/
## 14 Build finance queries for RTEMS on Beaglebone Black (ARM)
WORKDIR /home/build/dbtoaster
RUN mkdir -p bbb
RUN mkdir -p rootfs-bbb
RUN mkdir -p rootfs-bbb/data/
RUN curl https://raw.githubusercontent.com/dbtoaster/dbtoaster-experiments-data/master/finance/tiny/finance.csv > /home/build/dbtoaster/rootfs-bbb/data/finance.csv
RUN rm -f lib/libdbtoaster.a
RUN ./waf configure --rtems=$HOME/rtems/5 --rtems-bsp=arm/beagleboneblack
RUN /bin/bash -c 'for i in ${FINANCE_BIN}; do \
ROOTFS_SFX=-bbb QUERY=${i} ./waf build; \
rm -f /tmp/app.bin*; \
arm-rtems5-objcopy build/arm-rtems5-beagleboneblack/dbtoaster.exe -O binary /tmp/app.bin; \
gzip -9 /tmp/app.bin; \
mkimage -A arm -O linux -T kernel -a 0x80000000 -e 0x80000000 -n RTEMS -d /tmp/app.bin.gz bbb/${i}.img; \
done'