Use vcpkg to build deb, apk and macOS packages

BewareMyPower · Jul 9, 2024 · 8036924 · 8036924
1 parent 35bf161
commit 8036924
Show file tree

Hide file tree

Showing 12 changed files with 92 additions and 381 deletions.
diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml
@@ -208,17 +208,19 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: recursive
 
       - name: Install dependencies
-        run: |
-          export ARCH=${{ matrix.arch }}
-          ./pkg/mac/build-static-library.sh
+        run: ARCH=${{ matrix.arch }} ./pkg/mac/build-static-library.sh
 
       - name: Zip artifact
         run: |
-          cd ./pkg/mac/.install
-          zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/*
-          cp macos-${{ matrix.arch }}.zip ../../../
+          mv build/include/pulsar/Version.h include/pulsar/
+          mv build/lib/*.a lib/
+          mv build/lib/*.dylib lib/
+          zip -r macos-${{ matrix.arch }}.zip ./include/pulsar/* ./lib/*.a ./lib/*.dylib
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml
@@ -344,18 +344,21 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: recursive
 
       - name: Build libraries
-        run: ./pkg/mac/build-static-library.sh
+        run: ARCH=x86_64 ./pkg/mac/build-static-library.sh
 
       - name: Test static libraries
         run: |
-          export PULSAR_DIR=$PWD/pkg/mac/.install
+          export PULSAR_DIR=./build
           echo "Build with static library"
-          clang++ win-examples/example.cc -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/lib/libpulsarwithdeps.a
+          clang++ win-examples/example.cc -arch x86_64 -o static.out -std=c++11 -I $PULSAR_DIR/include $PULSAR_DIR/build/lib/libpulsarwithdeps.a -framework SystemConfiguration -framework CoreFoundation
           ./static.out
           echo "Build with dynamic library"
-          clang++ win-examples/example.cc -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/lib -Wl,-rpath $PULSAR_DIR/lib -lpulsar
+          clang++ win-examples/example.cc -arch x86_64 -o dynamic.out -std=c++11 -I $PULSAR_DIR/include -L $PULSAR_DIR/build/lib -Wl,-rpath $PULSAR_DIR/build/lib -lpulsar
           ./dynamic.out
 
   # Job that will be required to complete and depends on all the other jobs

diff --git a/build-support/merge_archives_vcpkg.sh b/build-support/merge_archives_vcpkg.sh
@@ -29,4 +29,4 @@ fi
 CMAKE_BUILD_DIRECTORY=$1
 ./merge_archives.sh $CMAKE_BUILD_DIRECTORY/libpulsarwithdeps.a \
     $CMAKE_BUILD_DIRECTORY/lib/libpulsar.a \
-    $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug)
+    $(find "$CMAKE_BUILD_DIRECTORY/vcpkg_installed" -name "*.a" | grep -v debug | grep "$TRIPLET")
diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD
@@ -38,19 +38,18 @@ build() {
 	if [ "$CBUILD" != "$CHOST" ]; then
 		CMAKE_CROSSOPTS="-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_HOST_SYSTEM_NAME=Linux"
 	fi
-	cmake \
-		-DCMAKE_INSTALL_PREFIX=/usr \
-		-DCMAKE_INSTALL_LIBDIR=lib \
-		-DBUILD_SHARED_LIBS=True \
-		-DCMAKE_BUILD_TYPE=Release \
-		-DCMAKE_CXX_FLAGS="$CXXFLAGS" \
-		-DCMAKE_C_FLAGS="$CFLAGS" \
-		-DBUILD_TESTS=OFF \
-                -DLINK_STATIC=ON  \
-		${CMAKE_CROSSOPTS} ${ROOT_DIR}
-	make -j6
+    git clone https://github.com/microsoft/vcpkg.git
+    mv vcpkg $ROOT_DIR/
+    if [[ $PLATFORM == "aarch64" ]]; then
+        export VCPKG_FORCE_SYSTEM_BINARIES=1
+    fi
+    cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX="$pkgdir" \
+        -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON \
+        $ROOT_DIR
+    cmake --build build -j8
 }
 
 package() {
-	make DESTDIR="$pkgdir" install
+    cmake --build build --target install
 }
diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile
@@ -21,91 +21,23 @@ FROM alpine:3.12
 
 ARG PLATFORM
 
-# perl is required to install OpenSSL
-RUN apk add \
-      build-base \
+RUN apk add build-base \
       linux-headers \
       abuild \
-      bash \
-      curl \
-      g++ \
       make \
+      curl \
       cmake \
-      python3 \
-      py3-pip \
+      git \
+      ninja \
+      zip \
+      unzip \
+      openssl-dev \
       perl \
-      sudo
-
-RUN pip3 install pyyaml
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-
-# Download and compile boost
-RUN BOOST_VERSION=$(dep-version.py boost) && \
-    curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \
-    tar zxf boost-${BOOST_VERSION}.tar.gz && \
-    cd boost-${BOOST_VERSION} && \
-    ./bootstrap.sh --with-libraries=regex && \
-    ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
-    rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION}
-
-# Download and compile protobuf
-RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
-    curl -O -L  https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    cd protobuf-${PROTOBUF_VERSION}/ && \
-    CXXFLAGS=-fPIC ./configure && \
-    make -j8 && make install && \
-    rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
-
-# ZLib
-RUN ZLIB_VERSION=$(dep-version.py zlib) && \
-    curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
-    tar xfz v${ZLIB_VERSION}.tar.gz && \
-    cd zlib-${ZLIB_VERSION} && \
-    CFLAGS="-fPIC -O3" ./configure && \
-    make -j8 && make install && \
-    rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
-
-# Zstandard
-RUN ZSTD_VERSION=$(dep-version.py zstd) && \
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
-    tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
-    cd zstd-${ZSTD_VERSION} && \
-    CFLAGS="-fPIC -O3" make -j8 && \
-    make install && \
-    rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
-
-# Snappy
-RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
-    tar xfz ${SNAPPY_VERSION}.tar.gz && \
-    cd snappy-${SNAPPY_VERSION} && \
-    CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
-    make -j8 && make install && \
-    rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
-
-RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
-    OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
-    ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \
-    make -j8 && make install && \
-    rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
-
-ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:
-ENV OPENSSL_ROOT_DIR /usr/local/ssl/
-
-# LibCurl
-RUN CURL_VERSION=$(dep-version.py curl) && \
-    CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
-    curl -O -L  https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
-    tar xfz curl-${CURL_VERSION}.tar.gz && \
-    cd curl-${CURL_VERSION} && \
-    CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \
-    make -j8 && make install && \
-    rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
-
-
+      bash
+
+# Build latest CMake for vcpkg
+RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0.tar.gz \
+    && tar zxf cmake-3.30.0.tar.gz \
+    && cd cmake-3.30.0 \
+    && ./configure && make -j8 && make install \
+    && cd .. && rm -rf cmake-3.30.0 cmake-3.30.0.tar.gz
diff --git a/pkg/apk/docker-build-apk-arm64.sh b/pkg/apk/docker-build-apk-arm64.sh
@@ -25,7 +25,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-arm64}
 
 docker run -v $ROOT_DIR:/pulsar-client-cpp \
-        --env PLATFORM=arm64 \
+        --env PLATFORM=aarch64 \
         $IMAGE_NAME \
         /pulsar-client-cpp/pkg/apk/build-apk.sh
 
diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile
@@ -29,88 +29,10 @@ RUN apt-get update -y && \
         curl \
         g++ \
         make \
-        perl \
         dpkg-dev \
-        python3 \
-        python3-pip
-
-RUN pip3 install pyyaml
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-
-# Download and compile boost
-RUN BOOST_VERSION=$(dep-version.py boost) && \
-    curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \
-    tar zxf boost-${BOOST_VERSION}.tar.gz && \
-    cd boost-${BOOST_VERSION} && \
-    ./bootstrap.sh --with-libraries=regex && \
-    ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
-    rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION}
-
-RUN CMAKE_VERSION=$(dep-version.py cmake) && \
-    curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
-    tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
-    cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \
-    cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \
-    rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz
-
-# Download and compile protobuf
-RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
-    curl -O -L  https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    cd protobuf-${PROTOBUF_VERSION}/ && \
-    CXXFLAGS=-fPIC ./configure && \
-    make -j8 && make install && ldconfig && \
-    rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
-
-# ZLib
-RUN ZLIB_VERSION=$(dep-version.py zlib) && \
-    curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
-    tar xfz v${ZLIB_VERSION}.tar.gz && \
-    cd zlib-${ZLIB_VERSION} && \
-    CFLAGS="-fPIC -O3" ./configure && \
-    make -j8 && make install && \
-    rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
-
-# Zstandard
-RUN ZSTD_VERSION=$(dep-version.py zstd) && \
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
-    tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
-    cd zstd-${ZSTD_VERSION} && \
-    CFLAGS="-fPIC -O3" make -j8 && \
-    make install && \
-    rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
-
-# Snappy
-RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
-    tar xfz ${SNAPPY_VERSION}.tar.gz && \
-    cd snappy-${SNAPPY_VERSION} && \
-    CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
-    make -j8 && make install && \
-    rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
-
-RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
-    OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
-    ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \
-    make -j8 && make install && \
-    rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
-
-ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:
-ENV OPENSSL_ROOT_DIR /usr/local/ssl/
-
-# LibCurl
-RUN CURL_VERSION=$(dep-version.py curl) && \
-    CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
-    curl -O -L  https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
-    tar xfz curl-${CURL_VERSION}.tar.gz && \
-    cd curl-${CURL_VERSION} && \
-    CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \
-    make -j8 && make install && \
-    rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
-
-
+        zip \
+        unzip \
+        tar \
+        git \
+        pkg-config \
+        ninja-build
diff --git a/pkg/deb/build-deb.sh b/pkg/deb/build-deb.sh
@@ -20,6 +20,10 @@
 
 set -e -x
 
+if [[ $# -gt 0 ]]; then
+    CMAKE_ARCH=$1
+fi
+
 cd /pulsar-client-cpp
 SRC_ROOT_DIR=$(pwd)
 cd pkg/deb
@@ -36,13 +40,24 @@ mkdir BUILD
 cd BUILD
 tar xfz $SRC_ROOT_DIR/apache-pulsar-client-cpp-$POM_VERSION.tar.gz
 pushd $CPP_DIR
-
-# link libraries for protoc
-export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-
+git clone https://github.com/microsoft/vcpkg.git
+if [[ $CMAKE_ARCH ]]; then
+    curl -O -L https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-$CMAKE_ARCH.tar.gz 
+    export PATH=$PWD/cmake-3.30.0-linux-$CMAKE_ARCH/bin/:$PATH
+    tar zxf cmake-*.tar.gz
+fi
 chmod +x $(find . -name "*.sh")
-cmake . -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF -DLINK_STATIC=ON
-make -j 3
+if [[ $CMAKE_ARCH == "aarch64" ]]; then
+    export VCPKG_FORCE_SYSTEM_BINARIES=1
+fi
+cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON
+cmake --build build -j8
+./build-support/merge_archives_vcpkg.sh $PWD/build
+
+cp build/lib/libpulsar.a lib/libpulsar.a
+cp build/lib/libpulsar.so lib/libpulsar.so
+cp build/libpulsarwithdeps.a lib/libpulsarwithdeps.a
 popd
 
 DEST_DIR=apache-pulsar-client

diff --git a/pkg/deb/docker-build-deb-arm64.sh b/pkg/deb/docker-build-deb-arm64.sh
@@ -27,4 +27,6 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64}
 docker run -v $ROOT_DIR:/pulsar-client-cpp \
         --env PLATFORM=arm64 \
         $IMAGE_NAME \
-        /pulsar-client-cpp/pkg/deb/build-deb.sh
+        /pulsar-client-cpp/pkg/deb/build-deb.sh \
+        aarch64
+        https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-aarch64.tar.gz
diff --git a/pkg/deb/docker-build-deb-x86_64.sh b/pkg/deb/docker-build-deb-x86_64.sh
@@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64}
 docker run -v $ROOT_DIR:/pulsar-client-cpp \
         --env PLATFORM=amd64 \
         $IMAGE_NAME \
-        /pulsar-client-cpp/pkg/deb/build-deb.sh
+        /pulsar-client-cpp/pkg/deb/build-deb.sh \
+        x86_64