From a87d4a5d5df18b89d01f855e3eb804b12d48c576 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 2 Jul 2024 14:46:15 +0200 Subject: [PATCH 1/2] switch to extension-ci-tools based CI --- .github/workflows/Linux.yml | 133 ------------------ .github/workflows/MacOS.yml | 83 ----------- .../workflows/MainDistributionPipeline.yml | 33 +++++ .github/workflows/WASM.yml | 42 ------ .github/workflows/Windows.yml | 75 ---------- .github/workflows/_extension_deploy.yml | 117 +++++++++++++++ .gitmodules | 3 + Makefile | 103 +------------- extension-ci-tools | 1 + extension_config.cmake | 7 + scripts/extension-upload.sh | 2 +- vcpkg.json | 4 + 12 files changed, 171 insertions(+), 432 deletions(-) delete mode 100644 .github/workflows/Linux.yml delete mode 100644 .github/workflows/MacOS.yml create mode 100644 .github/workflows/MainDistributionPipeline.yml delete mode 100644 .github/workflows/WASM.yml delete mode 100644 .github/workflows/Windows.yml create mode 100644 .github/workflows/_extension_deploy.yml create mode 160000 extension-ci-tools create mode 100644 extension_config.cmake create mode 100644 vcpkg.json diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml deleted file mode 100644 index 31ac4a8..0000000 --- a/.github/workflows/Linux.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Linux -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - linux: - name: Linux Release - runs-on: ubuntu-latest - container: ${{ matrix.container }} - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - # arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] - arch: ['linux_amd64', 'linux_amd64_gcc4'] # Disable linux_arm64 build for now - include: - - arch: 'linux_amd64' - container: 'ubuntu:18.04' - - arch: 'linux_amd64_gcc4' - container: 'quay.io/pypa/manylinux2014_x86_64' - #- arch: 'linux_arm64' - # container: 'ubuntu:18.04' - env: - GEN: ninja - DUCKDB_PLATFORM: ${{ matrix.arch }} - - steps: - - name: Install required ubuntu packages - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - - - name: Install Rust - run: curl https://sh.rustup.rs -sSf | bash -s -- -y - - - name: Install Rust Cross-compilation targets - if: ${{ matrix.arch == 'linux_arm64' }} - run: $HOME/.cargo/bin/rustup target add aarch64-unknown-linux-gnu - - - name: Install Git 2.18.5 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Patch manylinux_2014_setup - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} - run: | - sed -i 's@scripts/@duckdb/scripts/@g' ./duckdb/.github/actions/manylinux_2014_setup/action.yml - - - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} - uses: ./duckdb/.github/actions/manylinux_2014_setup - with: - openssl: 0 - ssh: 1 - - - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - uses: ./duckdb/.github/actions/ubuntu_18_setup - with: - aarch64_cross_compile: ${{ matrix.arch == 'linux_arm64' && 1 }} - - # Build extension - - name: Build extension - env: - GEN: ninja - STATIC_LIBCPP: 1 - EXTENSION_STATIC_BUILD: 1 - CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} - CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} - Rust_CARGO_TARGET: ${{ matrix.arch == 'linux_arm64' && 'aarch64-unknown-linux-gnu' || '' }} - run: | - make release - - - name: Build extension - if: ${{ matrix.arch != 'linux_arm64'}} - run: | - make test - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/prql/prql.duckdb_extension - - - name: Install SSH Key - uses: shimataro/ssh-key-action@v2.6.1 - with: - key: ${{ secrets.SSH_PRIVATE_KEY }} - known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} - - - name: Deploy - env: - SSH_USER: ${{ secrets.SSH_USER }} - SSH_HOST: ${{ secrets.SSH_HOST }} - UPLOAD_BASE_PATH: ${{ secrets.UPLOAD_BASE_PATH }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$SSH_HOST" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - ./scripts/extension-upload.sh prql ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - ./scripts/extension-upload.sh prql `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} false - fi \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml deleted file mode 100644 index 908767c..0000000 --- a/.github/workflows/MacOS.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: MacOS -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - macos: - name: MacOS Release - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - # Universal build (OSX_BUILD_UNIVERSAL: 1) not possible, see: https://github.com/corrosion-rs/corrosion/issues/115 - arch: ['osx_amd64', 'osx_arm64'] - runs-on: ${{ matrix.arch == 'osx_amd64' && 'macos-latest' || 'macos-14' }} - env: - GEN: ninja - DUCKDB_PLATFORM: ${{ matrix.arch }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Install Ninja - run: brew install ninja - - - uses: actions/setup-python@v2 - with: - python-version: '3.12' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - # Build extension - - name: Build extension - env: - # Static build on OS X breaks prql plugin, see https://github.com/duckdb/duckdb/issues/6521 - EXTENSION_STATIC_BUILD: 1 - shell: bash - run: make release - - - name: Build extension - run: make test - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/prql/prql.duckdb_extension - - - name: Install SSH Key - uses: shimataro/ssh-key-action@v2.6.1 - with: - key: ${{ secrets.SSH_PRIVATE_KEY }} - known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} - - - name: Deploy - env: - SSH_USER: ${{ secrets.SSH_USER }} - SSH_HOST: ${{ secrets.SSH_HOST }} - UPLOAD_BASE_PATH: ${{ secrets.UPLOAD_BASE_PATH }} - run: | - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$SSH_HOST" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - ./scripts/extension-upload.sh prql ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - ./scripts/extension-upload.sh prql `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} false - fi \ No newline at end of file diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml new file mode 100644 index 0000000..105c9e2 --- /dev/null +++ b/.github/workflows/MainDistributionPipeline.yml @@ -0,0 +1,33 @@ +# +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Main Extension Distribution Pipeline +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-stable-build: + name: Build extension binaries + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.0.0 + with: + duckdb_version: v1.0.0 + extension_name: prql + enable_rust: true + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;linux_arm64' + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable-build + uses: ./.github/workflows/_extension_deploy.yml + secrets: inherit + with: + extension_name: prql + duckdb_version: v1.0.0 + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;linux_arm64' + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/WASM.yml b/.github/workflows/WASM.yml deleted file mode 100644 index 14327a8..0000000 --- a/.github/workflows/WASM.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Wasm -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - wasm: - name: WASM Release - runs-on: 'macos-14' - env: - DUCKDB_PLATFORM: 'osx_arm64' - - steps: - - uses: actions/setup-python@v2 - with: - python-version: '3.12' - - - uses: mymindstorm/setup-emsdk@v14 - with: - version: 'latest' - - - name: Install Rust - run: curl https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain nightly - - - name: Install Rust Cross-compilation targets - run: $HOME/.cargo/bin/rustup target add wasm32-unknown-emscripten - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - # Build extension - - name: Build extension - run: | - make wasm_mvp - make wasm_eh - make wasm_threads diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml deleted file mode 100644 index 8d16796..0000000 --- a/.github/workflows/Windows.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Windows -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - windows: - name: Release - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - arch: ['windows_amd64'] - env: - DUCKDB_PLATFORM: ${{ matrix.arch }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Build extension - env: - EXTENSION_STATIC_BUILD: 1 - run: | - make release - build/release/test/Release/unittest.exe - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/prql/prql.duckdb_extension - - - name: Install SSH Key - uses: shimataro/ssh-key-action@v2.6.1 - with: - key: ${{ secrets.SSH_PRIVATE_KEY }} - known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} - - - name: Deploy - env: - SSH_USER: ${{ secrets.SSH_USER }} - SSH_HOST: ${{ secrets.SSH_HOST }} - UPLOAD_BASE_PATH: ${{ secrets.UPLOAD_BASE_PATH }} - run: | - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$SSH_HOST" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - ./scripts/extension-upload.sh prql ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - ./scripts/extension-upload.sh prql `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} false - fi \ No newline at end of file diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml new file mode 100644 index 0000000..d33d747 --- /dev/null +++ b/.github/workflows/_extension_deploy.yml @@ -0,0 +1,117 @@ +name: Extension Deployment +on: + workflow_call: + inputs: + # The name of the extension + extension_name: + required: true + type: string + # DuckDB version to build against + duckdb_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Whether to upload this deployment as the latest. This may overwrite a previous deployment. + deploy_latest: + required: false + type: boolean + default: false + # Whether to upload this deployment under a versioned path. These will not be deleted automatically + deploy_versioned: + required: false + type: boolean + default: false + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default deploy script with a custom script + deploy_script: + required: false + type: string + default: "./scripts/extension-upload.sh" + # Override the default matrix parse script with a custom script + matrix_parse_script: + required: false + type: string + default: "./duckdb/scripts/modify_distribution_matrix.py" + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - id: parse-matrices + run: | + python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + deploy_matrix="`cat deploy_matrix.json`" + echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} + path: | + /tmp/extension + + - name: Install SSH Key + uses: shimataro/ssh-key-action@v2.6.1 + with: + key: ${{ secrets.SSH_PRIVATE_KEY }} + known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} + + - name: Deploy + env: + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST: ${{ secrets.SSH_HOST }} + UPLOAD_BASE_PATH: ${{ secrets.UPLOAD_BASE_PATH }} + run: | + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + if [[ "$SSH_HOST" == "" ]] ; then + echo 'No key set, skipping' + elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then + ./scripts/extension-upload.sh prql ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} true + elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then + ./scripts/extension-upload.sh prql `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} false + fi diff --git a/.gitmodules b/.gitmodules index abb07d1..4d0f2eb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "corrosion"] path = corrosion url = https://github.com/corrosion-rs/corrosion.git +[submodule "extension-ci-tools"] + path = extension-ci-tools + url = git@github.com:duckdb/extension-ci-tools.git diff --git a/Makefile b/Makefile index c41c6e1..bfabf46 100644 --- a/Makefile +++ b/Makefile @@ -1,101 +1,8 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update - -all: release - -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -PROJ_DIR := $(dir $(MKFILE_PATH)) - -ifeq (${STATIC_LIBCPP}, 1) - STATIC_LIBCPP=-DEXTENSION_STATIC_BUILD=1 -endif - -CMAKE_OSX_ARCHITECTURES_FLAG= -ifneq (${CMAKE_OSX_ARCHITECTURES},) - CMAKE_OSX_ARCHITECTURES_FLAG=-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -endif - -Rust_CARGO_TARGET_FLAG= -ifneq ($(Rust_CARGO_TARGET),) - Rust_CARGO_TARGET_FLAG=-DRust_CARGO_TARGET=${Rust_CARGO_TARGET} -endif - -ifeq ($(GEN),ninja) - GENERATOR=-G "Ninja" - FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1 -endif - -#### Configuration for this extension -EXTENSION_NAME=PRQL -EXTENSION_FLAGS=\ --DDUCKDB_EXTENSION_NAMES="prql" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ --DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" - -BUILD_FLAGS=${STATIC_LIBCPP} $(EXTENSION_FLAGS) ${CMAKE_OSX_ARCHITECTURES_FLAG} ${Rust_CARGO_TARGET_FLAG} -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' - -ifeq (${EXTENSION_STATIC_BUILD}, 1) - BUILD_FLAGS:=${BUILD_FLAGS} -DEXTENSION_STATIC_BUILD=1 -endif - -pull: - git submodule init - git submodule update --recursive - -clean: - rm -rf build - rm -rf testext - cd duckdb && make clean - -#### Main build -# For regular CLI build, we link the quack extension directly into the DuckDB executable -CLIENT_FLAGS=-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 - -debug: - mkdir -p build/debug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \ - cmake --build build/debug --config Debug - -release: - mkdir -p build/release && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(CLIENT_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \ - cmake --build build/release --config Release - -# Main tests -test: test_release - -test_release: release - ./build/release/test/unittest --test-dir . "[sql]" - -test_debug: debug - ./build/debug/test/unittest --test-dir . "[sql]" - -format: - find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - cmake-format -i CMakeLists.txt - -update: - git submodule update --remote --merge - -WASM_LINK_TIME_FLAGS= +PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +# Configuration of extension EXT_NAME=prql +EXT_CONFIG=${PROJ_DIR}extension_config.cmake -wasm_mvp: - mkdir -p build/wasm_mvp - emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' - RUSTFLAGS="-Clink-arg=-sSIDE_MODULE=1" emmake make -j8 -C build/wasm_mvp - cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension.wasm $(WASM_LINK_TIME_FLAGS) - -wasm_eh: - mkdir -p build/wasm_eh - emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_eh -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' - RUSTFLAGS="-Clink-arg=-sSIDE_MODULE=1" emmake make -j8 -C build/wasm_eh - cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension.wasm $(WASM_LINK_TIME_FLAGS) - -wasm_threads: - mkdir -p ./build/wasm_threads - emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_threads -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' - RUSTFLAGS="-Clink-arg=-sSIDE_MODULE=1" emmake make -j8 -C build/wasm_threads - cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension.wasm $(WASM_LINK_TIME_FLAGS) +# Include the Makefile from extension-ci-tools +include extension-ci-tools/makefiles/duckdb_extension.Makefile \ No newline at end of file diff --git a/extension-ci-tools b/extension-ci-tools new file mode 160000 index 0000000..b9a8fa2 --- /dev/null +++ b/extension-ci-tools @@ -0,0 +1 @@ +Subproject commit b9a8fa2adf10712fded529159cc8c7482a7caf85 diff --git a/extension_config.cmake b/extension_config.cmake new file mode 100644 index 0000000..f555edc --- /dev/null +++ b/extension_config.cmake @@ -0,0 +1,7 @@ +# This file is included by DuckDB's build system. It specifies which extension to load + +# Extension from this repo +duckdb_extension_load(prql + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} + LOAD_TESTS +) \ No newline at end of file diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh index b8b2260..4fcc074 100755 --- a/scripts/extension-upload.sh +++ b/scripts/extension-upload.sh @@ -9,7 +9,7 @@ set -e -ext="build/release/extension/$1/$1.duckdb_extension" +ext="/tmp/extension/$1.duckdb_extension" # compress extension binary gzip < $ext > "$1.duckdb_extension.gz" diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..cb71275 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,4 @@ +{ + "dependencies": [ + ] +} \ No newline at end of file From b8d3b7cfe12df8def0389c56de47204b22dcb01f Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 2 Jul 2024 15:56:21 +0200 Subject: [PATCH 2/2] skip ssh key installation if not present --- .github/workflows/_extension_deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index d33d747..e155084 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -92,6 +92,9 @@ jobs: - name: Install SSH Key uses: shimataro/ssh-key-action@v2.6.1 + env: + key: ${{ secrets.SSH_PRIVATE_KEY }} + if: ${{ env.key != '' }} with: key: ${{ secrets.SSH_PRIVATE_KEY }} known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}