Many fixes and improvements + workflows pass again -> version 0.5.0

.github/workflows/end-to-end-test-macos.yml

@@ -16,59 +16,90 @@ jobs:
         os: [macos-12]
 
     steps:
-      - name: Checkout the QLever script
+      - name: Checkout the repository for the qlever script
         uses: actions/checkout@v3
         with:
           path: qlever-control
 
-      - name: Install dependencies for the QLever script
+      - name: Install qlever script locally + dependencies needed for testing
+        working-directory: ${{github.workspace}}/qlever-control
         run: |
+          python3 -m pip install --upgrade pip setuptools wheel
+          python3 --version
+          pip3 --version
+          pip3 show setuptools wheel
+          pip3 install -e .
+          brew install unzip expect
+          pip3 install flake8
           brew install llvm@16
           brew install conan@2
-          brew install unzip expect
-          pip3 install flake8 psutil termcolor
 
-      - name: Format and compile check
-        working-directory: ${{github.workspace}}/qlever-control
+      - name: Show version of built-in vs. installed clang
         run: |
-          flake8 qlever
-          python3 -m py_compile qlever
+          echo "Version of built-in clang:"
+          clang++ --version
+          echo "Version of installed clang:"
+          export PATH="/usr/local/opt/llvm@16/bin:$PATH"
+          clang++ --version
 
-      - name: Cache for conan (to make the next step much faster)
+      - name: Cache for conan modules
         uses: actions/cache@v3
         env:
-          cache-name: cache-conan-modules
+          cache-name: conan-modules
         with:
           path: ~/.conan2
-          key: ${{runner.os}}-build-${{env.cache-name}}-${{hashFiles('conanfile.txt')}}
+          key: ${{matrix.os}}-${{env.cache-name}}-2
 
-      - name: Download QLever binaries and install dependencies
+      - name: Cache for qlever code and binaries
+        uses: actions/cache@v3
+        env:
+          cache-name: qlever-code
+        with:
+          path: ${{github.workspace}}/qlever-code
+          key: ${{matrix.os}}-${{env.cache-name}}
+
+      - name: Update qlever repository (clone if not cached)
+        run: |
+          if [ ! -d qlever-code ]; then
+            git clone https://github.com/ad-freiburg/qlever.git qlever-code; fi
+          cd qlever-code
+          git pull
+
+      - name: Install dependencies using conan (takes long if not cached)
+        working-directory: ${{github.workspace}}/qlever-code
         run: |
-          git clone https://github.com/ad-freiburg/qlever.git qlever-code
-          mkdir -p qlever-code/build && cd $_
-          conan install .. -pr:b=../conanprofiles/clang-16-macos -pr:h=../conanprofiles/clang-16-macos -of=. --build=missing
-          cd ../..
-          mkdir qlever-binaries && cd $_
-          wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/macos-12/ServerMain
-          wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/macos-12/IndexBuilderMain
-          chmod 755 ServerMain IndexBuilderMain
+          mkdir -p build && cd $_
+          conan install .. -pr:b=../conanprofiles/clang-16-macos -pr:h=../conanprofiles/clang-16-macos -of=. --build=missing;
 
-      - name: Check that everything is found and runs
+      - name: Build qlever binaries
+        working-directory: ${{github.workspace}}/qlever-code
         run: |
-          pwd && ls -lh
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
+          export PATH="/usr/local/opt/llvm@16/bin:$PATH"
+          export LDFLAGS="-L/usr/local/opt/llvm@16/lib -L/usr/local/opt/llvm@16/lib/c++ -Wl,-rpath,/usr/local/opt/llvm@16/lib/c++"
+          export CPPFLAGS="-I/usr/local/opt/llvm@16/include"
+          cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE="$(pwd)/build/conan_toolchain.cmake" -DUSE_PARALLEL=true -DRUN_EXPENSIVE_TESTS=false -DENABLE_EXPENSIVE_CHECKS=true -DCMAKE_CXX_COMPILER=clang++ -DADDITIONAL_COMPILER_FLAGS="-fexperimental-library" -DADDITIONAL_LINKER_FLAGS="-L$(brew --prefix llvm)/lib/c++"
+          source build/conanrun.sh
+          make -C build ServerMain IndexBuilderMain
+
+      - name: Check that qlever binaries and qlever script are found and work
+        run: |
+          export PATH="$PATH:$(pwd)/qlever-code/build"
           source qlever-code/build/conanrun.sh
-          ServerMain --help > /dev/null
-          IndexBuilderMain --help > /dev/null
-          qlever
-          qlever help
+          ServerMain --help | head -3; echo "..."
+          IndexBuilderMain --help | head -3; echo "..."
+          qlever --help
 
-      - name: Test actions for olympics dataset, without Docker
+      - name: Test qlever script for olympics dataset, without Docker
+        timeout-minutes: 3
         run: |
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
-          mkdir -p ${{github.workspace}}/qlever-indices/olympics && cd $_
-          source ${{github.workspace}}/qlever-code/build/conanrun.sh
+          export PATH="$PATH:$(pwd)/qlever-code/build"
+          source qlever-code/build/conanrun.sh
+          export QLEVER_ARGCOMPLETE_ENABLED=1
+          mkdir -p qlever-indices/olympics && cd $_
           qlever setup-config olympics
-          qlever get-data docker.USE_DOCKER=false index index-stats
-          qlever docker.USE_DOCKER=false start status test-query stop restart test-query stop status start test-query status stop
+          qlever get-data
+          qlever index --system native
+          qlever start --system native
+          qlever status
+          qlever stop
           ls -lh

.github/workflows/end-to-end-test-ubuntu.yml

@@ -16,73 +16,92 @@ jobs:
         os: [ubuntu-22.04]
 
     steps:
-      - name: Checkout the QLever script
+      - name: Checkout the repository for the qlever script
         uses: actions/checkout@v3
         with:
           path: qlever-control
 
-      - name: Install dependencies for the QLever script
-        run: |
-          sudo apt update
-          sudo apt install unzip flake8 expect
-          pip3 install psutil termcolor
-
-      - name: Format and compile check
+      - name: Install qlever script locally + dependencies needed for testing
         working-directory: ${{github.workspace}}/qlever-control
         run: |
-          flake8 qlever
-          python3 -m py_compile qlever
+          python3 -m pip install --upgrade pip setuptools wheel
+          python3 --version
+          pip3 --version
+          pip3 show setuptools wheel
+          pip3 install -e .
+          sudo apt update && sudo apt install unzip expect
 
-      - name: Download QLever binaries and docker image and install dependencies
+      - name: Cache for qlever code and binaries
+        uses: actions/cache@v3
+        env:
+          cache-name: qlever-code
+        with:
+          path: ${{github.workspace}}/qlever-code
+          key: ${{matrix.os}}-${{env.cache-name}}
+
+      - name: Build qlever binaries from source + download docker image
         run: |
-          # Download Docker image.
+          sudo apt update
+          sudo apt install -y software-properties-common
+          sudo add-apt-repository -y ppa:mhier/libboost-latest
+          sudo apt install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev
+          if [ ! -d qlever-code ]; then
+            git clone https://github.com/ad-freiburg/qlever.git qlever-code; fi
+          cd qlever-code
+          git pull
+          mkdir -p build && cd $_
+          cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -GNinja ..
+          ninja ServerMain IndexBuilderMain
           docker pull adfreiburg/qlever
-          # Download QLever binaries.
-          mkdir qlever-binaries && cd $_
-          wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/ubuntu-22.04/ServerMain
-          wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/ubuntu-22.04/IndexBuilderMain
-          chmod 755 ServerMain IndexBuilderMain
-          # Install depencies needed for running the binaries.
-          curl -Gs https://raw.githubusercontent.com/ad-freiburg/qlever/master/Dockerfile | sed -En 's/(add-apt|apt|tee)/sudo \1/g; s/^RUN //p' | sed '/^cmake/q' | sed -E 's/^(cmake.*)/mkdir -p build \&\& cd build\n\1\ncd ../' | sed -n '/^sudo/p' > INSTALL
-          cat INSTALL
-          source ./INSTALL
 
-      - name: Check that everything is found and runs
+      - name: Check that qlever binaries, docker image, and qlever script are found and work
         run: |
           mkdir qlever-indices
-          pwd && ls -lh
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
-          docker run --entrypoint bash adfreiburg/qlever -c "ServerMain --help" > /dev/null
-          docker run --entrypoint bash adfreiburg/qlever -c "IndexBuilderMain --help" > /dev/null
-          ServerMain --help > /dev/null
-          IndexBuilderMain --help > /dev/null
-          qlever
-          qlever help
+          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-code/build"
+          docker run --entrypoint bash adfreiburg/qlever -c "ServerMain --help" | head -3
+          docker run --entrypoint bash adfreiburg/qlever -c "IndexBuilderMain --help" | head -3
+          ServerMain --help | head -3; echo "..."
+          IndexBuilderMain --help | head -3; echo "..."
+          qlever --help
 
-      - name: Test actions for olympics dataset, with Docker
+      - name: Test qlever script for olympics dataset, with docker
+        timeout-minutes: 1
         run: |
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
+          export PATH="$PATH:$(pwd)/qlever-code/build"
+          export QLEVER_ARGCOMPLETE_ENABLED=1
           mkdir -p ${{github.workspace}}/qlever-indices/olympics.1 && cd $_
           qlever setup-config olympics
-          unbuffer qlever get-data docker.USE_DOCKER=true index index-stats start status test-query restart status test-query stop status start stop start test-query stop status
+          qlever get-data
+          unbuffer qlever index
+          unbuffer qlever start
+          qlever status
+          qlever stop
           ls -lh
 
-      - name: Test actions for olympics dataset, without Docker
+      - name: Test qlever script for olympics dataset, without docker
+        timeout-minutes: 1
         run: |
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
+          export PATH="$PATH:$(pwd)/qlever-code/build"
+          export QLEVER_ARGCOMPLETE_ENABLED=1
           mkdir -p ${{github.workspace}}/qlever-indices/olympics.2 && cd $_
           qlever setup-config olympics
-          qlever get-data docker.USE_DOCKER=false index index-stats start status test-query restart status test-query stop status start stop start test-query stop status
+          qlever get-data
+          qlever index --system native
+          qlever start --system native
+          qlever status
+          qlever stop
           ls -lh
 
-      - name: Test actions for olympics dataset, with and without Docker
+      - name: Test qlever script for olympics dataset, with and without docker
+        timeout-minutes: 1
         run: |
-          export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
+          export PATH="$PATH:$(pwd)/qlever-code/build"
+          export QLEVER_ARGCOMPLETE_ENABLED=1
           mkdir -p ${{github.workspace}}/qlever-indices/olympics.3 && cd $_
           qlever setup-config olympics
-          unbuffer qlever get-data index index-stats
-          qlever docker.USE_DOCKER=false start status test-query
-          unbuffer qlever docker.USE_DOCKER=true stop status
-          unbuffer qlever docker.USE_DOCKER=true restart status test-query
-          qlever docker.USE_DOCKER=false stop status
+          qlever get-data
+          unbuffer qlever index
+          qlever start --system native
+          qlever status
+          qlever stop
           ls -lh

.github/workflows/format-check.yml

@@ -0,0 +1,32 @@
+name: Format check
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  merge_group:
+
+jobs:
+  format-check:
+    runs-on: ${{matrix.os}}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-22.04]
+
+    steps:
+      - name: Checkout the repository for the qlever script
+        uses: actions/checkout@v3
+        with:
+          path: qlever-control
+
+      - name: Install dependencies needed for checking
+        run: |
+          sudo apt update && sudo apt install flake8
+          pip install isort
+
+      - name: Check format, compilation, and imports
+        working-directory: ${{github.workspace}}/qlever-control
+        run: |
+          for PY in $(find src test -name "*.py"); do printf "$PY ... "; flake8 $PY && python3 -m py_compile $PY && isort -c $PY && echo "OK"; done

pyproject.toml

@@ -1,31 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "qlever"
 description = "Script for using the QLever SPARQL engine."
-version = "0.4.0"
+version = "0.5.0"
 authors = [
     { name = "Hannah Bast", email = "[email protected]" }
 ]
 readme = "README.md"
-license = { file = "LICENSE" }
+license = { text = "Apache-2.0" }
 requires-python = ">=3.8"
 
-keywords = ["SPARQL", "RDF", "knowledge graphs", "triple store"]
+keywords = ["SPARQL", "RDF", "Knowledge Graphs", "Triple Store"]
 
 classifiers = [
  "Topic :: Database :: Database Engines/Servers",
  "Topic :: Database :: Front-Ends"
 ]
 
-dependencies = [ "psutil", "termcolor" ]
+dependencies = [ "psutil", "termcolor", "argcomplete" ]
 
 [project.urls]
-Github = "https://github.com/ad-freiburg/qlever-control"
+Github = "https://github.com/ad-freiburg/qlever"
 
 [project.scripts]
 "qlever" = "qlever.qlever_main:main"
-"qlever-old" = "qlever.__main__:main"
+"qlever-old" = "qlever.qlever_old:main"
 
 [tool.setuptools]
-package-dir = { "" = "src" }
-packages = [ "qlever", "qlever.commands", "qlever.Qleverfiles" ]
-# package-data = { "qlever" = ["Qleverfiles/*"] }
+license-files = ["LICENSE"]
+package-data = { "qlever" = ["Qleverfiles/*"] }
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]

src/qlever/Qleverfiles/Qleverfile.ohm-planet

@@ -0,0 +1,37 @@
+# Qleverfile for OHM Planet, use with https://github.com/ad-freiburg/qlever-control
+#
+# qlever get-data  # ~20 mins (download PBF, convert to TTL, add GeoSPARQL triples)
+# qlever index     # ~10 mins and ~5 GB RAM (on an AMD Ryzen 9 5900X)
+# qlever start     # start the server (instantaneous)
+#
+# For `qlever get-data` to work, `osm2rdf` and `spatialjoin` must be installed
+# and included in the `PATH`.
+
+[data]
+NAME           = ohm-planet
+GET_DATA_URL   = $$(curl -s https://planet.openhistoricalmap.org/planet/state.txt)
+GET_DATA_CMD_1 = curl -LRfC - -o ${NAME}.pbf ${GET_DATA_URL} 2>&1 | tee ${NAME}.download-log.txt
+GET_DATA_CMD_2 = osm2rdf ${NAME}.pbf -o ${NAME}.ttl --source-dataset OHM --simplify-wkt 0 --write-ogc-geo-triples none 2>&1 | tee ${NAME}.osm2rdf-log.txt
+GET_DATA_CMD_3 = bzcat ${NAME}.ttl.bz2 | \grep "^osm2rdf" | sed -En 's/^osm2rdf(geom)?:(ohm_)?(node|rel|way)[a-z]*_([0-9]+) geo:asWKT "([^\"]+)".*/ohm\3:\4\t\5/p' | tee ${NAME}.spatialjoin-input.tsv | spatialjoin --contains ' ogc:sfContains ' --intersects ' ogc:sfIntersects ' --suffix $$' .\n' -o ${NAME}.spatialjoin-triples.ttl.bz2 2>&1 | tee ${NAME}.spatialjoin-log.txt && rm -f areas events lines points simplelines
+GET_DATA_CMD   = ${GET_DATA_CMD_1} && echo && ${GET_DATA_CMD_2} && echo && ${GET_DATA_CMD_3} && head -100 <(bzcat ${NAME}.ttl.bz2) | sed '/^@prefix/!d' > ${NAME}.prefix-definitions
+VERSION        = $$(date -r ${NAME}.pbf +%d.%m.%Y || echo "NO_DATE")
+DESCRIPTION    = OHM Planet, data from ${GET_DATA_URL} version ${VERSION} (with GeoSPARQL predicates ogc:sfContains and ogc:sfIntersects)
+
+[index]
+INPUT_FILES      = ${data:NAME}.prefix-definitions ${data:NAME}.spatialjoin-triples.ttl.bz2 ${data:NAME}.ttl.bz2
+CAT_INPUT_FILES  = bzcat -f ${INPUT_FILES}
+SETTINGS_JSON    = { "prefixes-external": [""], "ascii-prefixes-only": false, "parallel-parsing": true, "num-triples-per-batch": 5000000 }
+
+[server]
+PORT               = 7037
+ACCESS_TOKEN       = ${data:NAME}_32673264324
+MEMORY_FOR_QUERIES = 10G
+CACHE_MAX_SIZE     = 5G
+WARMUP_CMD         = curl -s http://localhost:${PORT} -H "Accept: application/qlever-results+json" --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT ?subject ?geometry WHERE { ?subject geo:hasGeometry ?m . ?m geo:asWKT ?geometry } INTERNAL SORT BY ?subject" --data-urlencode "access-token=${server:ACCESS_TOKEN}" --data-urlencode "pinresult=true" --data-urlencode "send=0" | jq .resultsize | xargs printf "Result size: %'d\n"
+
+
+[runtime]
+SYSTEM = native
+
+[ui]
+CONFIG = ohm-planet

src/qlever/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import sys
 from pathlib import Path