Skip to content

Commit

Permalink
Merge branch 'master' into parseAllUpdateQueries
Browse files Browse the repository at this point in the history
  • Loading branch information
Qup42 committed Oct 22, 2024
2 parents 7fcee75 + aec57fe commit ffb0182
Show file tree
Hide file tree
Showing 120 changed files with 8,375 additions and 2,587 deletions.
65 changes: 65 additions & 0 deletions .github/workflows/sparql-conformance-uploader.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Upload conformance tests result

on:
workflow_run:
workflows: [sparql-conformance]
types:
- completed

jobs:
upload:
env:
SERVER_URL: https://qlever.cs.uni-freiburg.de/sparql-conformance-uploader
API_KEY: ${{ secrets.SPARQL_CONFORMANCE_TOKEN }}
runs-on: ubuntu-latest
if: github.event.workflow_run.conclusion == 'success'
steps:
- name: 'Download artifact'
uses: actions/github-script@v6
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "conformance-report"
})[0];
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/conformance-report.zip', Buffer.from(download.data));
- run: unzip conformance-report.zip
# Read the metadata into environment variables.
- name: "Read github event"
run: echo "github_event=`cat event`" >> $GITHUB_ENV
- name: "Read PR number"
run: echo "pr_number=`cat pr`" >> $GITHUB_ENV
- name: "Read Github Ref"
run: echo "original_github_ref=`cat github_ref`" >> $GITHUB_ENV;
- name: "Read Github SHA"
run: echo "commit_sha=`cat sha`" >> $GITHUB_ENV;
- name: "Read Github Repository"
run: echo "original_github_repository=`cat github_repository`" >> $GITHUB_ENV;
- name: "Submit data to server"
run: |
response=$(curl -s -o temp_response.txt -w "%{http_code}" \
-H "x-api-key: $API_KEY" \
-H "event: ${{ env.github_event }}" \
-H "sha: ${{ env.commit_sha }}" \
-H "pr-number: ${{ env.pr_number }}" \
-F "file=@${{env.commit_sha}}.json.bz2" \
$SERVER_URL/upload)
echo "Server response:"
cat temp_response.txt
echo "HTTP Status: $response"
if [ "$response" -gt 200 ]; then
echo "Server did not respond with status 200. Failing the workflow."
exit 1
fi
86 changes: 86 additions & 0 deletions .github/workflows/sparql-conformance.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
name: sparql-conformance

on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
merge_group:

jobs:
build:
env:
compiler: clang
compiler-version: 16
build-type: Release
cmake-flags: "-DCMAKE_C_COMPILER=clang-16 -DCMAKE_CXX_COMPILER=clang++-16"
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
with:
submodules: "recursive"
path: qlever-code
- name: Checkout sparql-test-suite-files
uses: actions/checkout@v3
with:
repository: "w3c/rdf-tests"
path: sparql-test-suite
- name: Checkout qlever-test-suite
uses: actions/checkout@v3
with:
repository: "ad-freiburg/sparql-conformance"
path: qlever-test-suite
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install python dependencies
run: |
python -m pip install --upgrade pip
pip install requests
pip install rdflib
- name: Install dependencies
uses: ./qlever-code/.github/workflows/install-dependencies-ubuntu
- name: Install compiler
uses: ./qlever-code/.github/workflows/install-compiler-ubuntu
with:
compiler: "clang"
compiler-version: "16"
- name: Create build directory
run: mkdir ${{github.workspace}}/qlever-code/build
- name: Configure CMake
run: cmake -S ${{github.workspace}}/qlever-code/ -B ${{github.workspace}}/qlever-code/build ${{env.cmake-flags}} -DCMAKE_BUILD_TYPE=${{env.build-type}} -DLOGLEVEL=INFO -DUSE_PARALLEL=false
- name: Build IndexBuilderMain
run: cmake --build ${{github.workspace}}/qlever-code/build --target IndexBuilderMain --config ${{env.build-type}} -- -j $(nproc)
- name: Build ServerMain
run: cmake --build ${{github.workspace}}/qlever-code/build --target ServerMain --config ${{env.build-type}} -- -j $(nproc)
- name: Execute test suite
run: |
cd qlever-test-suite
python testsuite.py config http://0.0.0.0 7001 ${{github.workspace}}/sparql-test-suite/sparql/sparql11/ ${{github.workspace}}/qlever-code/build/ localhost sparql sparql
python testsuite.py extract
python testsuite.py ${{ github.sha }}
cd ..
- name: Save workflow information
# Note: If you change any of the filenames here, you also have to change them in `upload-conformance.yml`
run : |
mkdir -p conformance-report
echo ${{ github.event_name }} > ./conformance-report/event
echo ${{ github.event.number }} > ./conformance-report/pr
echo ${{ github.repository }} > ./conformance-report/github_repository
echo ${GITHUB_REF} > ./conformance-report/github_ref
- name: Save SHA and conformance report if it is a master commit.
if: github.event_name == 'push'
run : |
echo ${{github.sha}} > ./conformance-report/sha
mv ${{ github.workspace}}/qlever-test-suite/results/${{ github.sha }}.json.bz2 conformance-report/${{ github.sha }}.json.bz2
- name: Save SHA and conformance report if it is a PR.
if: github.event_name == 'pull_request'
run : |
echo ${{github.event.pull_request.head.sha}} > ./conformance-report/sha
mv ${{ github.workspace}}/qlever-test-suite/results/${{ github.sha }}.json.bz2 conformance-report/${{ github.event.pull_request.head.sha }}.json.bz2
- name: Upload coverage artifact
uses: actions/upload-artifact@v3
with:
name: conformance-report
path: conformance-report/
29 changes: 22 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.16)
cmake_minimum_required(VERSION 3.27)
project(QLever C CXX)

# C/C++ Versions
Expand Down Expand Up @@ -244,6 +244,18 @@ FetchContent_Declare(
GIT_TAG 93ac3a4f9ee7792af399cebd873ee99ce15aed08 # 2024-05-16
)

################################
# S2 Geometry
################################
SET (BUILD_TESTS OFF CACHE BOOL "no tests for s2")
FetchContent_Declare(
s2
GIT_REPOSITORY https://github.com/google/s2geometry.git
GI_TAG 5b5eccd54a08ae03b4467e79ffbb076d0b5f221e #version 0.11.1
SYSTEM
)


if (USE_PARALLEL)
include(FindOpenMP)
if (OPENMP_FOUND)
Expand Down Expand Up @@ -301,8 +313,13 @@ FetchContent_Declare(
################################
FetchContent_Declare(
fsst
GIT_REPOSITORY https://github.com/cwida/fsst.git
GIT_TAG c8719ef0aa3740da9685ad2738bb9c8ecc327944
#GIT_REPOSITORY https://github.com/cwida/fsst.git
#GIT_TAG c8719ef0aa3740da9685ad2738bb9c8ecc327944
#FSST Currently has a struct name `Encoder` that leads to ODR violations with the `Encoder`class of S2.
# I (joka921) have filed bug reports for both of them and a PR for fsst, but for now we use our own fork
# of Fsst.
GIT_REPOSITORY https://github.com/joka921/fsst.git
GIT_TAG 43fb2d1756f5a5d0e85e765c7b51f5e3be8cc83f
)


Expand All @@ -321,8 +338,9 @@ FetchContent_Declare(
################################
# Apply FetchContent
################################
FetchContent_MakeAvailable(googletest ctre abseil re2 stxxl fsst)
FetchContent_MakeAvailable(googletest ctre abseil re2 stxxl fsst s2)
# Disable some warnings in RE2, STXXL, and GTEST
target_compile_options(s2 PRIVATE -Wno-sign-compare -Wno-unused-parameter -Wno-class-memaccess -Wno-comment -Wno-redundant-move -Wno-unknown-warning-option -Wno-maybe-uninitialized -Wno-class-memaccess)
target_compile_options(re2 PRIVATE -Wno-unused-parameter)
target_compile_options(stxxl PRIVATE -Wno-deprecated-declarations)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
Expand Down Expand Up @@ -421,8 +439,5 @@ target_precompile_headers(ServerMain REUSE_FROM engine)
add_executable(VocabularyMergerMain src/VocabularyMergerMain.cpp)
qlever_target_link_libraries(VocabularyMergerMain index ${CMAKE_THREAD_LIBS_INIT})

add_executable(PermutationExporterMain src/index/PermutationExporterMain.cpp)
qlever_target_link_libraries(PermutationExporterMain index ${CMAKE_THREAD_LIBS_INIT})

add_executable(PrintIndexVersionMain src/PrintIndexVersionMain.cpp)
qlever_target_link_libraries(PrintIndexVersionMain util)
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENV LC_CTYPE C.UTF-8
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:mhier/libboost-latest
RUN apt-get update && apt-get install -y software-properties-common wget && add-apt-repository -y ppa:mhier/libboost-latest
RUN wget https://apt.kitware.com/kitware-archive.sh && chmod +x kitware-archive.sh &&./kitware-archive.sh

FROM base as builder
RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git libjemalloc-dev ninja-build libzstd-dev libssl-dev libboost1.81-dev libboost-program-options1.81-dev libboost-iostreams1.81-dev libboost-url1.81-dev
Expand Down
Loading

0 comments on commit ffb0182

Please sign in to comment.