Skip to content

Commit

Permalink
Refactor workflows and hopefully fix them
Browse files Browse the repository at this point in the history
  • Loading branch information
Hannah Bast committed Sep 17, 2023
1 parent beb5e0b commit f64cbaf
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 86 deletions.
30 changes: 13 additions & 17 deletions .github/workflows/end-to-end-test-macos.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ jobs:
os: [macos-12]

steps:
- name: Checkout script and install dependencies
- uses: actions/checkout@v3
- name: Install required tools and Python packages
with:
path: qlever-control
run: |
brew install llvm@16
brew install conan@2
Expand All @@ -34,37 +36,31 @@ jobs:
path: ~/.conan2
key: ${{runner.os}}-build-${{env.cache-name}}-${{hashFiles('conanfile.txt')}}

- name: Install latest version of QLever
working-directory: ${{github.workspace}}
- name: Download QLever binaries and install dependencies
run: |
# open -a docker
# docker pull adfreiburg/qlever
# docker run adfreiburg/qlever --help > /dev/null
git clone https://github.com/ad-freiburg/qlever.git qlever-code
mkdir -p qlever-code/build
cd qlever-code/build
mkdir -p qlever-code/build && cd $_
conan install .. -pr:b=../conanprofiles/clang-16-macos -pr:h=../conanprofiles/clang-16-macos -of=. --build=missing
cd ../..
# Download the QLever binaries.
mkdir qlever-binaries && cd $_
wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/macos-12/ServerMain
wget -q https://ad-research.cs.uni-freiburg.de/downloads/qlever/macos-12/IndexBuilderMain
chmod 755 ServerMain IndexBuilderMain
- name: Format and compile check
working-directory: ${{github.workspace}}/qlever-control
run: |
flake8 qlever.py
python3 -m py_compile qlever.py
flake8 qlever
python3 -m py_compile qlever
- name: Test actions for olympics dataset, without Docker
working-directory: ${{github.workspace}}
working-directory: ${{github.workspace}}/qlever-indices/olympics
run: |
export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
source qlever-code/build/conanrun.sh
export PATH="$PATH:$(pwd)/qlever-binaries"
ServerMain --help > /dev/null
IndexBuilderMain --help > /dev/null
mkdir olympics.without-docker && cd $_
cp -a ../Qleverfiles/Qleverfile.olympics Qleverfile.NEW
unbuffer python3 ../qlever.py get-data docker.USE_DOCKER=false index index-stats
unbuffer python3 ../qlever.py docker.USE_DOCKER=false start status stop start stop status start status stop
unbuffer qlever setup-config olympics
unbuffer qlever get-data docker.USE_DOCKER=false index index-stats
unbuffer qlever docker.USE_DOCKER=false start status stop start stop status start status stop
ls -lh
46 changes: 23 additions & 23 deletions .github/workflows/end-to-end-test-ubuntu.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ jobs:
os: [ubuntu-22.04]

steps:
- name: Checkout script and install dependencies
- uses: actions/checkout@v3
- name: Install required tools and Python packages
with:
path: qlever-control
run: |
sudo apt update
sudo apt install unzip flake8 expect
pip3 install psutil termcolor
- name: Install latest version of QLever (download Docker image and compile natively)
working-directory: ${{github.workspace}}
- name: Download QLever binaries and docker image and install dependencies
run: |
# Download Docker image.
docker pull adfreiburg/qlever
Expand All @@ -43,36 +44,35 @@ jobs:
./IndexBuilderMain --help > /dev/null
- name: Format and compile check
working-directory: ${{github.workspace}}/qlever-control
run: |
flake8 qlever.py
python3 -m py_compile qlever.py
flake8 qlever
python3 -m py_compile qlever
- name: Test actions for olympics dataset, with Docker
working-directory: ${{github.workspace}}
working-directory: ${{github.workspace}}/qlever-indices/olympics.1
run: |
mkdir olympics.with-docker && cd $_
cp -a ../Qleverfiles/Qleverfile.olympics Qleverfile.NEW
unbuffer python3 ../qlever.py get-data docker.USE_DOCKER=true index index-stats start status stop status
export PATH="$PATH:$(pwd)/qlever-control"
unbuffer qlever setup-config olympics
unbuffer qlever get-data docker.USE_DOCKER=true index index-stats start status stop status
ls -lh
- name: Test actions for olympics dataset, without Docker
working-directory: ${{github.workspace}}
working-directory: ${{github.workspace}}/qlever-indices/olympics.2
run: |
export PATH="$PATH:$(pwd)/qlever-binaries"
mkdir olympics.without-docker && cd $_
cp -a ../Qleverfiles/Qleverfile.olympics Qleverfile.NEW
unbuffer python3 ../qlever.py get-data docker.USE_DOCKER=false index index-stats start status stop status
export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
unbuffer qlever setup-config olympics
unbuffer qlever get-data docker.USE_DOCKER=false index index-stats start status stop status
ls -lh
- name: Test actions for olympics dataset, with and without Docker
working-directory: ${{github.workspace}}
working-directory: ${{github.workspace}}/qlever-indices/olympics.3
run: |
export PATH="$PATH:$(pwd)/qlever-binaries"
mkdir olympics.with-and-without-docker && cd $_
cp -a ../Qleverfiles/Qleverfile.olympics Qleverfile.NEW
unbuffer python3 ../qlever.py get-data index index-stats
unbuffer python3 ../qlever.py docker.USE_DOCKER=false start status
unbuffer python3 ../qlever.py docker.USE_DOCKER=true stop status
unbuffer python3 ../qlever.py docker.USE_DOCKER=true start status
unbuffer python3 ../qlever.py docker.USE_DOCKER=false stop status
export PATH="$PATH:$(pwd)/qlever-control:$(pwd)/qlever-binaries"
unbuffer qlever setup-config olympics
unbuffer qlever.py get-data index index-stats
unbuffer qlever docker.USE_DOCKER=false start status
unbuffer qlever docker.USE_DOCKER=true stop status
unbuffer qlever docker.USE_DOCKER=true start status
unbuffer qlever docker.USE_DOCKER=false stop status
ls -lh
93 changes: 47 additions & 46 deletions qlever
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class Actions:
except Exception as e:
log.error(f"Could not read \"Qleverfile\": {e}")
abort_script()
self.name = self.config['DEFAULT']['name']
self.name = self.config['data']['name']
self.yes_values = ["1", "true", "yes"]

# Default values for options that are not mandatory in the Qleverfile.
Expand Down Expand Up @@ -207,14 +207,13 @@ class Actions:
return exit_code == 0

@track_action_rank
def action_setup_config(self, only_show=False):
@staticmethod
def action_setup_config(config_name="default"):
"""
Setup a pre-filled Qleverfile in the current directory.
"""

log.info("{BLUE}Creating a pre-filled Qleverfile{NORMAL}")
if only_show:
return
log.info(f"{BLUE}Creating a pre-filled Qleverfile{NORMAL}")
log.info("")

# If there is already a Qleverfile in the current directory, exit.
Expand All @@ -227,17 +226,11 @@ class Actions:
log.info("")
abort_script()

# Get the name of the desired configuration (default: "default").
if len(sys.argv) == 2:
config = "default"
else:
config = sys.argv[2]

# Get the directory of this script and copy the Qleverfile for `config`
# to the current directory.
script_dir = os.path.dirname(os.path.realpath(__file__))
qleverfile_path = os.path.join(script_dir,
f"Qleverfiles/Qleverfile.{config}")
f"Qleverfiles/Qleverfile.{config_name}")
if not os.path.isfile(qleverfile_path):
log.error(f"File \"{qleverfile_path}\" does not exist")
log.info("")
Expand All @@ -248,10 +241,9 @@ class Actions:
log.error(f"Could not copy \"{qleverfile_path}\""
f" to current directory: {e}")
abort_script()
log.info(f"Created Qleverfile for config \"{config}\""
log.info(f"Created Qleverfile for config \"{config_name}\""
f" in current directory")
log.info("")
sys.exit(0)

@track_action_rank
def action_show_config(self, only_show=False):
Expand All @@ -263,16 +255,14 @@ class Actions:
print(f"{BLUE}Showing the current configuration, including default"
f" values for options that are not set explicitly in the"
f" Qleverfile{NORMAL}")
for section in ['DEFAULT'] + self.config.sections():
for section in self.config.sections():
print()
print(f"[{section}]")
max_option_length = max([len(option) for option in
self.config[section]])
for option in self.config[section]:
if section == "DEFAULT" or \
option not in self.config['DEFAULT']:
print(f"{option.upper().ljust(max_option_length)} = "
f"{self.config[section][option]}")
print(f"{option.upper().ljust(max_option_length)} = "
f"{self.config[section][option]}")

print()

Expand Down Expand Up @@ -746,29 +736,55 @@ def main():
log.error(f"Invalid log level: \"{log_level}\"")
abort_script()

action_names = [_ for _ in dir(Actions) if _.startswith("action_")]
action_names = [_.replace("action_", "") for _ in action_names]
action_names = [_.replace("_", "-") for _ in action_names]
actions = Actions()
# log.info(f"Actions available are: {', '.join(action_names)}")
# Show the log level as string.
# log.info(f"Log level: {logging.getLevelName(log.getEffectiveLevel())}")
# Helper function that executes an action.
def execute_action(actions, action_name, **kwargs):
log.info("")
log.info(f"{BOLD}Action: \"{action_name}\"{NORMAL}")
log.info("")
action = f"action_{action_name.replace('-', '_')}"
try:
getattr(actions, action)(**kwargs)
except ActionException as err:
print(f"{RED}{err}{NORMAL}")
abort_script()
except Exception as err:
line = traceback.extract_tb(err.__traceback__)[-1].lineno
print(f"{RED}Error in Python script (line {line}: {err})"
f", stack trace follows:{NORMAL}")
print()
raise err

# Check if the last argument is "show" (if yes, remember it and remove it).
only_show = True if len(sys.argv) > 1 and sys.argv[-1] == "show" else False
if only_show:
sys.argv = sys.argv[:-1]
log.info("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")

# If `setup-config` is among the command-line arguments, it must the first
# one, followed by at most one more argument.
if "setup-config" in sys.argv:
if sys.argv.index("setup-config") > 1:
log.setLevel(logging.ERROR)
log.error("Action `setup-config` must be the first argument")
abort_script()
if len(sys.argv) > 3:
log.setLevel(logging.ERROR)
log.error("Action `setup-config` must be followed by at most one "
"argument (the name of the desied configuration)")
abort_script()
log.setLevel(logging.INFO)
config_name = sys.argv[2] if len(sys.argv) == 3 else "default"
execute_action(Actions, "setup-config", config_name=config_name)
return

action_names = [_ for _ in dir(Actions) if _.startswith("action_")]
action_names = [_.replace("action_", "") for _ in action_names]
action_names = [_.replace("_", "-") for _ in action_names]
actions = Actions()
# log.info(f"Actions available are: {', '.join(action_names)}")
# Show the log level as string.
# log.info(f"Log level: {logging.getLevelName(log.getEffectiveLevel())}")

# Check if the last argument is "show" (if yes, remember it and remove it).
only_show = True if len(sys.argv) > 1 and sys.argv[-1] == "show" else False
if only_show:
sys.argv = sys.argv[:-1]

# Initalize actions.
# Execute the actions specified on the command line.
Expand All @@ -790,22 +806,7 @@ def main():
f"actions are: {', '.join(action_names)}")
abort_script()
# Execute the action (or only show what would be executed).
log.info("")
log.info(f"{BOLD}Action: \"{action_name}\"{NORMAL}")
log.info("")
action = f"action_{action_name.replace('-', '_')}"
try:
getattr(actions, action)(only_show=only_show)
except ActionException as err:
# line = traceback.extract_tb(err.__traceback__)[-1].lineno
print(f"{RED}{err}{NORMAL}")
abort_script()
except Exception as err:
line = traceback.extract_tb(err.__traceback__)[-1].lineno
print(f"{RED}Error in Python script (line {line}: {err})"
f", stack trace follows:{NORMAL}")
print()
raise err
execute_action(actions, action_name, only_show=only_show)
log.info("")


Expand Down

0 comments on commit f64cbaf

Please sign in to comment.