Skip to content

Commit

Permalink
Make sure that index_cmd fails if ulimit -Sn fails
Browse files Browse the repository at this point in the history
  • Loading branch information
Hannah Bast committed Feb 13, 2025
1 parent 73b20e3 commit 7cb6cc3
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 6 deletions.
6 changes: 2 additions & 4 deletions src/qlever/commands/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,10 +235,8 @@ def execute(self, args) -> bool:
# If the total file size is larger than 10 GB, set ulimit (such that a
# large number of open files is allowed).
total_file_size = get_total_file_size(shlex.split(args.input_files))
if args.ulimit is not None:
index_cmd = f"ulimit -Sn {args.ulimit}; {index_cmd}"
elif total_file_size > 1e10:
index_cmd = f"ulimit -Sn 1048576; {index_cmd}"
ulimit = args.ulimit if args.ulimit is not None else 1048576
index_cmd = f"ulimit -Sn {ulimit} && {index_cmd}"

# Run the command in a container (if so desired).
if args.system in Containerize.supported_systems():
Expand Down
2 changes: 1 addition & 1 deletion src/qlever/qlever_old.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ def action_index(self, only_show=False):
self.config["index"]["file_names"].split()
)
if total_file_size > 10:
cmdline = f"ulimit -Sn 524288 && {cmdline}"
cmdline = f"ulimit -Sn 1048576; {cmdline}"

# If we are using Docker, run the command in a Docker container.
# Here is how the shell script does it:
Expand Down
2 changes: 1 addition & 1 deletion test/qlever/commands/test_index_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ def test_execute_total_file_size_greater_than_ten_gb(

# Assertions
expected_index_cmd = (
f"ulimit -Sn 524288 && {args.cat_input_files} | {args.index_binary}"
f"ulimit -Sn 1048576; {args.cat_input_files} | {args.index_binary}"
f" -i {args.name} -s {args.name}.settings.json"
f" -F {args.format} -f -"
f" | tee {args.name}.index-log.txt"
Expand Down

0 comments on commit 7cb6cc3

Please sign in to comment.