asyncval.sh

#!/bin/bash
#SBATCH --nodes=1
#SBATCH --cpus-per-task=5
#SBATCH --mem=50G
#SBATCH --job-name=asyncval
#SBATCH --partition=ai
#SBATCH --account=a_ielab
#SBATCH --gres=gpu:a100:1
#SBATCH --time=24:00:00
#SBATCH -o logs/print_asyncval_generated_train_v2_bm25_hn_pubmedbert.txt
#SBATCH -e logs/error_asyncval_generated_train_v2_bm25_hn_pubmedbert.txt

module load anaconda3
source activate trec_ct

export WANDB_PROJECT=TREC_CT_2023

RUN=generated_train_v2_llama_bm25_hn
tokenizer=microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext

python -m asyncval \
--query_file data/tokenized/${tokenizer}/queries/ct_2022_queries.json \
--candidate_dir data/tokenized/${tokenizer}/corpus/corpus_t_e_s_d \
--ckpts_dir models/${RUN} \
--tokenizer_name_or_path ${tokenizer} \
--qrel_file data/qrels/ct_2022_qrels.txt \
--metrics 'RR(rel=2)' 'nDCG@10' 'P(rel=2)@10' 'Rprec(rel=2)' 'R(rel=2)@1000' \
--output_dir asyncval_runs/${RUN} \
--report_to wandb \
--depth 1000 \
--per_device_eval_batch_size 128 \
--q_max_len 256 \
--p_max_len 512 \
--write_run trec \
--write_embeddings True \
--fp16 \
--cache_dir cache