add eval datasets

Author: brianfitzgerald

cn_eval.out.538931

@@ -0,0 +1,5 @@
++ CONTROLNET_CKPT=/weka/home-brianf/comfy-models/controlnet/canny_8b.safetensors
++ DATASET=/weka/home-brianf/controlnet_eval/canny_eval_dataset
++ echo 'Evaluating controlnet model: /weka/home-brianf/comfy-models/controlnet/canny_8b.safetensors with dataset: /weka/home-brianf/controlnet_eval/canny_eval_dataset'
+Evaluating controlnet model: /weka/home-brianf/comfy-models/controlnet/canny_8b.safetensors with dataset: /weka/home-brianf/controlnet_eval/canny_eval_dataset
++ python evaluate.py --verbose True --input_dataset /weka/home-brianf/controlnet_eval/canny_eval_dataset --model /weka/home-brianf/sd35_models/sd3.5_large.safetensors --model_folder /weka/home-brianf/sd35_models --controlnet_ckpt /weka/home-brianf/comfy-models/controlnet/canny_8b.safetensors --vae /weka/applied-shared/sd3_ref/sd3_vae.safetensors --sampler euler --text_encoder_device cuda

evaluate.py

@@ -116,7 +116,6 @@ def _get_precomputed_cond(sample):
         conditioning = inferencer.get_cond(sample["txt"])
         seed_num = 42
         neg_cond = inferencer.get_cond("")
-        print(f"Sample {i}: prompt: {sample['txt']}")
         sampled_latent = inferencer.do_sampling(
             latent,
             seed_num,

submit_all_evals.sh

@@ -0,0 +1,3 @@
+sbatch submit_eval.sh /weka/home-brianf/comfy-models/controlnet/canny_8b.safetensors /weka/home-brianf/controlnet_eval/canny_eval_dataset
+sbatch submit_eval.sh /weka/home-brianf/comfy-models/controlnet/depth_8b.safetensors /weka/home-brianf/controlnet_eval/depth_eval_dataset
+sbatch submit_eval.sh /weka/home-brianf/comfy-models/controlnet/blur_8b.safetensors /weka/home-brianf/controlnet_eval/blur_eval_dataset

submit_eval.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#SBATCH --job-name=cn_eval
+#SBATCH --output=cn_eval.out.%j
+#SBATCH --error=cn_eval.out.%j
+#SBATCH --nodes=1
+#SBATCH --gpus=1
+#SBATCH --account=sd3
+#SBATCH --partition=p5
+
+set -ex
+
+CONTROLNET_CKPT=$1
+DATASET=$2
+
+echo "Evaluating controlnet model: $CONTROLNET_CKPT with dataset: $DATASET"
+
+python evaluate.py \
+    --verbose True \
+    --input_dataset $DATASET \
+    --model "/weka/home-brianf/sd35_models/sd3.5_large.safetensors" \
+    --model_folder "/weka/home-brianf/sd35_models" \
+    --controlnet_ckpt $CONTROLNET_CKPT \
+    --vae "/weka/applied-shared/sd3_ref/sd3_vae.safetensors" \
+    --sampler "euler" \
+    --text_encoder_device "cuda"