Add 20 questions eval

evals/elsuite/twenty_questions/eval.py

@@ -0,0 +1,204 @@
+import logging
+import random
+import re
+from typing import Any, Dict, List, Optional, Union
+
+import evals
+import evals.metrics
+from evals.api import CompletionFn
+from evals.elsuite.twenty_questions.utils import PROMPTS, generate_task_state_for
+from evals.eval import SolverEval
+from evals.record import Recorder
+from evals.registry import registry
+from evals.solvers.human_cli_solver import HumanCliSolver
+from evals.solvers.solver import Solver
+from evals.solvers.utils import maybe_wrap_with_solver
+from evals.task_state import Message
+
+logger = logging.getLogger(__name__)
+WORD_PATTERN = r"\[GUESS (.*?)\]"
+
+
+class TwentyQuestions(SolverEval):
+    def __init__(
+        self,
+        completion_fns: List[CompletionFn],
+        samples_jsonl: str,
+        gamemaster_spec: str,
+        max_questions: int = 20,
+        max_replies: int = 40,
+        num_shortlist_items: int = 20,
+        shortlist_variant: bool = False,
+        seed: int = 222024,
+        n_samples: Optional[int] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(completion_fns, seed=seed, *args, **kwargs)
+
+        self.samples_jsonl = samples_jsonl
+        self.gamemaster_solver = maybe_wrap_with_solver(
+            registry.make_completion_fn(gamemaster_spec)
+        )
+        self.max_questions = max_questions
+
+        if max_replies < max_questions:
+            logger.warn(
+                f"max_replies ({max_replies}) is less than max_questions ({max_questions}). Setting max_replies to {max_questions + 20}"
+            )
+        self.max_replies = max_replies if max_replies > max_questions else max_questions + 20
+        self.num_shortlist_items = num_shortlist_items
+        self.shortlist_variant = shortlist_variant
+
+        self.n_samples = n_samples
+        self.rng = random.Random(seed)
+
+    def eval_sample(self, solver: Solver, sample: Dict, rng: random.Random) -> Dict[str, Any]:
+        assert "word" in sample, "Sample must contain 'word' field"
+        assert "difficulty" in sample, "Sample must contain 'difficulty' field"
+
+        if not isinstance(solver, HumanCliSolver):
+            logging.info(f"Running sample: {sample['word']}")
+
+        # Generate the shortlist for the current sample if applicable.
+        if self.shortlist_variant:
+            assert self.num_shortlist_items <= len(
+                self.shortlist
+            ), "Number of shortlist items must be less than or equal to the total number of samples."
+            shortlist_for_sample = rng.sample(self.shortlist, self.num_shortlist_items)
+            if sample["word"] not in shortlist_for_sample:
+                random_index = rng.randint(0, len(shortlist_for_sample) - 1)
+                shortlist_for_sample[random_index] = sample["word"]
+        else:
+            shortlist_for_sample = None
+        response = self._conversation_loop(solver, sample, shortlist_for_sample)
+
+        return response
+
+    def run(self, recorder: Recorder) -> Dict[str, Union[float, int]]:
+        samples = self.get_samples()
+        self.rng.shuffle(samples)
+        samples = samples[: self.n_samples] if self.n_samples else samples
+
+        if self.shortlist_variant:
+            self.shortlist = [sample["word"] for sample in samples]
+
+        self.eval_all_samples(recorder, samples)
+        events = recorder.get_events("match")
+
+        scores = [event.data["score"] for event in events]
+        num_guesses = [event.data["num_guesses"] for event in events]
+        num_questions = [event.data["num_questions"] for event in events]
+        num_violations = [event.data["num_violations"] for event in events]
+        num_gamemaster_refusals = [event.data["num_gamemaster_refusals"] for event in events]
+        incorrect_guesses = [event.data["incorrect_guesses"] for event in events]
+        word_difficulties = [event.data["word_difficulty"] for event in events]
+
+        return {
+            "score": sum(scores) / len(scores),
+            "accuracy": evals.metrics.get_accuracy(events),
+            "bootstrap_std": evals.metrics.get_bootstrap_accuracy_std(events),
+            "average_num_guesses": sum(num_guesses) / len(num_guesses),
+            "average_num_questions": sum(num_questions) / len(num_questions),
+            "average_num_violations": sum(num_violations) / len(num_violations),
+            "average_num_gamemaster_refusals": sum(num_gamemaster_refusals)
+            / len(num_gamemaster_refusals),
+            "average_num_incorrect_guesses": sum((len(ig) for ig in incorrect_guesses))
+            / len(incorrect_guesses),
+            "average_word_difficulty": sum(word_difficulties) / len(word_difficulties),
+        }
+
+    def _conversation_loop(
+        self, solver: Solver, sample: Dict, shortlist: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """Maintains a conversation between the guesser and the gamemaster until the maximum number of questions is reached, or until a correct guess is made.
+
+        Args:
+            solver (Solver): any compatible solver, instantiated for the current sample.
+            sample (Dict): current sample – one word to guess, and its associated difficulty.
+
+        Returns:
+            Dict[str, Any]: a dictionary containing the final result and metrics of the conversation.
+        """
+
+        metrics = {
+            "num_guesses": 0,
+            "num_questions": 0,
+            "num_violations": 0,
+            "num_guesser_replies": 0,  # num_guesses + num_questions + num_violations
+            "num_gamemaster_refusals": 0,
+            "incorrect_guesses": [],
+        }
+        conversation = []
+
+        # Contains fall-back condition to avoid infinite loops for solvers which never output questions.
+        while (
+            metrics["num_questions"] < self.max_questions
+            and metrics["num_guesser_replies"] < self.max_replies
+        ):
+            task_state = generate_task_state_for(
+                "guesser", conversation, max_questions=self.max_questions, shortlist=shortlist
+            )
+            guesser_response = solver(task_state)
+            conversation += [Message(content=guesser_response.output, role="guesser")]
+            metrics["num_guesser_replies"] += 1
+
+            # Check if guess made:
+            match = re.search(WORD_PATTERN, guesser_response.output)
+            if match is not None:
+                metrics["num_guesses"] += 1
+                guess = match.group(1)
+                if guess.lower() == sample["word"].lower():
+                    response = {
+                        "correct": True,
+                        "score": self.max_questions - metrics["num_questions"],
+                        "expected": sample["word"],
+                        "word_difficulty": sample["difficulty"],
+                        "picked": guess,
+                        "num_guesses": metrics["num_guesses"],
+                        "num_questions": metrics["num_questions"],
+                        "num_violations": metrics["num_violations"],
+                        "num_gamemaster_refusals": metrics["num_gamemaster_refusals"],
+                        "incorrect_guesses": metrics["incorrect_guesses"],
+                    }
+                    evals.record.record_match(**response)
+                    return response
+                else:
+                    metrics["incorrect_guesses"] += [guess]
+                    conversation += [
+                        Message(
+                            content=PROMPTS["incorrect_guess"].format(guess=guess), role="system"
+                        )
+                    ]
+                    continue
+            elif "?" in guesser_response.output.strip():
+                metrics["num_questions"] += 1
+            else:  # Neither guess nor question.
+                # TODO: Maybe make the guesser retry here?
+                logger.warn(
+                    f"Rule violation, no guess or question in output: {guesser_response.output}"
+                )
+                metrics["num_violations"] += 1
+                conversation += [Message(content=PROMPTS["rule_violation"], role="system")]
+                continue
+
+            task_state = generate_task_state_for("gamemaster", conversation, sample["word"])
+            gamemaster_response = self.gamemaster_solver(task_state)
+            conversation += [Message(content=gamemaster_response.output, role="gamemaster")]
+            if gamemaster_response.output.lower() == "skip":
+                metrics["num_gamemaster_refusals"] += 1
+
+        logger.info(f"Ran out of questions for word: {sample['word']}")
+        response = {
+            "correct": False,
+            "score": 0,
+            "expected": sample["word"],
+            "word_difficulty": sample["difficulty"],
+            "num_guesses": metrics["num_guesses"],
+            "num_questions": metrics["num_questions"],
+            "num_violations": metrics["num_violations"],
+            "num_gamemaster_refusals": metrics["num_gamemaster_refusals"],
+            "incorrect_guesses": metrics["incorrect_guesses"],
+        }
+        evals.record.record_match(**response)
+        return response

evals/elsuite/twenty_questions/readme.md

@@ -0,0 +1,82 @@
+# 20 Questions
+
+This eval tests models' ability to generate and iterate over hypotheses by playing the game of "20 questions". In 20 questions, one of the players – the "gamemaster" – thinks of a word (in our case a noun) and the other player needs to guess it. To help them guess, the player can ask up to 20 yes-or-no questions, which the gamemaster must answer.
+
+## Usage
+Run with:
+```bash
+# Standard variant.
+oaieval <solver> twenty_questions.full
+
+# Shortlist variant.
+oaieval <solver> twenty_questions.shortlist.full
+```
+
+Where the solver can be any generation solver in `evals/registry/solvers/defaults.yaml`, e.g. `generation/cot/gpt-3.5-turbo-16k`, or the chain-of-thought solvers in `evals/registry/solvers/twenty_questions.yaml`.
+
+## Evaluation process
+We run a dialogue loop between two models for each sample: the evaluated model and the "gamemaster". By default, the gamemaster is gpt-4-turbo-preview – but this can be updated by specifying a different solver in `evals/registry/evals/twenty_questions.yaml`. 
+
+The dialogue continues until the word is guessed correctly, or until 20 questions have been asked, whichever comes first. We also terminate conversations that last longer than 40 replies, to ensure that models which do not ask questions don't have infinite conversations. Both the maximum questions and the maximum replies can be controlled from the eval YAML file.
+
+## Task State
+The task state can be found in `twenty_questions/utils.py`; it reads:
+```
+You are about to play the game '20 questions'. The other player has thought of a noun which you should try to guess. You can ask 20 yes/no questions, to which they will answer 'yes', 'no', or 'skip' (if they cannot answer your question). You should try to guess the word as soon as possible, using the least amount of questions. To guess a word, type [GUESS <word>] – for example to guess the word 'sibling', output [GUESS sibling]. Your score will be 0 if you do not guess correctly, and {max_questions} minus the number of questions you asked if you guess correctly. Start by asking your first question.
+```
+
+## Prompts
+See `twenty_questions/utils.py` to review/adjust the prompts used in this eval.
+
+## Datasets
+
+We use a dataset of 207 words, 177 of which were from [this lexicon](https://github.com/mounicam/lexical_simplification), annotated by our team with a difficulty category. This dataset comprises:
+- 47 words rated “easy”, e.g. ear, father, potato;
+- 91 words rated “medium”, e.g. cloth, hike, discount;
+- 69 words rated “hard”, e.g. prosperity, gland, philosopher;
+
+In addition to these common nouns, we include 30 proper nouns such as “Sherlock Holmes,” “The Beatles,” “Titanic,” and “Starbucks”, which span the easy and medium difficulties.
+
+## Metrics
+We measure the score each model achieves, defined as `score = max_questions - questions_asked`. We also track the win-rate, i.e. the % of samples the model guesses correctly. Auxiliary metrics such as average number of average number of questions asked, average number of incorrect guesses, and average number of gamemaster refusals (i.e. situations where the gamemaster says 'skip') are also tracked.
+
+
+## Variants
+
+We run two main variants of this evaluation:
+- **standard**: the main variant
+- **shortlist**: an easier variant where the evaluated model sees a shortlist of words in its system prompt. The word the gamemaster has selected is part of the list. In this variant, the evaluated model effectively has to narrow down the pool of candidate words until it finds the answer.
+
+## Token Usage Estimates
+
+Below is a rough estimate of the total number of tokens consumed by some variations the eval, including both input and output tokens:
+
+Variant | Model | Solver | Prompt tokens | Completion tokens | Total tokens
+| --- | --- | --- | --- | --- | --- |
+standard | direct | gpt-4-turbo-preview | 2,502,067 | 52,879 | 2,554,946
+standard | direct | gpt-4-base | 13,197,212 | 2,814,623 | 16,011,835
+standard | direct | gpt-3.5-turbo | 2,670,866 | 57,917 | 2,728,783
+standard | cot | gpt-4-turbo-preview | 73,765,861 | 1,881,455 | 75,647,316
+standard | cot | gpt-4-base | 51,777,817 | 6,397,472 | 58,175,289
+standard | cot | gpt-3.5-turbo | 38,236,500 | 199,831 | 38,436,331
+standard | cot | llama-2-70b | 6,785,634 | 581,421 | 7,367,055
+standard | cot | mixtral-8x7b-instruct | 175,956,903 | 5,327,393 | 181,284,296
+shortlist | direct | gpt-4-turbo-preview | 1,237,172 | 28,351 | 1,265,523
+shortlist | direct | gpt-4-base | 11,034,903 | 2,133,487 | 13,168,390
+shortlist | direct | gpt-3.5-turbo | 1,704,154 | 36,356 | 1,740,510
+shortlist | cot | gpt-4-turbo-preview | 10,951,215 | 545,945 | 11,497,160
+shortlist | cot | gpt-4-base | 45,591,363 | 596,429 | 46,187,792
+shortlist | cot | gpt-3.5-turbo | 19,798,263 | 165,731 | 19,963,994
+shortlist | cot | llama-2-70b | 5,980,667 | 528,879 | 6,509,546
+shortlist | cot | mixtral-8x7b-instruct | 143,646,924 | 4,315,806 | 147,962,730
+
+
+## Version History
+v0: Initial version released
+
+
+## Contribution statement
+
+Eval design, implementation, and results evaluation were primarily conducted by Andrei Alexandru with contributions from Dane Sherburn, under the guidance of (alphabetically by last-name) Steven Adler, James Aung, and Chan Jun Shern who scoped and managed the broader research project, including input on evaluation design, results analysis, and interpretation.
+
+