From 55c7991bbc5fc7e1bd226f9362cd36684f067ead Mon Sep 17 00:00:00 2001 From: AkshathRaghav Date: Wed, 15 May 2024 07:23:10 -0400 Subject: [PATCH] Minor Changes for 0.1.0 Signed-off-by: AkshathRaghav --- README.md | 18 ++++++++---------- guide.ipynb | 46 ++++++++++++++++++++++++++++++++++------------ setup.py | 2 +- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 18450ce..837510d 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,7 @@ Importantly, the package supports the generation of **GNBF grammar**, which inte The goal of this package was to overcome the issues faced when using LangChain's output parsers with instruct language models locally. While GPT-4 produces consistent results in returning the correct formats, local models from families like Llama and Mistral would cause parsing errors in my testing chains when I need more than just a single string response. Recently, GrammarFlow was extended to cover more features to help anyone trying to work with LLMs for complex use-cases: multi-grammar generation, regex patterns, etc. -Moreover, GrammarFlow is meant for use-cases with (any kind of) AI Agents, as well as extracting content from text or question-answering problems. This allows it to have an *edge over* batched LLM generation and schema recomposing. The above methods would require *much higher #calls* to an inference function, which will increase the total cost of an iteration if using a paid service like GPT or Gemini. - -Kindly go through [`Remarks!`](https://github.com/e-lab/SyntaxShaper/tree/main?tab=readme-ov-file#remarks) section to get a complete understanding of what we're doing. +Moreover, GrammarFlow is meant for use-cases with (any kind of) AI Agents, as well as extracting content from text or question-answering problems. This allows it to have an *edge over* *batched LLM generation* and *schema recomposing*. These methods, although foolproof, would require *many more calls* to an inference function, which will increase the total cost of a use-case if using a paid service like GPT or Gemini. Kindly go through [`Remarks!`](https://github.com/e-lab/SyntaxShaper/tree/main?tab=readme-ov-file#remarks) section to get a complete understanding of what this means. > Please reach out to `araviki[at]purdue[dot]edu` or open an issue on Github if you have any questions or inquiry related to GrammarFlow and its usage. @@ -43,9 +41,9 @@ Kindly go through [`Remarks!`](https://github.com/e-lab/SyntaxShaper/tree/main?t GrammarFlow was tested against popular LLM datasets, with a focus on constraining model outputs. The goal was to ensure that the final parsed output matched both the *structure and data types* of the ground truth. Some basic tests using Google's [BIG-bench](https://github.com/google/BIG-bench): - - [Logic Grid Puzzle](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/logic_grid_puzzle/) - - [StrategyQA](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/strategyqa/task.json) - - [ReasoningAboutColors](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/reasoning_about_colored_objects/) +- [Logic Grid Puzzle](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/logic_grid_puzzle/) - Nested Objects with lists. +- [StrategyQA](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/strategyqa/task.json) - Nested Objects with str/int. +- [ReasoningAboutColors](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/reasoning_about_colored_objects/) - Single object with list. ``` |------------------------------------------------------------------------------------------------------| @@ -59,9 +57,9 @@ Some basic tests using Google's [BIG-bench](https://github.com/google/BIG-bench) +------------------------------------------------------------------------------------------------------| ``` -For more complex tests: - - [PrOntoQA](https://github.com/Ber666/llm-reasoners/blob/main/examples/prontoqa/data/345hop_random_true.json) - Chain of Thought reasoning, with randomly-scattered supporting facts in prompt. Taken from [llm-reasoners](https://github.com/Ber666/llm-reasoners/). Tests the ability to place specific reasoning statements in the right place. - - [HotPotQA](http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_train_v1.1.json) - Multi-hop questions, with strong supervision for supporting facts. Integrated within the first ReAct prompting paper's [code](https://github.com/ysymyth/ReAct). Incremental steps, leading to large prompts. +More complex tests: +- [PrOntoQA](https://github.com/Ber666/llm-reasoners/blob/main/examples/prontoqa/data/345hop_random_true.json) - Chain of Thought reasoning, with randomly-scattered supporting facts in prompt. Taken from [llm-reasoners](https://github.com/Ber666/llm-reasoners/). **Tests the ability to place specific reasoning statements in the right place.** +- [HotPotQA](http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_train_v1.1.json) - Multi-hop questions, with strong supervision for supporting facts. Integrated within the first ReAct prompting paper's [code](https://github.com/ysymyth/ReAct). **Incremental steps, leading to large prompts.** ``` |------------------------------------------------------------------------------+-------------------------| @@ -260,7 +258,7 @@ We appreciate it if you would please cite this repo if you found the library use publisher = {GitHub}, journal = {GitHub repository}, howpublished = {\url{https://github.com/e-lab/GrammarFlow}}, - version = {0.1.1} + version = {0.1.0} } ``` diff --git a/guide.ipynb b/guide.ipynb index 2d7f922..ba9b9c9 100644 --- a/guide.ipynb +++ b/guide.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 14, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -24,7 +24,7 @@ "True" ] }, - "execution_count": 15, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -44,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -86,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -135,7 +135,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -189,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 43, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -506,13 +506,14 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 52, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "\n", "```\n", "{\n", "\"Project\": {\n", @@ -606,7 +607,8 @@ "{'Project': {'name': 'Multimodal Document Understanding Project', 'description': 'This project aims to develop a system that can understand and analyze documents using multiple modes of input such as text, images, and audio.', 'project_url': 'https://www.multimodalproject.com', 'team_members': {'TeamMember': [{'name': 'John Doe', 'role': 'Project Manager'}, {'name': 'Jane Smith', 'role': 'Lead Developer'}, {'name': 'Michael Johnson', 'role': 'Data Scientist'}]}, 'task': {'title': 'Data Collection and Preprocessing', 'description': 'Collect and preprocess text, image, and audio data for training the multimodal document understanding system.', 'assigned_to': 'Michael Johnson', 'due_date': typing.List[ForwardRef('2022-10-15')]}}}\n", "-------------\n", "{'before': 69, 'after': 252, 'factor': '2.7x'}\n", - "-------------\n" + "-------------\n", + "\n" ] } ], @@ -653,6 +655,26 @@ " log(manager.inflation_rate())" ] }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "typing.List[ForwardRef('2022-10-15')]" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response.Project.task.due_date" + ] + }, { "cell_type": "code", "execution_count": 29, diff --git a/setup.py b/setup.py index 31aa0da..a2326e1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="GrammarFlow", - version="0.1.1", + version="0.1.0", author="AkshathRaghav", author_email="araviki@purdue.edu", description="Ensuring parsability of LLM responses in agent chains",