From 03de8d989dbcd7e7b8fdbc174e6559dc4071edb7 Mon Sep 17 00:00:00 2001
From: Ben Sowell <ben@aryn.ai>
Date: Tue, 28 Jan 2025 13:15:06 -0800
Subject: [PATCH] Explicitly add tiktoken and relock.

---
 lib/sycamore/poetry.lock    |  2 +-
 lib/sycamore/pyproject.toml |  1 +
 poetry.lock                 | 11 ++++++-----
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/sycamore/poetry.lock b/lib/sycamore/poetry.lock
index 7e371cf67..11b4f1b70 100644
--- a/lib/sycamore/poetry.lock
+++ b/lib/sycamore/poetry.lock
@@ -9976,4 +9976,4 @@ weaviate = ["weaviate-client"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "af24062398383cb5a1767628913027e3842e09d4241928d3e853265f273d24d0"
+content-hash = "ec9791de75120499fa5627b69728ed2bda2d84adc76e4d92e216ae381b9356ec"
diff --git a/lib/sycamore/pyproject.toml b/lib/sycamore/pyproject.toml
index 8acf15181..83071196f 100644
--- a/lib/sycamore/pyproject.toml
+++ b/lib/sycamore/pyproject.toml
@@ -88,6 +88,7 @@ nltk = { version = "^3.9.0", optional = true }
 
 # LLM Dependencies
 anthropic = { version = "^0.42.0", optional = true }
+tiktoken = "^0.8.0"
 
 [tool.poetry.group.test.dependencies]
 flake8 = "4.0.1"
diff --git a/poetry.lock b/poetry.lock
index f6b7f6728..85c78f0d9 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
 
 [[package]]
 name = "aiobotocore"
@@ -5852,13 +5852,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
 
 [[package]]
 name = "openai"
-version = "1.59.9"
+version = "1.60.2"
 description = "The official Python library for the openai API"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "openai-1.59.9-py3-none-any.whl", hash = "sha256:61a0608a1313c08ddf92fe793b6dbd1630675a1fe3866b2f96447ce30050c448"},
-    {file = "openai-1.59.9.tar.gz", hash = "sha256:ec1a20b0351b4c3e65c6292db71d8233515437c6065efd4fd50edeb55df5f5d2"},
+    {file = "openai-1.60.2-py3-none-any.whl", hash = "sha256:993bd11b96900b9098179c728026f016b4982ded7ee30dfcf4555eab1171fff9"},
+    {file = "openai-1.60.2.tar.gz", hash = "sha256:a8f843e10f2855713007f491d96afb2694b11b5e02cb97c7d01a0be60bc5bb51"},
 ]
 
 [package.dependencies]
@@ -9244,7 +9244,7 @@ nanoid = "^2.0.0"
 neo4j = {version = "^5.21.0", optional = true}
 nltk = {version = "^3.9.0", optional = true}
 numpy = "<2.0.0"
-openai = "^1.40.2"
+openai = "^1.60.2"
 opensearch-py = {version = "^2.3.1", optional = true}
 overrides = "^7.7.0"
 paddleocr = {version = "^2.8.1", optional = true}
@@ -9267,6 +9267,7 @@ setuptools = ">70.0.0"
 sphinx = {version = "^7.2.5", optional = true}
 structlog = "^24.2.0"
 tenacity = "^8.4.1"
+tiktoken = "^0.8.0"
 timm = {version = "^0.9.12", optional = true}
 torch = {version = "^2.3.0", optional = true}
 torchvision = {version = "^0.18.1", optional = true}