push to hub

Author: shahules786

finetune-embedding/data_creation.ipynb

@@ -563,6 +563,14 @@
     "len(dataset.filter(lambda ex: ex[\"ragas_score\"]<=0.8))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "81d872f3-ba83-4d22-af08-4fdeb211f2a5",
+   "metadata": {},
+   "source": [
+    "## Mining for hard negatives"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 28,
@@ -657,10 +665,77 @@
     "len(finetuning_dataset)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "8944460b-e0db-4f7e-b0c3-bdc02c936965",
+   "metadata": {},
+   "source": [
+    "## Dataset push to hub"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e016b96e-c760-4492-94e3-01be637c5122",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = Dataset.from_json(\"finetuning_dataset.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "8522ccd1-c653-4750-b560-385e99a890e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = dataset.remove_columns([\"hard_negatives\",\"negatives\",\"positives\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "c78be3e2-8089-4e19-99d0-77542142c67f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "03153dac8c4e400b90d754fba87c7eee",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pushing dataset shards to the dataset hub:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "db77d857ce7c42708fe21265d42b954c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Creating parquet from Arrow format:   0%|          | 0/2 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset.push_to_hub(\"explodinggradients/qrecc_conversational_embeddings\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7565a404-96d2-40d5-a063-a6e8b3bb9a90",
+   "id": "cbbb2670-27db-466b-bf44-6fa6954235f2",
    "metadata": {},
    "outputs": [],
    "source": []