Compute embeddings for all kubeflow repositories. (kubeflow#124)

* This is the first step in creating an org wide model for all of Kubeflow (kubeflow#110)

* Modify the Get-GitHub-Issues.ipynb model to reuse code in the embeddings directory.
  * Add some missing packages to requirements.txt.
  * Use the GitHub graphql client to get a list of all repositories.

* Add some missing GCS utilities

* Remove some of the duplication between Get-GitHub-Issues.ipynb and our
  library methods (kubeflow#122)

* Start fetching the data from bigquery.

  * Using BigQuery turned out to be a lot better for bulk pulling all
    of the Kubeflow issues.

* Use hdf5 to save the data.

* Start a doc to keep track of notes for how to train a Kubeflow model. Add logic to save to hDF5 and do a sanity check compared to the inference code

* Add a function to fetch the data using the GitHub API
  * related to kubeflow#126 swiching the embeddings service to use the GraphQL API rather than html fetching
  * I added this function as a way to sanity check that we get the same data
    using bigquery as at inference time.

Author: jlewi

Issue_Embeddings/notebooks/01_AcquireData.ipynb

@@ -155,9 +155,7 @@
   {
    "cell_type": "code",
    "execution_count": 43,
-   "metadata": {
-    "scrolled": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
@@ -501,8 +499,20 @@
    "display_name": "Python 3",
    "language": "python",
    "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

Issue_Embeddings/notebooks/02_fastai_DataBunch.ipynb

@@ -267,9 +267,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

Issue_Embeddings/notebooks/03_Create_Model.ipynb

@@ -282,9 +282,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

Issue_Embeddings/notebooks/07_Get_Repo_TrainingData_BigQuery.ipynb

@@ -451,9 +451,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

Issue_Embeddings/notebooks/Get-GitHub-Issues.ipynb

@@ -17,15 +17,18 @@ cymem==2.0.2
 cytoolz==0.9.0.1
 decorator==4.4.0
 defusedxml==0.6.0
+dill==0.3.0
 entrypoints==0.3
 fastai==1.0.53.post3
 fastprogress==0.1.21
 flask-session==0.3.1
 flask==1.0.2
 ftfy==4.4.3
 gcsfs==0.2.1
-google-auth-oauthlib==0.3.0
-google-auth==1.6.3
+github3.py>=1.3.0
+google-auth-oauthlib
+google-auth
+google-cloud-bigquery
 gunicorn==19.9.0
 html5lib==1.0.1
 idna==2.8
@@ -40,13 +43,15 @@ jedi==0.13.3
 jinja2==2.10.1
 joblib==0.13.2
 jsonschema==3.0.1
+JSON-log-formatter==0.2.0
 jupyter-client==5.2.4
 jupyter-core==4.4.0
 kiwisolver==1.1.0
 markupsafe==1.1.1
 matplotlib==3.0.3
 mdparse==0.13
 mistune==0.8.4
+more-itertools==8.2.0
 murmurhash==1.0.2
 mwparserfromhell==0.5.3
 nbconvert==5.5.0
@@ -58,6 +63,7 @@ numpy==1.16.4
 oauthlib==3.0.1
 packaging==19.0
 pandas==0.24.2
+pandas-gbq 
 pandocfilters==1.4.2
 parso==0.4.0
 passlib==1.7.1
@@ -89,17 +95,19 @@ rsa==4.0
 scikit-learn==0.20.3
 scipy==1.2.1
 send2trash==1.5.0
-six==1.12.0
+six>=1.13.0
 soupsieve==1.9.1
 spacy==2.1.4
 srsly==0.0.7
+tables
 terminado==0.8.2
 testpath==0.4.2
 textacy==0.7.1
 thinc==7.0.4
 timeout-decorator==0.4.1
 toolz==0.9.0
 tornado==6.0.2
+torch==1.1.0
 tqdm==4.32.2
 traitlets==4.3.2
 typing==3.6.6