basic functionality

Author: lolipopshock

.gitignore

@@ -0,0 +1,141 @@
+*.bak
+.gitattributes
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
+
+# link checker
+checklink/cookies.txt
+
+# .gitconfig is now autogenerated
+.gitconfig
+

README.md

@@ -0,0 +1,40 @@
+# `notion-df`: Seamlessly Connecting Notion Database with Pandas DataFrame
+
+*Please Note: This project is currently in pre-alpha stage. The code are not appropriately documented and tested. Please report any issues you find.*
+
+## Installation
+
+```bash
+git clone https://github.com/lolipopshock/notion-df && cd notion-df
+pip install -e .
+```
+
+## Usage
+
+- Download your Notion table as a pandas DataFrame
+    ```python
+    import notion_df
+    df = notion_df.load(notion_database_url, api_key=api_key)
+    df.head()
+    ```
+
+- Append a local `df` to a Notion database:
+
+    ```python
+    import notion_df
+    notion_df.upload(df, notion_database_url, title="page-title", api_key=api_key)
+    ```
+
+- Upload a local `df` to a newly created database in a Notion page:
+    
+    ```python
+    import notion_df
+    notion_df.upload(df, notion_page_url, title="page-title", api_key=api_key)
+    ```
+
+- Tired of typing `api_key=api_key` each time?
+
+    ```python
+    import notion_df
+    notion_df.config(api_key=api_key)
+    ```

requirements.txt

@@ -0,0 +1,4 @@
+notion-client>=0.8.0
+pydantic~=1.9.0
+pandas 
+dataclasses

setup.py

@@ -0,0 +1,36 @@
+from setuptools import setup, find_packages
+import os
+
+
+def get_requirements(req_file):
+    reqs = []
+    with open(req_file, "r") as fp:
+        for line in fp.readlines():
+            if line.startswith("#") or line.strip() == "":
+                continue
+            else:
+                reqs.append(line.strip())
+    return reqs
+
+
+# A trick from https://github.com/jina-ai/jina/blob/79b302c93b01689e82cf4b52f46522eb7497c404/setup.py#L20
+libinfo_py = os.path.join("src", "notion_df", "__init__.py")
+libinfo_content = open(libinfo_py, "r", encoding="utf8").readlines()
+version_line = [l.strip() for l in libinfo_content if l.startswith("__version__")][0]
+exec(version_line)  # gives __version__
+
+setup(
+    name="notion-df",
+    version=__version__,
+    description="Notion-DF: Seamlessly Connecting Notion Database with Pandas DataFrame",
+    author="Zejiang Shen",
+    author_email="[email protected]",
+    license="Apache-2.0",
+    url="https://github.com/lolipopshock/notion-df",
+    package_dir={"": "src"},
+    packages=find_packages("src"),
+    long_description=open("README.md", "r", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    python_requires=">=3.6",
+    install_requires=get_requirements("requirements.txt"),
+)

src/notion_df/__init__.py

@@ -0,0 +1,3 @@
+from notion_df.agent import load, upload
+
+__version__ = "0.0.1"

src/notion_df/agent.py

@@ -0,0 +1,148 @@
+from typing import List, Dict, Optional
+from datetime import datetime
+import os
+from functools import wraps
+
+from notion_client import Client
+from notion_client.helpers import get_id
+
+from notion_df.values import PageProperties, PageProperty
+from notion_df.configs import DatabaseSchema, guess_align_schema_for_df
+
+API_KEY = None
+NOT_REVERSE_DATAFRAME = -1
+# whether to reverse the dataframe when performing uploading.
+# for some reason, notion will reverse the order of dataframe
+# when uploading.
+# -1 for reversing, for not reversing
+
+
+def config(api_key: str):
+    API_KEY = api_key
+
+
+def _load_api_key(api_key: str) -> str:
+    if api_key is not None:
+        return api_key
+    elif API_KEY is not None:
+        return API_KEY
+    elif os.environ.get("NOTION_API_KEY") is not None:
+        return os.environ.get("NOTION_API_KEY")
+    else:
+        raise ValueError("No API key provided")
+
+
+def _is_notion_database(notion_url):
+    return "?v=" in notion_url.split("/")[-1]
+
+
+def use_client(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        orig_client = client = kwargs.pop("client", None)
+
+        if client is None:
+            api_key = _load_api_key(kwargs.pop("api_key", None))
+            client = Client(auth=api_key)
+        out = func(client=client, *args, **kwargs)
+
+        if orig_client is None:
+            # Automatically close the client if it was not passed in
+            client.close()
+        return out
+
+    return wrapper
+
+
+@use_client
+def load(notion_url: str, *, api_key: str = None, client: Client = None):
+    assert _is_notion_database(notion_url)
+    database_id = get_id(notion_url)
+
+    query_results = client.databases.query(database_id=database_id)
+    assert query_results["object"] == "list"
+    properties = PageProperties.from_raw(
+        query_results["results"]
+    )  # TODO: handle pagination
+
+    retrieve_results = client.databases.retrieve(database_id=database_id)
+    schema = DatabaseSchema.from_raw(retrieve_results["properties"])
+
+    df = properties.to_frame()
+    df.schema = schema
+    return df
+
+
+def create_database(page_id:str, client: Client, schema:DatabaseSchema, title:str=""):
+    response = client.databases.create(
+            parent={"type": "page_id", "page_id": page_id},
+            title=[{"type": "text", "text": {"content": title}}],
+            properties=schema.query_dict(),
+        )
+    assert response['object'] == 'database'
+    return response
+    
+
+def upload_row_to_database(row, database_id, schema, client):
+    
+    properties = PageProperty.from_series(row, schema).query_dict()
+    client.pages.create(
+            parent={"database_id": database_id}, properties=properties
+        )
+
+
+def upload_to_database(df, databse_id, schema, client):
+    for _, row in df[::NOT_REVERSE_DATAFRAME].iterrows():
+        upload_row_to_database(row, databse_id, schema, client)
+
+
+def load_database_schema(database_id, client):
+    return DatabaseSchema.from_raw(
+        client.databases.retrieve(database_id=database_id)["properties"]
+    )
+
+
+def validate_df_with_schema(df, schema):
+    if hasattr(df, "schema"):
+        assert df.schema == schema
+    else:
+        for col in df.columns:
+            assert col in schema.configs.keys()
+            # When DF doesn't have a schema, we just ensure that their
+            # column names appear in the schema
+
+
+@use_client
+def upload(
+    df: "pd.DataFrame",
+    notion_url: str,
+    schema=None,
+    mode="a",
+    title: str = "",
+    *,
+    api_key: str = None,
+    client: Client = None,
+):
+    if not _is_notion_database(notion_url):
+        df, schema = guess_align_schema_for_df(df)
+        database_properties = create_database(get_id(notion_url), client, schema, title)
+        databse_id = database_properties['id']
+        notion_url = database_properties['url']
+    else:
+        databse_id = get_id(notion_url)
+        if schema is None:
+            schema = load_database_schema(databse_id, client)
+
+    # At this stage, we should have the appropriate schema
+    assert schema is not None
+
+    validate_df_with_schema(df, schema)
+
+    if mode not in ("a", "append"):
+        raise NotImplementedError
+        # TODO: clean the current values in the notion database (if any)
+
+    upload_to_database(df, databse_id, schema, client)
+    
+    print(f"Your dataframe has been uploaded to the Notion page: {notion_url} .")
+    return notion_url