Add support for Community Edition server

olsync/olbrowserlogin.py

@@ -14,24 +14,29 @@
 from PySide6.QtWebEngineWidgets import *
 from PySide6.QtWebEngineCore import QWebEngineProfile, QWebEngineSettings, QWebEnginePage
 
-# Where to get the CSRF Token and where to send the login request to
-LOGIN_URL = "https://www.overleaf.com/login"
-PROJECT_URL = "https://www.overleaf.com/project"  # The dashboard URL
+
 # JS snippet to extract the csrfToken
 JAVASCRIPT_CSRF_EXTRACTOR = "document.getElementsByName('ol-csrfToken')[0].content"
-# Name of the cookies we want to extract
-COOKIE_NAMES = ["overleaf_session2", "GCLB"]
-
 
 class OlBrowserLoginWindow(QMainWindow):
     """
     Overleaf Browser Login Utility
     Opens a browser window to securely login the user and returns relevant login data.
     """
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, ce_url=None, *args, **kwargs):
         super(OlBrowserLoginWindow, self).__init__(*args, **kwargs)
 
+        if ce_url is not None:
+            self._BASE_URL = ce_url
+            self._cookies_names = ["sharelatex.sid"]
+        else:
+            self._BASE_URL = "https://www.overleaf.com"  # The Overleaf Base URL
+            self._cookies_names = ["overleaf_session2", "GCLB"]
+
+        self._LOGIN_URL = f"{self._BASE_URL}/login"
+        self._PROJECT_URL = f"{self._BASE_URL}/project"
+
         self.webview = QWebEngineView()
 
         self._cookies = {}
@@ -47,7 +52,7 @@ def __init__(self, *args, **kwargs):
 
         webpage = QWebEnginePage(self.profile, self)
         self.webview.setPage(webpage)
-        self.webview.load(QUrl.fromUserInput(LOGIN_URL))
+        self.webview.load(QUrl.fromUserInput(self._LOGIN_URL))
         self.webview.loadFinished.connect(self.handle_load_finished)
 
         self.setCentralWidget(self.webview)
@@ -59,14 +64,14 @@ def callback(result):
             self._login_success = True
             QCoreApplication.quit()
 
-        if self.webview.url().toString() == PROJECT_URL:
+        if self.webview.url().toString() == self._PROJECT_URL:
             self.webview.page().runJavaScript(
                 JAVASCRIPT_CSRF_EXTRACTOR, 0, callback
             )
 
     def handle_cookie_added(self, cookie):
         cookie_name = cookie.name().data().decode('utf-8')
-        if cookie_name in COOKIE_NAMES:
+        if cookie_name in self._cookies_names:
             self._cookies[cookie_name] = cookie.value().data().decode('utf-8')
 
     @property
@@ -82,14 +87,14 @@ def login_success(self):
         return self._login_success
 
 
-def login():
+def login(ce_url=None):
     from PySide6.QtCore import QLoggingCategory
     QLoggingCategory.setFilterRules('''\
     qt.webenginecontext.info=false
     ''')
 
     app = QApplication([])
-    ol_browser_login_window = OlBrowserLoginWindow()
+    ol_browser_login_window = OlBrowserLoginWindow(ce_url)
     ol_browser_login_window.show()
     app.exec()
 

olsync/olclient.py

@@ -16,18 +16,9 @@
 from socketIO_client import SocketIO
 import time
 
-# Where to get the CSRF Token and where to send the login request to
-LOGIN_URL = "https://www.overleaf.com/login"
-PROJECT_URL = "https://www.overleaf.com/project"  # The dashboard URL
-# The URL to download all the files in zip format
-DOWNLOAD_URL = "https://www.overleaf.com/project/{}/download/zip"
-UPLOAD_URL = "https://www.overleaf.com/project/{}/upload"  # The URL to upload files
-FOLDER_URL = "https://www.overleaf.com/project/{}/folder"  # The URL to create folders
-DELETE_URL = "https://www.overleaf.com/project/{}/doc/{}"  # The URL to delete files
-COMPILE_URL = "https://www.overleaf.com/project/{}/compile?enable_pdf_caching=true"  # The URL to compile the project
-BASE_URL = "https://www.overleaf.com"  # The Overleaf Base URL
 PATH_SEP = "/"  # Use hardcoded path separator for both windows and posix system
 
+
 class OverleafClient(object):
     """
     Overleaf API Wrapper
@@ -43,9 +34,25 @@ def filter_projects(json_content, more_attrs=None):
                 if all(p.get(k) == v for k, v in more_attrs.items()):
                     yield p
 
-    def __init__(self, cookie=None, csrf=None):
+    def __init__(self, cookie=None, csrf=None, ce_url=None):
         self._cookie = cookie  # Store the cookie for authenticated requests
         self._csrf = csrf  # Store the CSRF token since it is needed for some requests
+        # Where to get the CSRF Token and where to send the login request to
+        if ce_url is not None:
+            self._ce = True
+            self._BASE_URL = ce_url
+        else:
+            self._BASE_URL = "https://www.overleaf.com"  # The Overleaf Base URL
+            self._ce = True
+
+        self._LOGIN_URL = self._BASE_URL + "/login"
+        self._PROJECT_URL = self._BASE_URL + "/project"  # The dashboard URL
+        # The URL to download all the files in zip format
+        self._DOWNLOAD_URL = self._BASE_URL + "/project/{}/download/zip"
+        self._UPLOAD_URL = self._BASE_URL + "/project/{}/upload"  # The URL to upload files
+        self._FOLDER_URL = self._BASE_URL + "/project/{}/folder"  # The URL to create folders
+        self._COMPILE_URL = self._BASE_URL + "/project/{}/compile?enable_pdf_caching=true"  # The URL to compile the project
+        self._DELETE_URL = self._BASE_URL + "/project/{}/doc/{}"  # The URL to delete files
 
     def login(self, username, password):
         """
@@ -55,28 +62,30 @@ def login(self, username, password):
         Returns: Dict of cookie and CSRF
         """
 
-        get_login = reqs.get(LOGIN_URL)
+        get_login = reqs.get(self._LOGIN_URL)
         self._csrf = BeautifulSoup(get_login.content, 'html.parser').find(
             'input', {'name': '_csrf'}).get('value')
         login_json = {
             "_csrf": self._csrf,
             "email": username,
             "password": password
         }
-        post_login = reqs.post(LOGIN_URL, json=login_json,
+        post_login = reqs.post(self._LOGIN_URL, json=login_json,
                                cookies=get_login.cookies)
 
         # On a successful authentication the Overleaf API returns a new authenticated cookie.
         # If the cookie is different than the cookie of the GET request the authentication was successful
-        if post_login.status_code == 200 and get_login.cookies["overleaf_session2"] != post_login.cookies[
-            "overleaf_session2"]:
+        if post_login.status_code == 200 and ((self._ce and get_login.cookies["sharelatex.sid"] != post_login.cookies[
+            "sharelatex.sid"]) or get_login.cookies["overleaf_session2"] != post_login.cookies[
+                                                  "overleaf_session2"]):
             self._cookie = post_login.cookies
 
             # Enrich cookie with GCLB cookie from GET request above
-            self._cookie['GCLB'] = get_login.cookies['GCLB']
+            if not self._ce:
+                self._cookie['GCLB'] = get_login.cookies['GCLB']
 
             # CSRF changes after making the login request, new CSRF token will be on the projects page
-            projects_page = reqs.get(PROJECT_URL, cookies=self._cookie)
+            projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie)
             self._csrf = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'}) \
                 .get('content')
 
@@ -87,7 +96,7 @@ def all_projects(self):
         Get all of a user's active projects (= not archived and not trashed)
         Returns: List of project objects
         """
-        projects_page = reqs.get(PROJECT_URL, cookies=self._cookie)
+        projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie)
         json_content = json.loads(
             BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-projects'}).get('content'))
         return list(OverleafClient.filter_projects(json_content))
@@ -99,7 +108,7 @@ def get_project(self, project_name):
         Returns: project object
         """
 
-        projects_page = reqs.get(PROJECT_URL, cookies=self._cookie)
+        projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie)
         json_content = json.loads(
             BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-projects'}).get('content'))
         return next(OverleafClient.filter_projects(json_content, {"name": project_name}), None)
@@ -110,7 +119,7 @@ def download_project(self, project_id):
         Params: project_id, the id of the project
         Returns: bytes string (zip file)
         """
-        r = reqs.get(DOWNLOAD_URL.format(project_id),
+        r = reqs.get(self._DOWNLOAD_URL.format(project_id),
                      stream=True, cookies=self._cookie)
         return r.content
 
@@ -133,7 +142,7 @@ def create_folder(self, project_id, parent_folder_id, folder_name):
         headers = {
             "X-Csrf-Token": self._csrf
         }
-        r = reqs.post(FOLDER_URL.format(project_id),
+        r = reqs.post(self._FOLDER_URL.format(project_id),
                       cookies=self._cookie, headers=headers, json=params)
 
         if r.ok:
@@ -162,15 +171,12 @@ def set_project_infos(a, project_infos_dict, c, d):
             project_infos = project_infos_dict
 
         # Convert cookie from CookieJar to string
-        cookie = "GCLB={}; overleaf_session2={}" \
-            .format(
-            self._cookie["GCLB"],
-            self._cookie["overleaf_session2"]
-        )
+        cookie = f"GCLB={self._cookie['GCLB']}; overleaf_session2={self._cookie['overleaf_session2']}" if not self._ce \
+            else f"sharelatex.sid={self._cookie['sharelatex.sid']}"
 
         # Connect to Overleaf Socket.IO, send a time parameter and the cookies
         socket_io = SocketIO(
-            BASE_URL,
+            self._BASE_URL,
             params={'t': int(time.time())},
             headers={'Cookie': cookie}
         )
@@ -237,7 +243,7 @@ def upload_file(self, project_id, project_infos, file_name, file_size, file):
         }
 
         # Upload the file to the predefined folder
-        r = reqs.post(UPLOAD_URL.format(project_id), cookies=self._cookie, params=params, files=files)
+        r = reqs.post(self._UPLOAD_URL.format(project_id), cookies=self._cookie, params=params, files=files)
 
         return r.status_code == str(200) and json.loads(r.content)["success"]
 
@@ -278,7 +284,8 @@ def delete_file(self, project_id, project_infos, file_name):
             "X-Csrf-Token": self._csrf
         }
 
-        r = reqs.delete(DELETE_URL.format(project_id, file['_id']), cookies=self._cookie, headers=headers, json={})
+        r = reqs.delete(self._DELETE_URL.format(project_id, file['_id']), cookies=self._cookie, headers=headers,
+                        json={})
 
         return r.status_code == str(204)
 
@@ -303,7 +310,7 @@ def download_pdf(self, project_id):
             "stopOnFirstError": False
         }
 
-        r = reqs.post(COMPILE_URL.format(project_id), cookies=self._cookie, headers=headers, json=body)
+        r = reqs.post(self._COMPILE_URL.format(project_id), cookies=self._cookie, headers=headers, json=body)
 
         if not r.ok:
             raise reqs.HTTPError()
@@ -315,7 +322,7 @@ def download_pdf(self, project_id):
 
         pdf_file = next(v for v in compile_result['outputFiles'] if v['type'] == 'pdf')
 
-        download_req = reqs.get(BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers)
+        download_req = reqs.get(self._BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers)
 
         if download_req.ok:
             return pdf_file['path'], download_req.content

olsync/olsync.py

@@ -44,10 +44,12 @@
 @click.option('-i', '--olignore', 'olignore_path', default=".olignore", type=click.Path(exists=False),
               help="Path to the .olignore file relative to sync path (ignored if syncing from remote to local). See "
                    "fnmatch / unix filename pattern matching for information on how to use it.")
+@click.option('-u', '--ce-url', 'ce_url', default="",
+              help="Base url for Community Edition (CE) server. Uses this if .olce file is not present")
 @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
 @click.version_option(package_name='overleaf-sync')
 @click.pass_context
-def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, verbose):
+def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, ce_url, verbose):
     if ctx.invoked_subcommand is None:
         if not os.path.isfile(cookie_path):
             raise click.ClickException(
@@ -56,7 +58,7 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
         with open(cookie_path, 'rb') as f:
             store = pickle.load(f)
 
-        overleaf_client = OverleafClient(store["cookie"], store["csrf"])
+        overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url))
 
         # Change the current directory to the specified sync path
         os.chdir(sync_path)
@@ -121,13 +123,15 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path
 @main.command()
 @click.option('--path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
               help="Path to store the persisted Overleaf cookie.")
+@click.option('-u', '--ce-url', 'ce_url', default="",
+              help="Base url for Community Edition (CE) server. Uses this if .olce file is not present")
 @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
-def login(cookie_path, verbose):
+def login(cookie_path, ce_url, verbose):
     if os.path.isfile(cookie_path) and not click.confirm(
             'Persisted Overleaf cookie already exist. Do you want to override it?'):
         return
     click.clear()
-    execute_action(lambda: login_handler(cookie_path), "Login",
+    execute_action(lambda: login_handler(cookie_path, ol_base_path(ce_url)), "Login",
                    "Login successful. Cookie persisted as `" + click.format_filename(
                        cookie_path) + "`. You may now sync your project.",
                    "Login failed. Please try again.", verbose)
@@ -136,8 +140,10 @@ def login(cookie_path, verbose):
 @main.command(name='list')
 @click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
               help="Relative path to load the persisted Overleaf cookie.")
+@click.option('-u', '--ce-url', 'ce_url', default="",
+              help="Base url for Community Edition (CE) server. Uses this if .olce file is not present")
 @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
-def list_projects(cookie_path, verbose):
+def list_projects(cookie_path, ce_url, verbose):
     def query_projects():
         for index, p in enumerate(sorted(overleaf_client.all_projects(), key=lambda x: x['lastUpdated'], reverse=True)):
             if not index:
@@ -152,7 +158,7 @@ def query_projects():
     with open(cookie_path, 'rb') as f:
         store = pickle.load(f)
 
-    overleaf_client = OverleafClient(store["cookie"], store["csrf"])
+    overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url))
 
     click.clear()
     execute_action(query_projects, "Querying all projects",
@@ -166,8 +172,10 @@ def query_projects():
 @click.option('--download-path', 'download_path', default=".", type=click.Path(exists=True))
 @click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False),
               help="Relative path to load the persisted Overleaf cookie.")
+@click.option('-u', '--ce-url', 'ce_url', default="",
+              help="Base url for Community Edition (CE) server. Uses this if .olce file is not present")
 @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.")
-def download_pdf(project_name, download_path, cookie_path, verbose):
+def download_pdf(project_name, download_path, cookie_path, ce_url, verbose):
     def download_project_pdf():
         nonlocal project_name
         project_name = project_name or os.path.basename(os.getcwd())
@@ -194,7 +202,7 @@ def download_project_pdf():
     with open(cookie_path, 'rb') as f:
         store = pickle.load(f)
 
-    overleaf_client = OverleafClient(store["cookie"], store["csrf"])
+    overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url))
 
     click.clear()
 
@@ -203,8 +211,8 @@ def download_project_pdf():
                    "Downloading project's PDF failed. Please try again.", verbose)
 
 
-def login_handler(path):
-    store = olbrowserlogin.login()
+def login_handler(path, ce_url=None):
+    store = olbrowserlogin.login(ce_url)
     if store is None:
         return False
     with open(path, 'wb+') as f:
@@ -385,5 +393,29 @@ def olignore_keep_list(olignore_path):
     return keep_list
 
 
+
+def ol_base_path(ce_url):
+
+    # if ce_url: return ce_url # set from
+    # Read .olce file for URL to local installation of Overleaf/Sharelatex
+    olce_file = ".olce"
+
+    click.echo("=" * 40)
+
+    if not os.path.isfile(olce_file) and (ce_url in ["", "https://www.overleaf.com"]):
+        click.echo("\nNotice: .olce file does not exist nor --ce-url specified, will sync with overleaf.com.")
+        return None
+    else:
+        if not ce_url:
+            with open(olce_file, 'r') as f:
+                ce_url = f.readline().strip().strip("\n")
+                f.close()
+            source = ".olce"
+        else:
+            source = "--ce-url"
+        click.echo(f"\nusing {ce_url} as CE server (from {source})")
+
+    return ce_url
+
 if __name__ == "__main__":
     main()