From 649264518c52448ede0ac6574863c44514553370 Mon Sep 17 00:00:00 2001 From: Simone Massaro Date: Thu, 26 Jan 2023 19:03:15 +0100 Subject: [PATCH] Added support for Community Edition server --- olsync/olbrowserlogin.py | 29 ++++++++++------- olsync/olclient.py | 69 ++++++++++++++++++++++------------------ olsync/olsync.py | 52 ++++++++++++++++++++++++------ 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/olsync/olbrowserlogin.py b/olsync/olbrowserlogin.py index f5a8e9a..56c17ef 100644 --- a/olsync/olbrowserlogin.py +++ b/olsync/olbrowserlogin.py @@ -14,14 +14,9 @@ from PySide6.QtWebEngineWidgets import * from PySide6.QtWebEngineCore import QWebEngineProfile, QWebEngineSettings, QWebEnginePage -# Where to get the CSRF Token and where to send the login request to -LOGIN_URL = "https://www.overleaf.com/login" -PROJECT_URL = "https://www.overleaf.com/project" # The dashboard URL + # JS snippet to extract the csrfToken JAVASCRIPT_CSRF_EXTRACTOR = "document.getElementsByName('ol-csrfToken')[0].content" -# Name of the cookies we want to extract -COOKIE_NAMES = ["overleaf_session2", "GCLB"] - class OlBrowserLoginWindow(QMainWindow): """ @@ -29,9 +24,19 @@ class OlBrowserLoginWindow(QMainWindow): Opens a browser window to securely login the user and returns relevant login data. """ - def __init__(self, *args, **kwargs): + def __init__(self, ce_url=None, *args, **kwargs): super(OlBrowserLoginWindow, self).__init__(*args, **kwargs) + if ce_url is not None: + self._BASE_URL = ce_url + self._cookies_names = ["sharelatex.sid"] + else: + self._BASE_URL = "https://www.overleaf.com" # The Overleaf Base URL + self._cookies_names = ["overleaf_session2", "GCLB"] + + self._LOGIN_URL = f"{self._BASE_URL}/login" + self._PROJECT_URL = f"{self._BASE_URL}/project" + self.webview = QWebEngineView() self._cookies = {} @@ -47,7 +52,7 @@ def __init__(self, *args, **kwargs): webpage = QWebEnginePage(self.profile, self) self.webview.setPage(webpage) - self.webview.load(QUrl.fromUserInput(LOGIN_URL)) + self.webview.load(QUrl.fromUserInput(self._LOGIN_URL)) self.webview.loadFinished.connect(self.handle_load_finished) self.setCentralWidget(self.webview) @@ -59,14 +64,14 @@ def callback(result): self._login_success = True QCoreApplication.quit() - if self.webview.url().toString() == PROJECT_URL: + if self.webview.url().toString() == self._PROJECT_URL: self.webview.page().runJavaScript( JAVASCRIPT_CSRF_EXTRACTOR, 0, callback ) def handle_cookie_added(self, cookie): cookie_name = cookie.name().data().decode('utf-8') - if cookie_name in COOKIE_NAMES: + if cookie_name in self._cookies_names: self._cookies[cookie_name] = cookie.value().data().decode('utf-8') @property @@ -82,14 +87,14 @@ def login_success(self): return self._login_success -def login(): +def login(ce_url=None): from PySide6.QtCore import QLoggingCategory QLoggingCategory.setFilterRules('''\ qt.webenginecontext.info=false ''') app = QApplication([]) - ol_browser_login_window = OlBrowserLoginWindow() + ol_browser_login_window = OlBrowserLoginWindow(ce_url) ol_browser_login_window.show() app.exec() diff --git a/olsync/olclient.py b/olsync/olclient.py index a993de1..c9c1d13 100644 --- a/olsync/olclient.py +++ b/olsync/olclient.py @@ -16,18 +16,9 @@ from socketIO_client import SocketIO import time -# Where to get the CSRF Token and where to send the login request to -LOGIN_URL = "https://www.overleaf.com/login" -PROJECT_URL = "https://www.overleaf.com/project" # The dashboard URL -# The URL to download all the files in zip format -DOWNLOAD_URL = "https://www.overleaf.com/project/{}/download/zip" -UPLOAD_URL = "https://www.overleaf.com/project/{}/upload" # The URL to upload files -FOLDER_URL = "https://www.overleaf.com/project/{}/folder" # The URL to create folders -DELETE_URL = "https://www.overleaf.com/project/{}/doc/{}" # The URL to delete files -COMPILE_URL = "https://www.overleaf.com/project/{}/compile?enable_pdf_caching=true" # The URL to compile the project -BASE_URL = "https://www.overleaf.com" # The Overleaf Base URL PATH_SEP = "/" # Use hardcoded path separator for both windows and posix system + class OverleafClient(object): """ Overleaf API Wrapper @@ -43,9 +34,25 @@ def filter_projects(json_content, more_attrs=None): if all(p.get(k) == v for k, v in more_attrs.items()): yield p - def __init__(self, cookie=None, csrf=None): + def __init__(self, cookie=None, csrf=None, ce_url=None): self._cookie = cookie # Store the cookie for authenticated requests self._csrf = csrf # Store the CSRF token since it is needed for some requests + # Where to get the CSRF Token and where to send the login request to + if ce_url is not None: + self._ce = True + self._BASE_URL = ce_url + else: + self._BASE_URL = "https://www.overleaf.com" # The Overleaf Base URL + self._ce = True + + self._LOGIN_URL = self._BASE_URL + "/login" + self._PROJECT_URL = self._BASE_URL + "/project" # The dashboard URL + # The URL to download all the files in zip format + self._DOWNLOAD_URL = self._BASE_URL + "/project/{}/download/zip" + self._UPLOAD_URL = self._BASE_URL + "/project/{}/upload" # The URL to upload files + self._FOLDER_URL = self._BASE_URL + "/project/{}/folder" # The URL to create folders + self._COMPILE_URL = self._BASE_URL + "/project/{}/compile?enable_pdf_caching=true" # The URL to compile the project + self._DELETE_URL = self._BASE_URL + "/project/{}/doc/{}" # The URL to delete files def login(self, username, password): """ @@ -55,7 +62,7 @@ def login(self, username, password): Returns: Dict of cookie and CSRF """ - get_login = reqs.get(LOGIN_URL) + get_login = reqs.get(self._LOGIN_URL) self._csrf = BeautifulSoup(get_login.content, 'html.parser').find( 'input', {'name': '_csrf'}).get('value') login_json = { @@ -63,20 +70,22 @@ def login(self, username, password): "email": username, "password": password } - post_login = reqs.post(LOGIN_URL, json=login_json, + post_login = reqs.post(self._LOGIN_URL, json=login_json, cookies=get_login.cookies) # On a successful authentication the Overleaf API returns a new authenticated cookie. # If the cookie is different than the cookie of the GET request the authentication was successful - if post_login.status_code == 200 and get_login.cookies["overleaf_session2"] != post_login.cookies[ - "overleaf_session2"]: + if post_login.status_code == 200 and ((self._ce and get_login.cookies["sharelatex.sid"] != post_login.cookies[ + "sharelatex.sid"]) or get_login.cookies["overleaf_session2"] != post_login.cookies[ + "overleaf_session2"]): self._cookie = post_login.cookies # Enrich cookie with GCLB cookie from GET request above - self._cookie['GCLB'] = get_login.cookies['GCLB'] + if not self._ce: + self._cookie['GCLB'] = get_login.cookies['GCLB'] # CSRF changes after making the login request, new CSRF token will be on the projects page - projects_page = reqs.get(PROJECT_URL, cookies=self._cookie) + projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie) self._csrf = BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-csrfToken'}) \ .get('content') @@ -87,7 +96,7 @@ def all_projects(self): Get all of a user's active projects (= not archived and not trashed) Returns: List of project objects """ - projects_page = reqs.get(PROJECT_URL, cookies=self._cookie) + projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie) json_content = json.loads( BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-projects'}).get('content')) return list(OverleafClient.filter_projects(json_content)) @@ -99,7 +108,7 @@ def get_project(self, project_name): Returns: project object """ - projects_page = reqs.get(PROJECT_URL, cookies=self._cookie) + projects_page = reqs.get(self._PROJECT_URL, cookies=self._cookie) json_content = json.loads( BeautifulSoup(projects_page.content, 'html.parser').find('meta', {'name': 'ol-projects'}).get('content')) return next(OverleafClient.filter_projects(json_content, {"name": project_name}), None) @@ -110,7 +119,7 @@ def download_project(self, project_id): Params: project_id, the id of the project Returns: bytes string (zip file) """ - r = reqs.get(DOWNLOAD_URL.format(project_id), + r = reqs.get(self._DOWNLOAD_URL.format(project_id), stream=True, cookies=self._cookie) return r.content @@ -133,7 +142,7 @@ def create_folder(self, project_id, parent_folder_id, folder_name): headers = { "X-Csrf-Token": self._csrf } - r = reqs.post(FOLDER_URL.format(project_id), + r = reqs.post(self._FOLDER_URL.format(project_id), cookies=self._cookie, headers=headers, json=params) if r.ok: @@ -162,15 +171,12 @@ def set_project_infos(a, project_infos_dict, c, d): project_infos = project_infos_dict # Convert cookie from CookieJar to string - cookie = "GCLB={}; overleaf_session2={}" \ - .format( - self._cookie["GCLB"], - self._cookie["overleaf_session2"] - ) + cookie = f"GCLB={self._cookie['GCLB']}; overleaf_session2={self._cookie['overleaf_session2']}" if not self._ce \ + else f"sharelatex.sid={self._cookie['sharelatex.sid']}" # Connect to Overleaf Socket.IO, send a time parameter and the cookies socket_io = SocketIO( - BASE_URL, + self._BASE_URL, params={'t': int(time.time())}, headers={'Cookie': cookie} ) @@ -237,7 +243,7 @@ def upload_file(self, project_id, project_infos, file_name, file_size, file): } # Upload the file to the predefined folder - r = reqs.post(UPLOAD_URL.format(project_id), cookies=self._cookie, params=params, files=files) + r = reqs.post(self._UPLOAD_URL.format(project_id), cookies=self._cookie, params=params, files=files) return r.status_code == str(200) and json.loads(r.content)["success"] @@ -278,7 +284,8 @@ def delete_file(self, project_id, project_infos, file_name): "X-Csrf-Token": self._csrf } - r = reqs.delete(DELETE_URL.format(project_id, file['_id']), cookies=self._cookie, headers=headers, json={}) + r = reqs.delete(self._DELETE_URL.format(project_id, file['_id']), cookies=self._cookie, headers=headers, + json={}) return r.status_code == str(204) @@ -303,7 +310,7 @@ def download_pdf(self, project_id): "stopOnFirstError": False } - r = reqs.post(COMPILE_URL.format(project_id), cookies=self._cookie, headers=headers, json=body) + r = reqs.post(self._COMPILE_URL.format(project_id), cookies=self._cookie, headers=headers, json=body) if not r.ok: raise reqs.HTTPError() @@ -315,7 +322,7 @@ def download_pdf(self, project_id): pdf_file = next(v for v in compile_result['outputFiles'] if v['type'] == 'pdf') - download_req = reqs.get(BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers) + download_req = reqs.get(self._BASE_URL + pdf_file['url'], cookies=self._cookie, headers=headers) if download_req.ok: return pdf_file['path'], download_req.content diff --git a/olsync/olsync.py b/olsync/olsync.py index 56c5f9e..70f4fa9 100644 --- a/olsync/olsync.py +++ b/olsync/olsync.py @@ -44,10 +44,12 @@ @click.option('-i', '--olignore', 'olignore_path', default=".olignore", type=click.Path(exists=False), help="Path to the .olignore file relative to sync path (ignored if syncing from remote to local). See " "fnmatch / unix filename pattern matching for information on how to use it.") +@click.option('-u', '--ce-url', 'ce_url', default="", + help="Base url for Community Edition (CE) server. Uses this if .olce file is not present") @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.") @click.version_option(package_name='overleaf-sync') @click.pass_context -def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, verbose): +def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path, ce_url, verbose): if ctx.invoked_subcommand is None: if not os.path.isfile(cookie_path): raise click.ClickException( @@ -56,7 +58,7 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path with open(cookie_path, 'rb') as f: store = pickle.load(f) - overleaf_client = OverleafClient(store["cookie"], store["csrf"]) + overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url)) # Change the current directory to the specified sync path os.chdir(sync_path) @@ -121,13 +123,15 @@ def main(ctx, local, remote, project_name, cookie_path, sync_path, olignore_path @main.command() @click.option('--path', 'cookie_path', default=".olauth", type=click.Path(exists=False), help="Path to store the persisted Overleaf cookie.") +@click.option('-u', '--ce-url', 'ce_url', default="", + help="Base url for Community Edition (CE) server. Uses this if .olce file is not present") @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.") -def login(cookie_path, verbose): +def login(cookie_path, ce_url, verbose): if os.path.isfile(cookie_path) and not click.confirm( 'Persisted Overleaf cookie already exist. Do you want to override it?'): return click.clear() - execute_action(lambda: login_handler(cookie_path), "Login", + execute_action(lambda: login_handler(cookie_path, ol_base_path(ce_url)), "Login", "Login successful. Cookie persisted as `" + click.format_filename( cookie_path) + "`. You may now sync your project.", "Login failed. Please try again.", verbose) @@ -136,8 +140,10 @@ def login(cookie_path, verbose): @main.command(name='list') @click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False), help="Relative path to load the persisted Overleaf cookie.") +@click.option('-u', '--ce-url', 'ce_url', default="", + help="Base url for Community Edition (CE) server. Uses this if .olce file is not present") @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.") -def list_projects(cookie_path, verbose): +def list_projects(cookie_path, ce_url, verbose): def query_projects(): for index, p in enumerate(sorted(overleaf_client.all_projects(), key=lambda x: x['lastUpdated'], reverse=True)): if not index: @@ -152,7 +158,7 @@ def query_projects(): with open(cookie_path, 'rb') as f: store = pickle.load(f) - overleaf_client = OverleafClient(store["cookie"], store["csrf"]) + overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url)) click.clear() execute_action(query_projects, "Querying all projects", @@ -166,8 +172,10 @@ def query_projects(): @click.option('--download-path', 'download_path', default=".", type=click.Path(exists=True)) @click.option('--store-path', 'cookie_path', default=".olauth", type=click.Path(exists=False), help="Relative path to load the persisted Overleaf cookie.") +@click.option('-u', '--ce-url', 'ce_url', default="", + help="Base url for Community Edition (CE) server. Uses this if .olce file is not present") @click.option('-v', '--verbose', 'verbose', is_flag=True, help="Enable extended error logging.") -def download_pdf(project_name, download_path, cookie_path, verbose): +def download_pdf(project_name, download_path, cookie_path, ce_url, verbose): def download_project_pdf(): nonlocal project_name project_name = project_name or os.path.basename(os.getcwd()) @@ -194,7 +202,7 @@ def download_project_pdf(): with open(cookie_path, 'rb') as f: store = pickle.load(f) - overleaf_client = OverleafClient(store["cookie"], store["csrf"]) + overleaf_client = OverleafClient(store["cookie"], store["csrf"], ol_base_path(ce_url)) click.clear() @@ -203,8 +211,8 @@ def download_project_pdf(): "Downloading project's PDF failed. Please try again.", verbose) -def login_handler(path): - store = olbrowserlogin.login() +def login_handler(path, ce_url=None): + store = olbrowserlogin.login(ce_url) if store is None: return False with open(path, 'wb+') as f: @@ -385,5 +393,29 @@ def olignore_keep_list(olignore_path): return keep_list + +def ol_base_path(ce_url): + + # if ce_url: return ce_url # set from + # Read .olce file for URL to local installation of Overleaf/Sharelatex + olce_file = ".olce" + + click.echo("=" * 40) + + if not os.path.isfile(olce_file) and (ce_url in ["", "https://www.overleaf.com"]): + click.echo("\nNotice: .olce file does not exist nor --ce-url specified, will sync with overleaf.com.") + return None + else: + if not ce_url: + with open(olce_file, 'r') as f: + ce_url = f.readline().strip().strip("\n") + f.close() + source = ".olce" + else: + source = "--ce-url" + click.echo(f"\nusing {ce_url} as CE server (from {source})") + + return ce_url + if __name__ == "__main__": main()