From a748a7b3c96928dbd284d807025102f79728b34b Mon Sep 17 00:00:00 2001 From: stboch Date: Mon, 15 Jun 2020 09:59:53 -0400 Subject: [PATCH] Commit app v1.0.0 Initial Commit of App --- phbrowserlessio/__init__.py | 0 phbrowserlessio/browserlessio.json | 315 +++++++++++++++ phbrowserlessio/browserlessio.svg | 1 + phbrowserlessio/browserlessio_connector.py | 438 +++++++++++++++++++++ phbrowserlessio/browserlessio_consts.py | 1 + phbrowserlessio/browserlessio_dark.svg | 1 + phbrowserlessio/exclude_files.txt | 3 + phbrowserlessio/readme.html | 4 + 8 files changed, 763 insertions(+) create mode 100644 phbrowserlessio/__init__.py create mode 100644 phbrowserlessio/browserlessio.json create mode 100644 phbrowserlessio/browserlessio.svg create mode 100644 phbrowserlessio/browserlessio_connector.py create mode 100644 phbrowserlessio/browserlessio_consts.py create mode 100644 phbrowserlessio/browserlessio_dark.svg create mode 100644 phbrowserlessio/exclude_files.txt create mode 100644 phbrowserlessio/readme.html diff --git a/phbrowserlessio/__init__.py b/phbrowserlessio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/phbrowserlessio/browserlessio.json b/phbrowserlessio/browserlessio.json new file mode 100644 index 0000000..0a7141f --- /dev/null +++ b/phbrowserlessio/browserlessio.json @@ -0,0 +1,315 @@ +{ + "appid": "a8ca5e0d-8209-4476-a9b6-5213ed314319", + "name": "Browserless chrome", + "description": "Browserless.io Service and browserless/chrome App", + "type": "sandbox", + "product_vendor": "browerless.io", + "logo": "browserlessio.svg", + "logo_dark": "browserlessio_dark.svg", + "product_name": "chrome", + "python_version": "3", + "product_version_regex": ".*", + "publisher": "Stboch", + "license": "Copyright (c) Stboch, 2020", + "app_version": "1.0.0", + "utctime_updated": "2020-06-01T21:36:33.805158Z", + "package_name": "phantom_browserlessio", + "main_module": "browserlessio_connector.pyc", + "min_phantom_version": "4.8.23319", + "app_wizard_version": "1.0.0", + "configuration": { + "URL": { + "description": "URL for browserless.io (if using hosted service) https://chrome.browserless.io", + "data_type": "string", + "required": true, + "value_list": [], + "default": "", + "order": 0 + }, + "token": { + "description": "token for accessing service (not required if self hosting)", + "data_type": "password", + "required": false, + "order": 1 + } + }, + "actions": [ + { + "action": "test connectivity", + "identifier": "test_connectivity", + "description": "Validate the asset configuration for connectivity using supplied configuration", + "verbose": "", + "type": "test", + "read_only": true, + "parameters": {}, + "output": [], + "versions": "EQ(*)" + }, + { + "action": "get pdf", + "identifier": "get_pdf", + "description": "PDF screenshot of URL", + "verbose": "Download a PDF of the URL", + "type": "investigate", + "read_only": true, + "parameters": { + "url": { + "description": "URL of website", + "data_type": "string", + "required": true, + "primary": false, + "contains": [ + "url" + ], + "value_list": [], + "default": "", + "order": 0 + }, + "headerfooter": { + "description": "Display Header and Footer", + "data_type": "boolean", + "required": true, + "primary": false, + "contains": [], + "default": true, + "order": 1 + }, + "printbackground": { + "description": "Display background images in PDF", + "data_type": "boolean", + "required": true, + "primary": false, + "contains": [], + "default": false, + "order": 2 + }, + "landscape": { + "description": "Print as Landscape", + "data_type": "boolean", + "required": true, + "primary": false, + "contains": [], + "default": false, + "order": 3 + } + }, + "output": [ + { + "data_path": "action_result.parameter.url", + "data_type": "string", + "contains": [ + "url" + ], + "column_name": "url", + "column_order": 0 + }, + { + "data_path": "action_result.parameter.headerfooter", + "data_type": "boolean", + "contains": [], + "column_name": "headerfooter", + "column_order": 1 + }, + { + "data_path": "action_result.parameter.printbackground", + "data_type": "boolean", + "contains": [], + "column_name": "printbackground", + "column_order": 2 + }, + { + "data_path": "action_result.parameter.landscape", + "data_type": "boolean", + "contains": [], + "column_name": "landscape", + "column_order": 3 + }, + { + "data_path": "action_result.status", + "data_type": "string", + "column_name": "status", + "column_order": 4 + }, + { + "data_path": "action_result.message", + "data_type": "string" + }, + { + "data_path": "summary.total_objects", + "data_type": "numeric" + }, + { + "data_path": "summary.total_objects_successful", + "data_type": "numeric" + } + ], + "render": { + "type": "table" + }, + "versions": "EQ(*)" + }, + { + "action": "get content", + "identifier": "get_content", + "description": "Get HTML contents of a webpage", + "verbose": "Download the HTML code for a URL provided", + "type": "investigate", + "read_only": true, + "parameters": { + "url": { + "description": "URL of website", + "data_type": "string", + "required": true, + "primary": false, + "contains": [ + "url" + ], + "value_list": [], + "default": "", + "order": 0 + } + }, + "output": [ + { + "data_path": "action_result.parameter.url", + "data_type": "string", + "contains": [ + "url" + ], + "column_name": "url", + "column_order": 0 + }, + { + "data_path": "action_result.status", + "data_type": "string", + "column_name": "status", + "column_order": 1 + }, + { + "data_path": "action_result.message", + "data_type": "string" + }, + { + "data_path": "summary.total_objects", + "data_type": "numeric" + }, + { + "data_path": "summary.total_objects_successful", + "data_type": "numeric" + } + ], + "render": { + "type": "table" + }, + "versions": "EQ(*)" + }, + { + "action": "get screenshot", + "identifier": "get_screenshot", + "description": "Get Screenshot of URL", + "verbose": "Create a screenshot of a URL", + "type": "investigate", + "read_only": true, + "parameters": { + "url": { + "description": "URL of website", + "data_type": "string", + "required": true, + "primary": true, + "contains": [ + "url" + ], + "value_list": [], + "default": "", + "order": 0 + }, + "type": { + "description": "Type of Screenshot PNG or JEPG", + "data_type": "string", + "required": true, + "primary": false, + "contains": [], + "value_list": [ + "jpeg", + "png" + ], + "default": "png", + "order": 1 + }, + "quality": { + "description": "Quality of image", + "data_type": "numeric", + "required": true, + "primary": false, + "contains": [], + "value_list": [], + "default": "80", + "order": 2 + }, + "fullpage": { + "description": "Full Page Screenshot", + "data_type": "boolean", + "required": true, + "primary": false, + "contains": [], + "default": true, + "order": 3 + } + }, + "output": [ + { + "data_path": "action_result.parameter.url", + "data_type": "string", + "contains": [ + "url" + ], + "column_name": "url", + "column_order": 0 + }, + { + "data_path": "action_result.parameter.type", + "data_type": "string", + "contains": [], + "column_name": "type", + "column_order": 1 + }, + { + "data_path": "action_result.parameter.quality", + "data_type": "numeric", + "contains": [], + "column_name": "quality", + "column_order": 2 + }, + { + "data_path": "action_result.parameter.fullpage", + "data_type": "boolean", + "contains": [], + "column_name": "fullpage", + "column_order": 3 + }, + { + "data_path": "action_result.status", + "data_type": "string", + "column_name": "status", + "column_order": 4 + }, + { + "data_path": "action_result.message", + "data_type": "string" + }, + { + "data_path": "summary.total_objects", + "data_type": "numeric" + }, + { + "data_path": "summary.total_objects_successful", + "data_type": "numeric" + } + ], + "render": { + "type": "table" + }, + "versions": "EQ(*)" + } + ] +} \ No newline at end of file diff --git a/phbrowserlessio/browserlessio.svg b/phbrowserlessio/browserlessio.svg new file mode 100644 index 0000000..2ec7b33 --- /dev/null +++ b/phbrowserlessio/browserlessio.svg @@ -0,0 +1 @@ + diff --git a/phbrowserlessio/browserlessio_connector.py b/phbrowserlessio/browserlessio_connector.py new file mode 100644 index 0000000..f81f89a --- /dev/null +++ b/phbrowserlessio/browserlessio_connector.py @@ -0,0 +1,438 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# ----------------------------------------- +# Phantom sample App Connector python file +# ----------------------------------------- + +# Python 3 Compatibility imports +from __future__ import print_function, unicode_literals + +# Phantom App imports +import phantom.app as phantom +from phantom.base_connector import BaseConnector +from phantom.action_result import ActionResult +from phantom.vault import Vault + +# Usage of the consts file is recommended +# from browserlessio_consts import * +import requests +import json +import os +import hashlib + + +class RetVal(tuple): + + def __new__(cls, val1, val2=None): + return tuple.__new__(RetVal, (val1, val2)) + + +class BrowserlessIoConnector(BaseConnector): + + def __init__(self): + + # Call the BaseConnectors init first + super(BrowserlessIoConnector, self).__init__() + + self._state = None + + # Variable to hold a base_url in case the app makes REST calls + # Do note that the app json defines the asset config, so please + # modify this as you deem fit. + self._base_url = None + + def _process_empty_response(self, r, action_result): + if r.status_code == 200: + return RetVal(phantom.APP_SUCCESS, {}) + + return RetVal( + action_result.set_status( + phantom.APP_ERROR, "Empty response and no information in the header" + ), None + ) + + def _process_file_response(self, r, action_result): + # An html response, treat it like an error + status_code = r.status_code + if 200 <= status_code < 399: + # Send contents to Function + return RetVal(action_result.set_status(phantom.APP_SUCCESS), r.content) + return RetVal(action_result.set_status(phantom.APP_ERROR, "Unable to extract files"), None) + + def _process_json_response(self, r, action_result): + # Try a json parse + try: + resp_json = r.json() + except Exception as e: + return RetVal( + action_result.set_status( + phantom.APP_ERROR, "Unable to parse JSON response. Error: {0}".format(str(e)) + ), None + ) + + # Please specify the status codes here + if 200 <= r.status_code < 399: + return RetVal(phantom.APP_SUCCESS, resp_json) + + # You should process the error returned in the json + message = "Error from server. Status Code: {0} Data from server: {1}".format( + r.status_code, + r.text.replace(u'{', '{{').replace(u'}', '}}') + ) + + return RetVal(action_result.set_status(phantom.APP_ERROR, message), None) + + def _process_response(self, r, action_result): + # store the r_text in debug data, it will get dumped in the logs if the action fails + if hasattr(action_result, 'add_debug_data'): + action_result.add_debug_data({'r_status_code': r.status_code}) + action_result.add_debug_data({'r_text': r.text}) + action_result.add_debug_data({'r_headers': r.headers}) + + # Process each 'Content-Type' of response separately + if 'json' in r.headers.get('Content-Type', ''): + return self._process_json_response(r, action_result) + + if 'html' in r.headers.get('Content-Type', ''): + return self._process_file_response(r, action_result) + + if 'image' in r.headers.get('Content-Type', ''): + return self._process_file_response(r, action_result) + + if 'pdf' in r.headers.get('Content-Type', ''): + return self._process_file_response(r, action_result) + + # it's not content-type that is to be parsed, handle an empty response + if not r.text: + return self._process_empty_response(r, action_result) + + # everything else is actually an error at this point + message = "Can't process response from server. Status Code: {0} Data from server: {1}".format( + r.status_code, + r.text.replace('{', '{{').replace('}', '}}') + ) + + return RetVal(action_result.set_status(phantom.APP_ERROR, message), None) + + def _make_rest_call(self, endpoint, action_result, method="post", **kwargs): + # **kwargs can be any additional parameters that requests.request accepts + + config = self.get_config() + params = dict() + resp_json = None + + try: + request_func = getattr(requests, method) + except AttributeError: + return RetVal( + action_result.set_status(phantom.APP_ERROR, "Invalid method: {0}".format(method)), + resp_json + ) + + # Create a URL to connect to + url = self._rest_url + endpoint + if self._rest_token: + params['token'] = self._rest_token + + try: + r = request_func( + url, + params=params, + verify=config.get('verify_server_cert', False), + headers={'Content-Type': 'application/json' }, + **kwargs + ) + except Exception as e: + return RetVal( + action_result.set_status( + phantom.APP_ERROR, "Error Connecting to server. Details: {0}".format(str(e)) + ), resp_json + ) + + return self._process_response(r, action_result) + + def _handle_test_connectivity(self, param): + # Add an action result object to self (BaseConnector) to represent the action for this param + action_result = self.add_action_result(ActionResult(dict(param))) + + self.save_progress("Connecting to endpoint") + query = '{"url":"https://google.com"}' + # make rest call + ret_val, response = self._make_rest_call('/stats', action_result, data=json.dumps(query)) + + if phantom.is_fail(ret_val): + # the call to the 3rd party device or service failed, action result should contain all the error details + # for now the return is commented out, but after implementation, return from here + self.save_progress("Test Connectivity Failed.") + return action_result.get_status() + + if response.status_code == 200: + # Return success + self.save_progress("Test Connectivity Passed") + return action_result.set_status(phantom.APP_SUCCESS) + + # For now return Error with a message, in case of success we don't set the message, but use the summary + # return action_result.set_status(phantom.APP_ERROR, "Action not yet implemented") + + def _handle_get_pdf(self, param): + # Implement the handler here + # use self.save_progress(...) to send progress messages back to the platform + self.save_progress("In action handler for: {0}".format(self.get_action_identifier())) + + # Add an action result object to self (BaseConnector) to represent the action for this param + action_result = self.add_action_result(ActionResult(dict(param))) + + # Access action parameters passed in the 'param' dictionary + + # Required values can be accessed directly + s_url = param['url'] + headerfooter = param['headerfooter'] + printbackground = param['printbackground'] + landscape = param['landscape'] + query = {"url": "{}".format(s_url), "options": { + "displayHeaderFooter": "{}".format(headerfooter), + "printBackground": "{}".format(printbackground), + "landscape": "{}".format(landscape) + }} + + # make rest call + ret_val, response = self._make_rest_call( + '/pdf', action_result, data=json.dumps(query)) + + if phantom.is_fail(ret_val): + return action_result.get_status() + else: + file_name = s_url + "_screenshot.pdf" + if hasattr(Vault, 'create_attachment'): + vault_ret = Vault.create_attachment(response, self.get_container_id(), file_name=file_name) + else: + if hasattr(Vault, 'get_vault_tmp_dir'): + temp_dir = Vault.get_vault_tmp_dir() + else: + temp_dir = '/opt/phantom/vault/tmp' + temp_dir = temp_dir + ('/{}').format(hashlib.md5(file_name).hexdigest()) + os.makedirs(temp_dir) + file_path = os.path.join(temp_dir, 'tmpfile') + with open(file_path, 'wb') as (f): + f.write(response) + vault_ret = Vault.add_attachment(file_path, self.get_container_id(), file_name=file_name) + if vault_ret.get('succeeded'): + action_result.set_status(phantom.APP_SUCCESS, 'Downloaded PDF') + summary = {phantom.APP_JSON_VAULT_ID: vault_ret[phantom.APP_JSON_HASH], + phantom.APP_JSON_NAME: file_name, + 'vault_file_path': Vault.get_file_path(vault_ret[phantom.APP_JSON_HASH]), + phantom.APP_JSON_SIZE: vault_ret.get(phantom.APP_JSON_SIZE)} + action_result.update_summary(summary) + return action_result.get_status() + + def _handle_get_content(self, param): + # Implement the handler here + # use self.save_progress(...) to send progress messages back to the platform + self.save_progress("In action handler for: {0}".format(self.get_action_identifier())) + + # Add an action result object to self (BaseConnector) to represent the action for this param + action_result = self.add_action_result(ActionResult(dict(param))) + + # Required values can be accessed directly + s_url = param['url'] + + query = {"url": "{}".format(s_url)} + # make rest call + ret_val, response = self._make_rest_call( + '/content', action_result, data=json.dumps(query)) + + if phantom.is_fail(ret_val): + return action_result.get_status() + else: + file_name = s_url + "_contents.txt" + if hasattr(Vault, 'create_attachment'): + vault_ret = Vault.create_attachment(response, self.get_container_id(), file_name=file_name) + else: + if hasattr(Vault, 'get_vault_tmp_dir'): + temp_dir = Vault.get_vault_tmp_dir() + else: + temp_dir = '/opt/phantom/vault/tmp' + temp_dir = temp_dir + ('/{}').format(hashlib.md5(file_name).hexdigest()) + os.makedirs(temp_dir) + file_path = os.path.join(temp_dir, 'tmpfile') + with open(file_path, 'wb') as (f): + f.write(response) + vault_ret = Vault.add_attachment(file_path, self.get_container_id(), file_name=file_name) + if vault_ret.get('succeeded'): + action_result.set_status(phantom.APP_SUCCESS, 'Downloaded HTML Contents') + summary = {phantom.APP_JSON_VAULT_ID: vault_ret[phantom.APP_JSON_HASH], + phantom.APP_JSON_NAME: file_name, + 'vault_file_path': Vault.get_file_path(vault_ret[phantom.APP_JSON_HASH]), + phantom.APP_JSON_SIZE: vault_ret.get(phantom.APP_JSON_SIZE)} + action_result.update_summary(summary) + return action_result.get_status() + + def _handle_get_screenshot(self, param): + # Implement the handler here + # use self.save_progress(...) to send progress messages back to the platform + self.save_progress("In action handler for: {0}".format(self.get_action_identifier())) + + # Add an action result object to self (BaseConnector) to represent the action for this param + action_result = self.add_action_result(ActionResult(dict(param))) + + # Required values can be accessed directly + s_url = param['url'] + ftype = param['type'] + quality = param['quality'] + fullpage = param['fullpage'] + + jpeg_query = {"url": "{}".format(s_url), "options": { + "type": "{}".format(ftype), + "quality": "{}".format(quality), + "fullPage": "{}".format(fullpage) + }} + png_query = {"url": "{}".format(s_url), "options": { + "type": "{}".format(ftype), + "fullPage": "{}".format(fullpage) + }} + + if ftype == "png": + query = png_query + else: + query = jpeg_query + + # make rest call + ret_val, response = self._make_rest_call( + '/screenshot', action_result, data=json.dumps(query)) + + if phantom.is_fail(ret_val): + return action_result.get_status() + else: + file_name = s_url + "_screenshot." + ftype + if hasattr(Vault, 'create_attachment'): + vault_ret = Vault.create_attachment(response, self.get_container_id(), file_name=file_name) + else: + if hasattr(Vault, 'get_vault_tmp_dir'): + temp_dir = Vault.get_vault_tmp_dir() + else: + temp_dir = '/opt/phantom/vault/tmp' + temp_dir = temp_dir + ('/{}').format(hashlib.md5(file_name).hexdigest()) + os.makedirs(temp_dir) + file_path = os.path.join(temp_dir, 'tmpfile') + with open(file_path, 'wb') as (f): + f.write(response) + vault_ret = Vault.add_attachment(file_path, self.get_container_id(), file_name=file_name) + if vault_ret.get('succeeded'): + action_result.set_status(phantom.APP_SUCCESS, 'Downloaded Screenshot') + summary = {phantom.APP_JSON_VAULT_ID: vault_ret[phantom.APP_JSON_HASH], + phantom.APP_JSON_NAME: file_name, + 'vault_file_path': Vault.get_file_path(vault_ret[phantom.APP_JSON_HASH]), + phantom.APP_JSON_SIZE: vault_ret.get(phantom.APP_JSON_SIZE)} + action_result.update_summary(summary) + return action_result.get_status() + + def handle_action(self, param): + ret_val = phantom.APP_SUCCESS + + # Get the action that we are supposed to execute for this App Run + action_id = self.get_action_identifier() + + self.debug_print("action_id", self.get_action_identifier()) + + if action_id == 'test_connectivity': + ret_val = self._handle_test_connectivity(param) + + elif action_id == 'get_pdf': + ret_val = self._handle_get_pdf(param) + + elif action_id == 'get_content': + ret_val = self._handle_get_content(param) + + elif action_id == 'get_screenshot': + ret_val = self._handle_get_screenshot(param) + + return ret_val + + def initialize(self): + # Load the state in initialize, use it to store data + # that needs to be accessed across actions + self._state = self.load_state() + + # get the asset config + config = self.get_config() + + self._rest_url = config.get('URL') + self._rest_token = config.get('token') + self._base_url = config.get('base_url') + + return phantom.APP_SUCCESS + + def finalize(self): + # Save the state, this data is saved across actions and app upgrades + self.save_state(self._state) + return phantom.APP_SUCCESS + + +def main(): + import pudb + import argparse + + pudb.set_trace() + + argparser = argparse.ArgumentParser() + + argparser.add_argument('input_test_json', help='Input Test JSON file') + argparser.add_argument('-u', '--username', help='username', required=False) + argparser.add_argument('-p', '--password', help='password', required=False) + + args = argparser.parse_args() + session_id = None + + username = args.username + password = args.password + + if username is not None and password is None: + + # User specified a username but not a password, so ask + import getpass + password = getpass.getpass("Password: ") + + if username and password: + try: + login_url = BrowserlessIoConnector._get_phantom_base_url() + '/login' + + print("Accessing the Login page") + r = requests.get(login_url, verify=False) + csrftoken = r.cookies['csrftoken'] + + data = dict() + data['username'] = username + data['password'] = password + data['csrfmiddlewaretoken'] = csrftoken + + headers = dict() + headers['Cookie'] = 'csrftoken=' + csrftoken + headers['Referer'] = login_url + + print("Logging into Platform to get the session id") + r2 = requests.post(login_url, verify=False, data=data, headers=headers) + session_id = r2.cookies['sessionid'] + except Exception as e: + print("Unable to get session id from the platform. Error: " + str(e)) + exit(1) + + with open(args.input_test_json) as f: + in_json = f.read() + in_json = json.loads(in_json) + print(json.dumps(in_json, indent=4)) + + connector = BrowserlessIoConnector() + connector.print_progress_message = True + + if session_id is not None: + in_json['user_session_token'] = session_id + connector._set_csrf_info(csrftoken, headers['Referer']) + + ret_val = connector._handle_action(json.dumps(in_json), None) + print(json.dumps(json.loads(ret_val), indent=4)) + + exit(0) + + +if __name__ == '__main__': + main() diff --git a/phbrowserlessio/browserlessio_consts.py b/phbrowserlessio/browserlessio_consts.py new file mode 100644 index 0000000..5d8df23 --- /dev/null +++ b/phbrowserlessio/browserlessio_consts.py @@ -0,0 +1 @@ +# Define your constants here diff --git a/phbrowserlessio/browserlessio_dark.svg b/phbrowserlessio/browserlessio_dark.svg new file mode 100644 index 0000000..eb17f94 --- /dev/null +++ b/phbrowserlessio/browserlessio_dark.svg @@ -0,0 +1 @@ + diff --git a/phbrowserlessio/exclude_files.txt b/phbrowserlessio/exclude_files.txt new file mode 100644 index 0000000..3b0afcd --- /dev/null +++ b/phbrowserlessio/exclude_files.txt @@ -0,0 +1,3 @@ +.git* +*.py +.vscode \ No newline at end of file diff --git a/phbrowserlessio/readme.html b/phbrowserlessio/readme.html new file mode 100644 index 0000000..58ee2ec --- /dev/null +++ b/phbrowserlessio/readme.html @@ -0,0 +1,4 @@ + + + Replace this text in the app's readme.html to contain more detailed information +