diff --git a/Packs/GenericAPIEventCollector/.pack-ignore b/Packs/GenericAPIEventCollector/.pack-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/GenericAPIEventCollector/.secrets-ignore b/Packs/GenericAPIEventCollector/.secrets-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.py b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.py new file mode 100644 index 000000000000..4a2ea1410b53 --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.py @@ -0,0 +1,636 @@ +import copy +import enum +from base64 import b64encode +from collections import namedtuple +from json import JSONDecodeError + +import urllib3 + +from CommonServerPython import * + +# Disable insecure warnings +urllib3.disable_warnings() + +DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' +DEFAULT_LIMIT = "1000" +MAX_INCIDENTS_TO_FETCH = 10_000 +PaginationLogic = namedtuple( + "PaginationLogic", + ( + "pagination_needed", + "pagination_field_name", + "pagination_flag", + ), + defaults=(False, '', ''), +) +TimestampFieldConfig = namedtuple( + "TimestampFieldConfig", + ( + "timestamp_field_name", + "timestamp_format", + ), + defaults=("", DATE_FORMAT), +) +RequestData = namedtuple( + "RequestData", + ( + "request_data", + "request_json", + "query_params", + ), + defaults=(True, None, None), +) + + +class PlaceHolders(enum.Enum): + LAST_FETCHED_ID = "@last_fetched_id" + LAST_FETCHED_DATETIME = "@last_fetched_datetime" + FIRST_FETCH_DATETIME = "@first_fetch_datetime" + FETCH_SIZE_LIMIT = "@fetch_size_limit" + + +class IdTypes(enum.Enum): + INTEGER = "integer" + STRING = "string" + + +ALL_ID_TYPES = [ + IdTypes.INTEGER.value, + IdTypes.STRING.value, +] + + +def datetime_to_timestamp_format(dt: datetime, timestamp_format: str) -> str: + demisto.debug(f"converting {dt} using format:{timestamp_format}") + if timestamp_format == "epoch": + return str(dt.timestamp()) + return dt.strftime(timestamp_format) + + +def timestamp_format_to_datetime(dt: str, timestamp_format: str) -> datetime: + demisto.debug(f"converting {dt} using format:{timestamp_format}") + if timestamp_format == "epoch": + return datetime.fromtimestamp(float(dt)) + return datetime.strptime(dt, timestamp_format) + + +def recursive_replace(org_dict: dict[Any, Any] | None, substitutions: list[tuple[Any, Any]]) -> dict[Any, Any] | None: + """ + Recursively replace values in a dictionary with provided substitutions. + Args: + org_dict: The dictionary to be modified. + substitutions: A list of tuples containing the old and new values to be replaced. + + Returns: The modified dictionary with the provided substitutions. + Examples: + >>> org_dict1 = {'a': 1, 'b': {'x': 'old', 'y': 2}} + >>> substitutions1 = [('old', 'new')] + >>> recursive_replace(org_dict1, substitutions1) + {'a': 1, 'b': {'x': 'new', 'y': 2}} + + """ + if org_dict is None: + return None + # Create a deep copy of the dictionary to avoid modifying the original + copy_dict = copy.deepcopy(org_dict) + + for key, value in copy_dict.items(): + if isinstance(value, dict): + # If value is a dictionary, recursively call this function + copy_dict[key] = recursive_replace(value, substitutions) + elif isinstance(value, str): + # If value is a string, perform substitutions + for old, new in substitutions: + value = value.replace(old, new) + copy_dict[key] = value + + return copy_dict + + +class Client(BaseClient): + """ + Client class to interact with the service API + + This Client implements API calls and does not contain any Demisto logic. + Should only do requests and return data. + It inherits from BaseClient defined in CommonServerPython. + Most calls use _http_request() that handles proxy, SSL verification, etc. + """ + + def search_events(self, + endpoint: str, + http_method: str, + request_data: RequestData, + ok_codes: list[int]) -> dict[Any, Any]: + """ + Searches for events using the API endpoint. + All the parameters are passed directly to the API as HTTP POST parameters in the request + + Args: + endpoint: API endpoint to send the request to. + http_method: HTTP method to use in the request. + request_data: data to send in the body of the request. + ok_codes: list of allowed response codes. + Returns: + dict: The raw response returned by the API. + """ + demisto.debug(f"Searching events for {endpoint}") + + return self._http_request( # type: ignore + method=http_method, + url_suffix=endpoint, + json_data=request_data.request_json, + ok_codes=tuple(ok_codes), + data=request_data.request_data, + params=request_data.query_params, + ) + + +def organize_events_to_xsiam_format(raw_events: Any, events_keys: list[str]) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = dict_safe_get(raw_events, events_keys, [], list, True) # type: ignore + return events + + +def get_time_field_from_event_to_dt(event: dict[str, Any], timestamp_field_config: TimestampFieldConfig) -> datetime: + timestamp: str | None = dict_safe_get(event, timestamp_field_config.timestamp_field_name) # noqa + if timestamp is None: + raise DemistoException(f"Timestamp field: {timestamp_field_config.timestamp_field_name} not found in event") + timestamp_str: str = iso8601_to_datetime_str(timestamp) + # Convert the timestamp to the desired format. + return timestamp_format_to_datetime(timestamp_str, timestamp_field_config.timestamp_format) + + +def is_pagination_needed(events: dict[Any, Any], pagination_logic: PaginationLogic) -> tuple[bool, Any]: + next_page_value = None + if pagination_needed := pagination_logic.pagination_needed: + if dict_safe_get(events, pagination_logic.pagination_flag): + pagination_needed = True + next_page_value = dict_safe_get(events, pagination_logic.pagination_field_name) + demisto.debug(f"Pagination needed - Next page value: {next_page_value}") + else: + demisto.debug("Pagination not detected in the response") + pagination_needed = False + else: + demisto.debug("Pagination not configured") + return pagination_needed, next_page_value + + +def fetch_events(client: Client, + params: dict[str, Any], + last_run: dict[Any, Any], + first_fetch_datetime: datetime, + endpoint: str, + http_method: str, + ok_codes: list[int], + events_keys: list[str], + timestamp_field_config: TimestampFieldConfig) -> tuple[dict[Any, Any], list[dict[Any, Any]]]: + last_fetched_datetime, pagination_logic, request_data = setup_search_events( + first_fetch_datetime, last_run, params, timestamp_field_config) + + # region Gets events & Searches for pagination + all_events_list: list[dict[str, Any]] = [] + pagination_needed: bool = True + id_type_lower: str | None = None + id_keys: list[str] = argToList(params.get('id_keys'), '.') + if id_keys: + id_type: str = params.get('id_type') # type: ignore[arg-type,assignment] + if id_type: + if id_type.lower() not in ALL_ID_TYPES: + return_error(f"ID type {id_type} but must be one of {', '.join(ALL_ID_TYPES)}") + return {}, [] + demisto.debug(f"ID type:{id_type}") + else: + return_error("ID type was not specified") + return {}, [] + + while pagination_needed: + + raw_events = client.search_events(endpoint=endpoint, + http_method=http_method, + request_data=request_data, + ok_codes=ok_codes) + events_list = organize_events_to_xsiam_format(raw_events, events_keys) + all_events_list.extend(events_list) + demisto.debug(f"{len(all_events_list)} events fetched") + pagination_needed, next_page_value = is_pagination_needed(raw_events, pagination_logic) + if pagination_needed: + request_json = {pagination_logic.pagination_field_name: next_page_value} + request_data = RequestData(request_data.request_data, request_json, request_data.query_params) + + # endregion + + # region Collect all events based on their last fetch time. + latest_created_datetime: datetime = last_fetched_datetime + last_fetched_id: Any | None = last_run.get(PlaceHolders.LAST_FETCHED_ID.value) + returned_event_list: list[dict[str, Any]] = [] + for event in all_events_list: + try: + incident_created_dt = get_time_field_from_event_to_dt(event, timestamp_field_config) + except DemistoException as e: + demisto.error(f"Error parsing timestamp for event: {event} exception: {e}") + continue + event['_time'] = incident_created_dt.isoformat() + + # to prevent duplicates, we are only adding events with creation_time > last fetched incident + if incident_created_dt > last_fetched_datetime: + demisto.debug(f"Adding event with creation time: {incident_created_dt}") + returned_event_list.append(event) + latest_created_datetime = max(latest_created_datetime, incident_created_dt) + else: + demisto.debug(f'This event is to old to pull, creation time: {incident_created_dt}') + + # Handle the last event id. + if id_keys: + current_id: Any = dict_safe_get(event, id_keys) + demisto.debug(f'Current event id: {current_id}') + if ( + last_fetched_id is not None + and id_type_lower == IdTypes.INTEGER.value + ): + last_fetched_id = str(max(int(last_fetched_id), int(current_id))) # noqa + else: + # We assume the last event contains the last id. + last_fetched_id = current_id + + # region Saves important parameters here to Integration context / last run + demisto.debug(f'next run:{latest_created_datetime}') + # Save the next_run as a dict with the last_fetch key to be stored + next_run = { + PlaceHolders.LAST_FETCHED_DATETIME.value: latest_created_datetime.isoformat(), + PlaceHolders.FIRST_FETCH_DATETIME.value: last_run[PlaceHolders.FIRST_FETCH_DATETIME.value], + } + if last_fetched_id is not None: + next_run[PlaceHolders.LAST_FETCHED_ID.value] = str(last_fetched_id) + # endregion + + return next_run, returned_event_list + + +def setup_search_events(first_fetch_datetime: datetime, + last_run: dict, + params: dict, + timestamp_field_config: TimestampFieldConfig) -> tuple[datetime, PaginationLogic, RequestData]: + # region Gets first fetch time. + if PlaceHolders.FIRST_FETCH_DATETIME.value not in last_run: + first_fetch_datetime_str = datetime_to_timestamp_format(first_fetch_datetime, + timestamp_field_config.timestamp_format) + demisto.debug(f"Setting first fetch datetime: {first_fetch_datetime_str}") + last_run[PlaceHolders.FIRST_FETCH_DATETIME.value] = first_fetch_datetime_str + # endregion + + # region Handle first fetch time + last_fetched_datetime_str: str | None = last_run.get(PlaceHolders.LAST_FETCHED_DATETIME.value) + if last_fetched_datetime_str is None: + # if missing, use what provided via first_fetch_timestamp + last_fetched_datetime: datetime = first_fetch_datetime + first_fetch_for_this_integration: bool = True + demisto.debug(f"First fetch for integration, Last fetched datetime: {last_fetched_datetime_str}") + else: + # otherwise, use the stored last fetch + demisto.debug(f"Last fetched datetime: {last_fetched_datetime_str}") + last_fetched_datetime = datetime.fromisoformat(last_fetched_datetime_str) + first_fetch_for_this_integration = False + # endregion + + # region load request arguments + request_data: dict[Any, Any] | None = parse_json_param(params.get('request_data'), 'request_data') + request_json: dict[Any, Any] | None = parse_json_param(params.get('request_json'), 'request_json') + query_params: dict[Any, Any] | None = parse_json_param(params.get('query_params'), 'query_params') + pagination_logic = extract_pagination_params(params) + # If we've an initial query argument, we try with it. + if first_fetch_for_this_integration: + demisto.debug("First fetch for integration, Checking if one of the 'initial_*' params is set to get the initial request") + if initial_query_params := params.get('initial_query_params'): + demisto.debug(f"Initial query params: {initial_query_params}") + query_params = parse_json_param(initial_query_params, 'initial_query_params') + if initial_pagination_params := params.get('initial_pagination_params'): + demisto.debug(f"Initial pagination params: {initial_pagination_params}") + pagination_logic = extract_pagination_params(initial_pagination_params) + if initial_request_data := params.get('initial_request_data'): + demisto.debug(f"Initial request data: {initial_request_data}") + request_data = parse_json_param(initial_request_data, 'initial_request_data') + if initial_request_json := params.get('initial_request_json'): + demisto.debug(f"Initial request json: {initial_request_json}") + request_json = parse_json_param(initial_request_json, 'initial_request_json') + + # endregion + + # region Handle substitutions + + # We're replacing the placeholders in the request parameters with the actual values from the last run. + # This is how we make the requests from the API to be more dynamic and reusable. + substitutions: list[tuple[str, str]] = [ + (place_holder.value, last_run.get(place_holder.value)) for place_holder in PlaceHolders # type: ignore[misc] + if last_run.get(place_holder.value) is not None + ] + + # region request size limit + if 'limit' in params: + limit = int(params['limit']) + demisto.debug(f"Setting request size limit to: {limit}") + if limit > MAX_INCIDENTS_TO_FETCH: + return_error(f"The maximum allowed limit is {MAX_INCIDENTS_TO_FETCH} events per fetch. " + f"Please update the limit parameter to a value of {MAX_INCIDENTS_TO_FETCH} or less.") + substitutions.append((PlaceHolders.FETCH_SIZE_LIMIT.value, str(limit))) + # endregion + substitutions_query_params: dict[Any, Any] | None = recursive_replace(query_params, substitutions) + demisto.debug(f"Query params subs: {substitutions_query_params}") + substitutions_request_json: dict[Any, Any] | None = recursive_replace(request_json, substitutions) + demisto.debug(f"Request json subs: {substitutions_request_json}") + substitutions_request_data: dict[Any, Any] | None = recursive_replace(request_data, substitutions) + demisto.debug(f"Request data subs: {substitutions_request_data}") + # endregion + return last_fetched_datetime, pagination_logic, RequestData(substitutions_request_data, substitutions_request_json, + substitutions_query_params) + + +def iso8601_to_datetime_str(iso8601_time: str) -> str: + # In case the time format is ISO 8601 - ISO supports 7 digits while datetime in python supports only 6, + # so we need to reduce 1 number from the nanoseconds + if '.' in iso8601_time: + timestamp_without_nanoseconds, nanoseconds = re.split("[.]", iso8601_time, maxsplit=1) + fractional = nanoseconds.rstrip('Z')[:6] # Keep only the first 6 digits. + new_iso8601_time = f"{timestamp_without_nanoseconds}.{fractional}Z" + demisto.debug(f"Converted ISO 8601:{iso8601_time} to:{new_iso8601_time}") + return new_iso8601_time + return iso8601_time + + +def test_module(client: Client, + endpoint: str, + http_method: str, + ok_codes: list[int], + request_data: RequestData): + try: + events = client.search_events(endpoint=endpoint, + http_method=http_method, + request_data=request_data, + ok_codes=ok_codes) + demisto.debug(f"{events!s}") + except DemistoException as e: + error = str(e) + if 'Forbidden' in error or 'Unauthorized' in error: + return_error(f'Authorization Error: make sure Username/Password/Token is correctly set.\nError:{error}') + else: + raise e + + return_results("ok") + + +def parse_json_param(json_param_value: Any, json_param_name) -> dict | None: + if json_param_value and json_param_value != 'None': + try: + demisto.debug(f"parsing argument: {json_param_name}") + return safe_load_json(json_param_value) + except JSONDecodeError as exception: + err_msg = f"Argument {json_param_name} could not be parsed as a valid JSON: {exception}" + demisto.error(err_msg) + raise DemistoException(err_msg, exception) from exception + return None + + +def generate_headers(params: dict[str, Any]) -> dict[Any, Any]: + + headers = generate_authentication_headers(params) + if ((add_fields_to_header := str(params.get('add_fields_to_header'))) + and (parsed := parse_json_param(add_fields_to_header, 'add_fields_to_header')) is not None): + headers.update(parsed) + return headers + + +def generate_authentication_headers(params: dict[Any, Any]) -> dict[Any, Any]: + authentication = params.get('authentication') + if authentication == 'Basic': + username = params.get("credentials", {}).get("identifier") + password = params.get("credentials", {}).get("password") + if password: + demisto.debug("Adding Password to sensitive logs strings") + add_sensitive_log_strs(password) + else: + demisto.error("Password is required for Basic Authentication.") + return_error("Password is required for Basic Authentication.") + demisto.debug(f"Authenticating with Basic Authentication, username: {username}") + # encode username and password in a basic authentication method + auth_credentials = f'{username}:{password}' + encoded_credentials = b64encode(auth_credentials.encode()).decode('utf-8') + add_sensitive_log_strs(encoded_credentials) + return { + 'Authorization': f'Basic {encoded_credentials}', + } + if authentication == 'Bearer': + demisto.debug("Authenticating with Bearer Authentication") + if token := params.get('token', {}).get("password"): + demisto.debug("Adding Token to sensitive logs strings") + add_sensitive_log_strs(token) + else: + demisto.error("API Token is required.") + return_error("API Token is required.") + return { + 'Authorization': f'Bearer {token}', + } + if authentication == 'Token': + demisto.debug("Authenticating with Token Authentication") + if token := params.get('token', {}).get("password"): + demisto.debug("Adding Token to sensitive logs strings") + add_sensitive_log_strs(token) + else: + demisto.error("API Token is required.") + return_error("API Token is required.") + return { + 'Authorization': f'Token {token}', + } + if authentication == 'Api-Key': + demisto.debug("Authenticating with Api-Key Authentication") + if token := params.get('token', {}).get("password"): + demisto.debug("Adding Token to sensitive logs strings") + add_sensitive_log_strs(token) + else: + demisto.error("API Token is required.") + return_error("API Token is required.") + return { + 'api-key': f'{token}', + } + if authentication == 'RawToken': + demisto.debug("Authenticating with raw token") + if token := params.get('token', {}).get("password"): + demisto.debug("Adding Token to sensitive logs strings") + add_sensitive_log_strs(token) + else: + demisto.error("API Token is required.") + return_error("API Token is required.") + return { + 'Authorization': f'{token}', + } + if authentication == 'No Authorization': + demisto.debug("Connecting without Authorization") + return {} + + err_msg = ("Please insert a valid authentication method, options are: Basic, Bearer, Token, Api-Key, RawToken" + f"No Authorization, got: {authentication}") + demisto.error(err_msg) + return_error(err_msg) + return {} + + +def get_events_command(client: Client, + endpoint: str, + http_method: str, + ok_codes: list[int], + request_data: RequestData, + events_keys: list[str], + limit: int) -> tuple[Dict[str, Any], CommandResults]: + raw_events = client.search_events(endpoint=endpoint, + http_method=http_method, + request_data=request_data, + ok_codes=ok_codes) + events = organize_events_to_xsiam_format(raw_events, events_keys) + demisto.debug(f"Got {len(events)} events") + return raw_events, CommandResults( + readable_output=tableToMarkdown('Generic Events', events[:limit], sort_headers=False), + ) + + +def extract_pagination_params(params: dict[str, str]) -> PaginationLogic: + pagination_needed: bool = argToBoolean(params.get('pagination_needed', False)) + pagination_field_name: list[str] | None = argToList(params.get('pagination_field_name'), '.') + pagination_flag: list[str] | None = argToList(params.get('pagination_flag'), '.') + pagination_logic = PaginationLogic(pagination_needed, pagination_field_name, pagination_flag) + if pagination_logic.pagination_needed: + demisto.debug("Pagination logic - Pagination Needed, " + f"pagination_field_name: {pagination_logic.pagination_field_name}, " + f"pagination_flag: {pagination_logic.pagination_flag}") + if not pagination_logic.pagination_field_name: + return_error('Pagination field name is missing') + if not pagination_logic.pagination_flag: + return_error('Pagination flag is missing') + else: + demisto.debug("Pagination logic - Pagination Not Needed") + return pagination_logic + + +def main() -> None: # pragma: no cover + """ + main function, parses params and runs command functions. + """ + try: + params = demisto.params() + + # region Gets the service API url endpoint and method. + base_url: str = params.get('base_url') + endpoint: str = params.get('endpoint') + http_method: str | None = params.get('http_method') + ok_codes: list[int] = argToList(params.get('ok_codes', '200,201,202'), transform=int) + demisto.debug(f"base url: {base_url}, endpoint: {endpoint}, http method: {http_method}, ok codes: {ok_codes}") + if not base_url: + return_error('Base URL is missing') + if not endpoint: + return_error('Endpoint is missing') + if http_method is None: + return_error('HTTP method is missing') + if not http_method or http_method.upper() not in ['GET', 'POST']: + return_error(f'HTTP method is not valid, please choose between GET and POST, got: {http_method}') + # endregion + + # region Gets the timestamp field configuration + if not (timestamp_field_name_param := params.get('timestamp_field_name')): + return_error('Timestamp field is missing') + timestamp_field_name: list[str] = argToList(timestamp_field_name_param, '.') + timestamp_field_config = TimestampFieldConfig(timestamp_field_name, params.get('timestamp_format', DATE_FORMAT)) + demisto.debug(f"Timestamp field configuration - field_name: {timestamp_field_config.timestamp_field_name}, " + f"format: {timestamp_field_config.timestamp_format}") + # endregion + + # region Gets the events keys + events_keys: list[str] = argToList(params.get('events_keys'), '.') + demisto.debug(f"Events keys: {events_keys}") + # endregion + + # How much time before the first fetch to retrieve incidents. + first_fetch_datetime: datetime = arg_to_datetime( # type: ignore[assignment] + arg=params.get('first_fetch', '3 days'), + arg_name='First fetch time', + required=True + ) + + # if your Client class inherits from BaseClient, it handles system proxy + # out of the box, pass ``proxy`` to the Client constructor. + proxy: bool = argToBoolean(params.get('proxy', False)) + verify: bool = not argToBoolean(params.get('insecure', False)) + + # Create a client object. + client = Client( + base_url=base_url, + verify=verify, + headers=generate_headers(params), + proxy=proxy + ) + vendor: str = params.get('vendor').lower() + raw_product: str = params.get('product').lower() + product: str = f"{raw_product}_generic" + demisto.debug(f"Vendor: {vendor}, Raw Product: {raw_product}, Product: {product}") + + command: str = demisto.command() + demisto.debug(f"Command being called is {command}") + if command == 'test-module': + # Forcing the limit to 1 to ensure that the test module runs quickly. + params['limit'] = 1 + _, _, request_data = setup_search_events( + first_fetch_datetime, {}, params, timestamp_field_config) + test_module( + client=client, + endpoint=endpoint, + http_method=http_method, # type: ignore[arg-type] + request_data=request_data, + ok_codes=ok_codes, + ) + + elif command == 'fetch-events': + last_run = demisto.getLastRun() # getLastRun() gets the last run dict. + demisto.debug(f"Last run: {last_run}") + next_run, events = fetch_events( + client=client, + params=params, + last_run=last_run, + first_fetch_datetime=first_fetch_datetime, + endpoint=endpoint, + http_method=http_method, # type: ignore[arg-type] + ok_codes=ok_codes, + events_keys=events_keys, + timestamp_field_config=timestamp_field_config, + ) + + # Send to XSIAM dataset. + demisto.debug(f"Sending {len(events)} events from fetch") + send_events_to_xsiam(events, vendor=vendor, product=product) # noqa + + # saves next_run for the time fetch-incidents are invoked. + demisto.debug(f"setting last run:{next_run}") + demisto.setLastRun(next_run) + + elif command == "generic-api-event-collector-get-events": + args: dict[Any, Any] = demisto.args() + should_push_events: bool = argToBoolean(args.get("should_push_events")) + limit: int = arg_to_number(args.get("limit", DEFAULT_LIMIT), "limit", True) # type: ignore[assignment] + demisto.debug(f"should_push_events: {should_push_events}, limit: {limit}") + last_fetched_datetime, pagination_logic, request_data = setup_search_events( + first_fetch_datetime, demisto.getLastRun(), params, timestamp_field_config) + raw_events, results = get_events_command(client, endpoint, + http_method, # type: ignore[arg-type] + ok_codes, + request_data, events_keys, limit) + demisto.debug("Fetched events") + return_results(results) + + if should_push_events: + events = organize_events_to_xsiam_format(raw_events, events_keys) + demisto.debug(f"Sending {len(events)} events from command") + send_events_to_xsiam(events, vendor=vendor, product=product) # noqa + + except Exception as e: + # Log exceptions and return errors + demisto.error(traceback.format_exc()) # print the traceback + return_error(f"Failed to execute {demisto.command()} command.\nError:\n{str(e)}") + + +if __name__ in ('__main__', '__builtin__', 'builtins'): # pragma: no cover + main() diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.yml b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.yml new file mode 100644 index 000000000000..bcb5071f0d1a --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.yml @@ -0,0 +1,255 @@ +category: Analytics & SIEM +sectionOrder: +- Connect +- Collect +commonfields: + id: GenericAPIEventCollector + version: -1 +configuration: + - name: base_url + section: Connect + display: Server URL + type: 0 + required: true + - name: endpoint + section: Connect + display: Endpoint + type: 0 + required: true + additionalinfo: Add the endpoint you want to collect data from (Alert/Events, etc.). + - name: authentication + section: Connect + display: Authentication Type + type: 15 + required: true + options: + - Basic + - Token + - Bearer + - Api-Key + - RawToken + - No Authorization + additionalinfo: Select the authentication method. + - name: http_method + section: Connect + display: 'HTTP Method' + defaultvalue: GET + type: 15 + required: true + additionalinfo: The HTTP method of the request to the API. + options: + - GET + - POST + - name: token + type: 9 + displaypassword: API Token + hiddenusername: true + required: false + additionalinfo: "API Key to access the service REST API." + section: Connect + - name: credentials + display: Username + type: 9 + required: false + section: Connect + displaypassword: Password + additionalinfo: Username & Password to use for basic authentication. + - name: add_fields_to_header + section: Connect + display: Add Fields To header + type: 0 + required: false + advanced: true + additionalinfo: 'If the additional header is required, + add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: proxy + section: Connect + display: Use system proxy settings + defaultvalue: 'false' + type: 8 + required: false + advanced: true + - name: insecure + section: Connect + display: Trust any certificate (not secure) + type: 8 + required: false + advanced: true + - name: vendor + section: Collect + display: Vendor + type: 0 + required: true + additionalinfo: Enter vendor name for dataset. + - name: product + section: Collect + display: Product + type: 0 + required: true + additionalinfo: Enter product name for dataset. + - name: request_data + section: Collect + display: Request data + type: 0 + required: false + advanced: true + additionalinfo: 'If required to pass DATA when calling the API to collect data, + add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: initial_request_data + section: Collect + display: Initial request data + type: 0 + required: false + advanced: true + additionalinfo: 'If the product requires a different initial DATA, add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: request_json + section: Collect + display: Request JSON parameters + type: 0 + required: false + advanced: true + additionalinfo: 'If required to pass JSON data when calling the API to collect data, add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: initial_request_json + section: Collect + display: Initial request JSON parameters + type: 0 + required: false + advanced: true + additionalinfo: 'If the product requires a different initial request JSON, add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: query_params + section: Collect + display: Query parameters + type: 0 + required: false + advanced: true + additionalinfo: 'If required to filter the results using query parameters + please add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: initial_query_params + section: Collect + display: Initial query parameters + type: 0 + required: false + advanced: true + additionalinfo: 'If the product requires a different initial query parameters for the first fetch call, + add it here in dictionary format {unique_field : 286}. If there''s + a need to add more then one, use it in this format: {''field-1'': value_example, + ''field-2'': value_2, ''field-3'': value_3}' + - name: pagination_needed + section: Collect + display: Is pagination needed + type: 8 + additionalinfo: 'If the API JSON response supports events pagination.' + - name: pagination_field_name + section: Collect + display: Pagination field name + type: 0 + required: false + additionalinfo: 'Next page field in JSON response, e.g., "cursor", "next_page"' + - name: pagination_flag + section: Collect + display: Pagination flag + type: 0 + required: false + additionalinfo: 'Next page existence in JSON response e.g., "has_more", "next"' + - name: timestamp_format + section: Collect + display: 'Timestamp format of the event creation time or "epoch".' + type: 0 + required: false + additionalinfo: 'Python compatible datetime formatting (e.g., "%Y-%m-%dT%H:%M:%S.%fZ" or "%Y.%m.%d %H:%M:%S") or "epoch" to use UNIX epoch time.' + - name: timestamp_field_name + section: Collect + display: Timestamp field + type: 0 + required: true + additionalinfo: 'The name of the event creation time in the response data, e.g., "timestamp" or "created_at".' + - name: events_keys + section: Collect + display: 'Events lookup path in the response JSON, dot-separated, e.g. ,"data.items".' + type: 0 + required: false + additionalinfo: 'Where within the response object to find the events list.' + - name: id_keys + section: Collect + display: 'Event ID lookup path in the event response JSON, dot-separated, e.g., "id".' + type: 0 + required: false + additionalinfo: 'Where within the event object to find the event ID.' + - name: id_type + section: Collect + display: 'The type of ID field, either "integer" or "string"' + options: + - integer + - string + type: 15 + required: false + additionalinfo: 'ID field of type integer are comparable and when last fetched ID is the + maximum ID between the fetched events, when the type is string, the last fetched ID is the last event returned from the API.' + - name: ok_codes + section: Collect + display: 'Allowed HTTP status codes for successful response from the API' + type: 0 + required: false + defaultvalue: '200' + additionalinfo: 'OK codes is a comma-separated list (e.g., "200,201,202"). Default is "200".' + - name: limit + display: Number of incidents to fetch per fetch. + type: 0 + defaultvalue: 1000 + section: Collect + required: false + - name: isFetchEvents + section: Collect + display: Fetch Events + advanced: true + type: 8 + required: false + - name: eventFetchInterval + section: Collect + display: Events Fetch Interval + advanced: true + defaultvalue: "1" + type: 19 + required: false +description: Collect logs from 3rd party vendors using API. +display: Generic API Event Collector (Beta) +name: GenericAPIEventCollector +beta: true +script: + commands: + - name: generic-api-event-collector-get-events + description: Gets events from 3rd party vendor. + arguments: + - name: should_push_events + auto: PREDEFINED + defaultValue: 'false' + description: If true, the command will create events, otherwise it will only display them. + predefined: + - 'true' + - 'false' + required: true + - name: limit + description: Maximum number of results to return. + runonce: false + isfetchevents: true + script: '-' + type: python + subtype: python3 + dockerimage: demisto/python3:3.12.8.1983910 +fromversion: 6.8.0 +marketplaces: + - marketplacev2 +tests: + - No tests (auto formatted) diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_description.md b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_description.md new file mode 100644 index 000000000000..616041aad943 --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_description.md @@ -0,0 +1,33 @@ + +The Generic API Event Collector allows you to ingest data from any API endpoint into Cortex. +By configuring this collector, you can gather data from various systems and bring it into the Cortex ecosystem for better analysis and correlation. +Please note that this integration is currently in Beta, and as such, it may be subject to future changes. +# Configuration Guide +To successfully set up the Generic API Event Collector, you need to provide the following mandatory configuration fields: +1. Vendor and Product + This information is required to define the dataset name for storing the collected data. It is crucial that the correct + Vendor and Product values are added so that data can be ingested and categorized properly. The name of the ingested dataset will be in the format: `{Vendor}_{Product}_generic_raw` +2. Server URL + This is the URL of the server to which the collector will connect to gather data. Ensure that the URL is accessible and correct to enable proper data retrieval. +3. API Endpoint + The specific API endpoint that the collector should reach out to. + This endpoint will determine which data is retrieved by the collector. +4. Authentication Type + The authentication method required by the server must be specified. The supported authentication types include: + - Basic Authentication (username and password) + - Token Based Authentication (Token key) + - Bearer Token (API key) + - Raw Token (for custom token-based authentication) + - No Authorization (for publicly accessible data) +5. HTTP Method + Specify the HTTP method the collector should use to reach the API endpoint. The supported methods are: + - GET (to retrieve information) + - POST (if the endpoint requires sending specific parameters to retrieve data) + +# Additional Information +Once the collector is configured, it will begin to collect data periodically as per your configuration. +The collected data will be stored in a dataset defined by the Vendor and Product values provided. +You can use this data to create alerts, run queries, and generate reports within Cortex. + +## Disclaimer +Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve. diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_image.png b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_image.png new file mode 100644 index 000000000000..4c08a9054a14 Binary files /dev/null and b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_image.png differ diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_test.py b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_test.py new file mode 100644 index 000000000000..d31ee4b07f0e --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_test.py @@ -0,0 +1,411 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest + +from CommonServerPython import DemistoException +from GenericAPIEventCollector import ( + datetime_to_timestamp_format, timestamp_format_to_datetime, recursive_replace, + get_time_field_from_event_to_dt, is_pagination_needed, iso8601_to_datetime_str, + parse_json_param, generate_authentication_headers, + extract_pagination_params, PaginationLogic, TimestampFieldConfig, organize_events_to_xsiam_format, setup_search_events, + RequestData, generate_headers +) + + +def test_datetime_to_timestamp_format(): + dt = datetime(2023, 5, 1, 12, 0, 0) + assert datetime_to_timestamp_format(dt, '%Y-%m-%dT%H:%M:%SZ') == '2023-05-01T12:00:00Z' + assert datetime_to_timestamp_format(dt, 'epoch') == str(dt.timestamp()) + + +def test_timestamp_format_to_datetime(): + dt_str = '2023-05-01T12:00:00Z' + assert timestamp_format_to_datetime(dt_str, '%Y-%m-%dT%H:%M:%SZ') == datetime(2023, 5, 1, 12, 0, 0) + epoch_str = str(datetime(2023, 5, 1, 12, 0, 0).timestamp()) + assert timestamp_format_to_datetime(epoch_str, 'epoch') == datetime(2023, 5, 1, 12, 0, 0) + + +def test_recursive_replace(): + org_dict = {'key1': 'value1', 'key2': {'key3': 'value3'}} + substitutions = [('value1', 'new_value1'), ('value3', 'new_value3')] + result = recursive_replace(org_dict, substitutions) + assert result == {'key1': 'new_value1', 'key2': {'key3': 'new_value3'}} + + +def test_recursive_replace_with_none(): + assert recursive_replace(None, []) is None + + +def test_get_time_field_from_event_to_dt(): + event = {'timestamp': '2023-05-01T12:00:00Z'} + config = TimestampFieldConfig(['timestamp'], '%Y-%m-%dT%H:%M:%SZ') + result = get_time_field_from_event_to_dt(event, config) + assert isinstance(result, datetime) + assert result == datetime(2023, 5, 1, 12, 0, 0) + + +def test_get_time_field_from_event_to_dt_throws_exception(): + event = {'non_timestamp_field': '2023-05-01T12:00:00Z'} + config = TimestampFieldConfig(['timestamp'], '%Y-%m-%dT%H:%M:%SZ') + with pytest.raises(DemistoException) as exc_info: + get_time_field_from_event_to_dt(event, config) + assert "Timestamp field: ['timestamp'] not found in event" in str(exc_info.value) + + +def test_is_pagination_needed(): + events = {'next_page': 'page2', 'has_more': True} + pagination_logic = PaginationLogic(True, ['next_page'], ['has_more']) + needed, next_page = is_pagination_needed(events, pagination_logic) + assert needed is True + assert next_page == 'page2' + + +def test_is_pagination_not_needed(): + events = {'has_more': False} + pagination_logic = PaginationLogic(True, ['next_page'], ['has_more']) + needed, next_page = is_pagination_needed(events, pagination_logic) + assert needed is False + assert next_page is None + + +def test_iso8601_to_datetime_str(): + iso_time = '2023-05-01T12:00:00.1234567Z' + result = iso8601_to_datetime_str(iso_time) + assert result == '2023-05-01T12:00:00.123456Z' + + +def test_parse_json_param(): + json_param_value = '{"key": "value"}' + result = parse_json_param(json_param_value, 'test_param') + assert result == {"key": "value"} + + +def test_parse_json_param_none_value(): + result = parse_json_param(None, 'test_param') + assert result is None + + +def test_organize_events_to_xsiam_format(): + raw_events = { + "data": { + "events": [ + {"id": 1, "name": "event1"}, + {"id": 2, "name": "event2"} + ] + } + } + events_keys = ["data", "events"] + expected_output = [ + {"id": 1, "name": "event1"}, + {"id": 2, "name": "event2"} + ] + assert organize_events_to_xsiam_format(raw_events, events_keys) == expected_output + + +def test_organize_events_to_xsiam_format_empty(): + raw_events = {} + events_keys = ["data", "events"] + expected_output = [] + assert organize_events_to_xsiam_format(raw_events, events_keys) == expected_output + + +def test_organize_events_to_xsiam_format_no_events_key(): + raw_events = { + "data": { + "no_events": [ + {"id": 1, "name": "event1"}, + {"id": 2, "name": "event2"} + ] + } + } + events_keys = ["data", "events"] + expected_output = [] + assert organize_events_to_xsiam_format(raw_events, events_keys) == expected_output + + +def test_generate_authentication_headers_basic(): + params = { + "authentication": "Basic", + "credentials": {"identifier": "user", "password": "pass"} + } + headers = generate_authentication_headers(params) + assert headers["Authorization"].startswith("Basic ") + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_basic_no_password(mock_error, mock_return_error): + params = { + "authentication": "Basic", + "credentials": {"identifier": "user"} + } + generate_authentication_headers(params) + mock_error.assert_called_once_with("Password is required for Basic Authentication.") + mock_return_error.assert_called_once_with("Password is required for Basic Authentication.") + + +def test_generate_authentication_headers_bearer(): + params = { + "authentication": "Bearer", + "token": {"password": "test_token"} + } + headers = generate_authentication_headers(params) + assert headers["Authorization"] == "Bearer test_token" + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_bearer_no_token(mock_error, mock_return_error): + params = { + "authentication": "Bearer" + } + generate_authentication_headers(params) + mock_error.assert_called_once_with("API Token is required.") + mock_return_error.assert_called_once_with("API Token is required.") + + +def test_generate_authentication_headers_token(): + params = { + "authentication": "Token", + "token": {"password": "test_token"} + } + headers = generate_authentication_headers(params) + assert headers["Authorization"] == "Token test_token" + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_token_no_token(mock_error, mock_return_error): + params = { + "authentication": "Token" + } + generate_authentication_headers(params) + mock_error.assert_called_once_with("API Token is required.") + mock_return_error.assert_called_once_with("API Token is required.") + + +def test_generate_authentication_headers_api_key(): + params = { + "authentication": "Api-Key", + "token": {"password": "test_token"} + } + headers = generate_authentication_headers(params) + assert headers["api-key"] == "test_token" + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_api_key_no_token(mock_error, mock_return_error): + params = { + "authentication": "Api-Key" + } + generate_authentication_headers(params) + mock_error.assert_called_once_with("API Token is required.") + mock_return_error.assert_called_once_with("API Token is required.") + + +def test_generate_authentication_headers_raw_token(): + params = { + "authentication": "RawToken", + "token": {"password": "test_token"} + } + headers = generate_authentication_headers(params) + assert headers["Authorization"] == "test_token" + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_raw_token_no_token(mock_error, mock_return_error): + params = { + "authentication": "RawToken" + } + generate_authentication_headers(params) + mock_error.assert_called_once_with("API Token is required.") + mock_return_error.assert_called_once_with("API Token is required.") + + +def test_generate_authentication_headers_no_auth(): + params = { + "authentication": "No Authorization" + } + headers = generate_authentication_headers(params) + assert headers == {} + + +@patch('GenericAPIEventCollector.return_error') +@patch('GenericAPIEventCollector.demisto.error') +def test_generate_authentication_headers_invalid_auth(mock_error, mock_return_error): + params = { + "authentication": "InvalidAuth" + } + generate_authentication_headers(params) + mock_error.assert_called_once_with( + "Please insert a valid authentication method, options are: Basic, Bearer, Token, Api-Key, RawToken" + "No Authorization, got: InvalidAuth" + ) + mock_return_error.assert_called_once_with( + "Please insert a valid authentication method, options are: Basic, Bearer, Token, Api-Key, RawToken" + "No Authorization, got: InvalidAuth" + ) + + +def test_extract_pagination_params(): + params = { + "pagination_needed": "true", + "pagination_field_name": "next_page", + "pagination_flag": "has_more" + } + pagination_logic = extract_pagination_params(params) + assert pagination_logic.pagination_needed is True + assert pagination_logic.pagination_field_name == ["next_page"] + assert pagination_logic.pagination_flag == ["has_more"] + + +def test_setup_search_events(): + first_fetch_datetime = datetime(2023, 1, 1, 0, 0, 0) + last_run = {} + params = { + 'request_data': '{"key": "value"}', + 'request_json': '{"json_key": "json_value"}', + 'query_params': '{"param_key": "param_value"}', + 'pagination_needed': 'true', + 'pagination_field_name': 'next_page', + 'pagination_flag': 'has_more', + 'timestamp_field_name': 'timestamp', + 'timestamp_format': '%Y-%m-%dT%H:%M:%SZ' + } + timestamp_field_config = TimestampFieldConfig(['timestamp'], '%Y-%m-%dT%H:%M:%SZ') + + last_fetched_datetime, pagination_logic, request_data = setup_search_events( + first_fetch_datetime, last_run, params, timestamp_field_config + ) + + assert last_fetched_datetime == first_fetch_datetime + assert pagination_logic == PaginationLogic(True, ['next_page'], ['has_more']) + assert request_data == RequestData({'key': 'value'}, {'json_key': 'json_value'}, {'param_key': 'param_value'}) + + +def test_setup_search_events_with_last_run(): + first_fetch_datetime = datetime(2023, 1, 1, 0, 0, 0) + last_run = {'@last_fetched_datetime': '2023-01-02T00:00:00'} + params = { + 'request_data': '{"key": "value"}', + 'request_json': '{"json_key": "json_value"}', + 'query_params': '{"param_key": "param_value"}', + 'pagination_needed': 'true', + 'pagination_field_name': 'next_page', + 'pagination_flag': 'has_more', + 'timestamp_field_name': 'timestamp', + 'timestamp_format': '%Y-%m-%dT%H:%M:%SZ' + } + timestamp_field_config = TimestampFieldConfig(['timestamp'], '%Y-%m-%dT%H:%M:%SZ') + + last_fetched_datetime, pagination_logic, request_data = setup_search_events( + first_fetch_datetime, last_run, params, timestamp_field_config + ) + + assert last_fetched_datetime == datetime(2023, 1, 2, 0, 0, 0) + assert pagination_logic == PaginationLogic(True, ['next_page'], ['has_more']) + assert request_data == RequestData({'key': 'value'}, {'json_key': 'json_value'}, {'param_key': 'param_value'}) + + +def test_setup_search_events_first_fetch(): + first_fetch_datetime = datetime(2023, 1, 1, 0, 0, 0) + last_run = {} + params = { + 'request_data': '{"key": "value"}', + 'request_json': '{"json_key": "json_value"}', + 'query_params': '{"param_key": "param_value"}', + 'pagination_needed': 'true', + 'pagination_field_name': 'next_page', + 'pagination_flag': 'has_more', + 'timestamp_field_name': 'timestamp', + 'timestamp_format': '%Y-%m-%dT%H:%M:%SZ', + 'initial_query_params': '{"initial_param_key": "initial_param_value"}', + 'initial_pagination_params': {"pagination_needed": "true", "pagination_field_name": "next_page", + "pagination_flag": "has_more"}, + 'initial_request_data': '{"initial_key": "initial_value"}', + 'initial_request_json': '{"initial_json_key": "initial_json_value"}' + } + timestamp_field_config = TimestampFieldConfig(['timestamp'], '%Y-%m-%dT%H:%M:%SZ') + + last_fetched_datetime, pagination_logic, request_data = setup_search_events( + first_fetch_datetime, last_run, params, timestamp_field_config + ) + + assert last_fetched_datetime == first_fetch_datetime + assert pagination_logic == PaginationLogic(True, ['next_page'], ['has_more']) + assert request_data == RequestData({'initial_key': 'initial_value'}, {'initial_json_key': 'initial_json_value'}, + {'initial_param_key': 'initial_param_value'}) + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_basic(mock_generate_authentication_headers): + params = { + 'authentication': 'Basic', + 'credentials': {'identifier': 'user', 'password': 'pass'}, + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {'Authorization': 'Basic d5Nl4jp3YX2z'} + headers = generate_headers(params) + assert headers == {'Authorization': 'Basic d5Nl4jp3YX2z', 'Custom-Header': 'CustomValue'} + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_bearer(mock_generate_authentication_headers): + params = { + 'authentication': 'Bearer', + 'token': {'password': 'test_token'}, + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {'Authorization': 'Bearer test_token'} + headers = generate_headers(params) + assert headers == {'Authorization': 'Bearer test_token', 'Custom-Header': 'CustomValue'} + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_token(mock_generate_authentication_headers): + params = { + 'authentication': 'Token', + 'token': {'password': 'test_token'}, + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {'Authorization': 'Token test_token'} + headers = generate_headers(params) + assert headers == {'Authorization': 'Token test_token', 'Custom-Header': 'CustomValue'} + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_api_key(mock_generate_authentication_headers): + params = { + 'authentication': 'Api-Key', + 'token': {'password': 'test_token'}, + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {'api-key': 'test_token'} + headers = generate_headers(params) + assert headers == {'api-key': 'test_token', 'Custom-Header': 'CustomValue'} + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_raw_token(mock_generate_authentication_headers): + params = { + 'authentication': 'RawToken', + 'token': {'password': 'test_token'}, + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {'Authorization': 'test_token'} + headers = generate_headers(params) + assert headers == {'Authorization': 'test_token', 'Custom-Header': 'CustomValue'} + + +@patch('GenericAPIEventCollector.generate_authentication_headers') +def test_generate_headers_no_auth(mock_generate_authentication_headers): + params = { + 'authentication': 'No Authorization', + 'add_fields_to_header': '{"Custom-Header": "CustomValue"}' + } + mock_generate_authentication_headers.return_value = {} + headers = generate_headers(params) + assert headers == {'Custom-Header': 'CustomValue'} diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/README.md b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/README.md new file mode 100644 index 000000000000..99924caf46c3 --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/README.md @@ -0,0 +1,332 @@ +The **Generic API Event Collector** allows you to ingest data from any API endpoint into Cortex. +By configuring this collector, you can gather data from various systems and bring it into the Cortex ecosystem for better analysis and correlation. + +Note: This pack is currently in **Beta**, and as such, it may be subject to future changes and may not work on all types of APIs and Authentication. + +This is the default integration for this content pack when configured by the Data Onboarder in Cortex XSIAM. + +## Configure Generic API Event Collector (Beta) in Cortex + + +| **Parameter** | **Description** | **Required** | +|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| Server URL | | True | +| Endpoint | Add the endpoint you want to collect data from \(Alert/Events etc.\). | True | +| Authentication Type | Select the authentication method. | True | +| HTTP Method | The HTTP method of the request to the API. | True | +| API Token | API Key to access the service REST API. | False | +| Username | Username & Password to use for basic authentication. | False | +| Password | | False | +| Add Fields To header | If the product authentication requires more fields to add to the header add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Use system proxy settings | | False | +| Trust any certificate (not secure) | | False | +| Vendor | Enter vendor name for dataset. | True | +| Product | Enter product name for dataset. | True | +| Request data | If the product authentication requires more fields to add to the DATA add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Initial request data | If the product requires a different initial DATA, add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Request JSON parameters | If the product authentication requires more fields to add to the body as JSON add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Initial request JSON parameters | If the product requires a different initial request JSON, add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Query parameters | If the product authentication allows to filter the results using query Parameters add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Initial query parameters | If the product requires a different initial query parameters for the first fetch call, add it here in dictionary format \{unique_field : 286\}. If there's a need to add more then one, use it in this format: \{'field-1': value_example, 'field-2': value_2, 'field-3': value_3\} | False | +| Is pagination needed | If the API JSON response supports events pagination. | | +| Pagination field name | Next page field in JSON response, e.g., "cursor", "next_page" | False | +| Pagination flag | Next page existence in JSON response e.g., "has_more", "next" | False | +| Timestamp format of the event creation time or "epoch". | Python compatible datetime formatting \(e.g. ,"%Y-%m-%dT%H:%M:%S.%fZ" or "%Y.%m.%d %H:%M:%S"\) or "epoch" to use UNIX epoch time. | False | +| Timestamp field | The name of the event creation time in the response data, e.g., "timestamp" or "created_at". | True | +| Events lookup path in the response JSON, dot-separated, e.g., "data.items". | Where within the response object to find the events list. | False | +| Event ID lookup path in the event response JSON, dot-separated, e.g., "id". | Where within the event object to find the event ID. | False | +| The type of ID field, either "integer" or "string" | ID field of type integer are comparable and when last fetched ID is the maximum ID between the fetched events, when the type is string, the last fetched ID is the last event returned from the API. | False | +| OK codes | Allowed HTTP status codes for successful response from the API | False | +| Limit | Number of incidents to fetch per fetch. | False | +| Fetch Events | | False | +| Events Fetch Interval | | False | + +## How to configure the event collector +--- + +### Authentication +You must specify the authentication method required by the server. +The supported authentication types include: +- Basic authentication (username and password) +- Token-based authentication +- Bearer token +- Api-Key token +- Raw Token (for custom token-based authentication) +- No Authorization (for publicly accessible data) + +### Pagination + +When the API supports pagination in the response, the collector can fetch more pages of data using the following parameters: +- Is pagination needed If the API JSON response supports events pagination. +- Pagination field name, Next page field in JSON response, e.g., "cursor" or "next_page" | False | +- Pagination flag, The Next page existence in JSON response e.g., "has_more" or "next" + +In the below example the pagination flag is `pagination.has_more` +The pagination field name is `pagination.next_page` +```json +{ + "data": [ + { + "id": 1, + "name": "John Doe", + "occupation": "Software Engineer" + }, + { + "id": 2, + "name": "Jane Smith", + "occupation": "Data Scientist" + } + ], + "pagination": { + "current_page": 1, + "next_page": "https://api.example.com/users?page=2", + "has_more": true + } +} +``` + +### Request Data (And initial request data) +If the product authentication requires more fields to add to the `DATA`. +Add it here in dictionary format. + +For example: +```json +{"field-1": "value_example", "field-2": 1, "field-3": "value_3"} +``` +Note: Using the initial request data parameter will only be used in the first request to collect events. +### Request JSON (And initial request JSON) +If the product authentication requires more fields to add to the body as JSON, add it +here in dictionary format. + +For example: +```json +{"date": "2021-08-01", "field-2": 1, "field-3": "value_3"} +``` + +Note: Using the initial request JSON parameter will only be used in the first request to collect events. + +### Query parameters (And Initial Query parameters) +If the product authentication allows filtering the results using query parameters, add it here in dictionary format: +```json +{"ordering": "id", "limit": 1, "created_after": "@first_fetch_datetime"} +``` + +Note: Using the initial query parameters parameter will only be used in the first request to collect events. + + +### Timestamp field +The name of the event creation time in the response data, e.g., "timestamp" or "created_at". +In the following API response: +```json +{ + "data": [ + { + "id": 3, + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "id": 4, + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ], + "pagination": { + "current_page": 2, + "next_page": "https://api.example.com/users?page=3", + "has_more": true + } +} +``` +the timestamp field is `created` + +### Timestamp format +The timestamp format of the event creation time or "epoch" to use UNIX epoch time. +The formatting supported is Python-compatible datetime formatting (e.g., "%Y-%m-%dT%H:%M:%S.%fZ" or "%Y.%m.%d %H:%M:%S"). +In the following API response: +```json +{ + "data": [ + { + "id": 3, + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "id": 4, + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ], + "pagination": { + "current_page": 2, + "next_page": "https://api.example.com/users?page=3", + "has_more": true + } +} +``` +The timestamp format is python format "%Y-%m-%dT%H:%M:%S.%fZ" + +Note: To learn more about Python date and time formats, see: https://docs.python.org/3/library/datetime.html#format-codes + +### Events +Where within the response JSON to search for the events, dot-separated (e.g., "data.items"). + +Example 1: +```json +{ + "data": [ + { + "id": 4, + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "id": 3, + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ] +} +``` +The events are within the "data" in the response. + +Example 2: +```json +{ + "data": { + "items": [ + { + "id": 4, + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "id": 3, + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ] + } +} +``` +The events are within the "data.items" in the response. + +### Event ID & Type +Event ID lookup path in the event response JSON, dot-separated, e.g., "id" +Where within the event object to find the event ID. + +The type of ID field, either "integer" or "string": +- ID field of type integer is comparable, and when last fetched ID is the maximum ID between the fetched events. +- ID field of the type is string, the last fetched ID is the last event returned from the API. + +Example 1: +```json +{ + "data": [ + { + "id": 4, + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "id": 3, + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ], + "pagination": { + "current_page": 2, + "next_page": "https://api.example.com/users?page=3", + "has_more": true + } +} +``` +The event ID field should be "id" and the type should be integer, and the last fetched ID will be 4. + +Example 2: +```json +{ + "data": [ + { + "uuid": "123e4567-e89b-12d3-a456-426614174000", + "name": "Alice Brown", + "occupation": "Network Engineer", + "created": "2021-10-05T19:45:20.789012Z" + }, + { + "uuid": "123e4567-e89b-12d3-a456-426614174001", + "name": "Dave Wilson", + "occupation": "Cybersecurity Analyst", + "created": "2021-10-06T10:15:45.654321Z" + } + ], + "pagination": { + "current_page": 2, + "next_page": "https://api.example.com/users?page=3", + "has_more": true + } +} +``` +The event ID field should be "uuid" and the type should be string, and the last fetched ID will be "123e4567-e89b-12d3-a456-426614174001". + +## Substitutions in API requests calls +To make the API calls more dynamic against the API endpoint, we added a few placeholders that will be substituted before calling the API endpoint. +- `@last_fetched_id` - The last ID that was fetched from the API, if this is the first fetch, the value will be empty. +- `@last_fetched_datetime` - The last fetched event time from the API, if this is the first fetch, the value will be empty. +- `@first_fetch_datetime` - The first fetch time, when the integration first started to fetch events. +- `@fetch_size_limit` - The number of incidents to fetch per fetch. + +Examples being used in query parameters: +- This will substitute the `@last_fetched_id` with the last fetched ID from a previous fetch call. +```json +{"ordering": "id", "limit": 100, "id__gt": "@last_fetched_id"} +``` +The resulting API query parameters will be: +```json +{"ordering": "id", "limit": 100, "id__gt": "4"} +``` + +- This will substitute the `@first_fetch_datetime` with the first fetch time. +```json +{"ordering": "id", "limit": 1, "created_after": "@first_fetch_datetime"} +``` +The resulting API query parameters will be: +```json +{"ordering": "id", "limit": 1, "created_after": "2021-10-06T10:15:45.654321Z"} +``` + +## Commands + +You can execute these commands from the CLI, as part of an automation, or in a playbook. +After you successfully execute a command, a DBot message appears in the War Room with the command details. + +### generic-api-event-collector-get-events + +*** +Gets events from 3rd-party vendor. + +#### Base Command + +`generic-api-event-collector-get-events` + +#### Input + +| **Argument Name** | **Description** | **Required** | +|--------------------|-----------------------------------------------------------------------------------------------------------------------------------|--------------| +| should_push_events | If true, the command will create events, otherwise it will only display them. Possible values are: true, false. Default is false. | Required | +| limit | Maximum number of results to return. | Optional | + +#### Context Output + +There is no context output for this command. diff --git a/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/command_examples.txt b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/command_examples.txt new file mode 100644 index 000000000000..dcf675ab809a --- /dev/null +++ b/Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/command_examples.txt @@ -0,0 +1 @@ +!generic-api-event-collector-get-events limit=1 should_push_events=false diff --git a/Packs/GenericAPIEventCollector/README.md b/Packs/GenericAPIEventCollector/README.md new file mode 100644 index 000000000000..a424d6a50258 --- /dev/null +++ b/Packs/GenericAPIEventCollector/README.md @@ -0,0 +1,15 @@ + +# Generic API Event Collector +## Overview +The **Generic API Event Collector** allows you to ingest data from any API endpoint into Cortex. +By configuring this collector, you can gather data from various systems and bring it into the Cortex ecosystem for better analysis and correlation. +**Note:** This pack is currently in **Beta**, and as such, it may be subject to future changes and may not work on all types of APIs and Authentication. + +## What Does This Pack Do? +This pack provides an integration that enables you to: +- Collect events automatically from various API sources using the **Generic API Event Collector**. +- Manually fetch events using the `generic-api-event-collector-get-events` command. + +## Use cases +- Ingest logs and event data from third-party systems that expose an API. +- Enhance threat detection and correlation by bringing external events into Cortex. diff --git a/Packs/GenericAPIEventCollector/pack_metadata.json b/Packs/GenericAPIEventCollector/pack_metadata.json new file mode 100644 index 000000000000..393bde4d7d21 --- /dev/null +++ b/Packs/GenericAPIEventCollector/pack_metadata.json @@ -0,0 +1,23 @@ +{ + "name": "GenericAPIEventCollector", + "description": "This pack provides a generic API event collector integration that can be used to collect events from various sources.", + "support": "xsoar", + "serverMinVersion": "6.0.0", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Analytics & SIEM" + ], + "tags": [], + "created": "2024-11-28T01:23:00Z", + "useCases": [], + "keywords": [], + "price": 0, + "dependencies": {}, + "marketplaces": [ + "marketplacev2" + ], + "defaultDataSource": "GenericAPIEventCollector" +} \ No newline at end of file