From 576e1e7f73d20c8f4c164b1aed1ba92cfcfd7dc2 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 3 Jun 2021 20:13:18 +0000 Subject: [PATCH 01/31] Adding whitelist index --- assemblyline_ui/helper/search.py | 2 ++ test/test_search.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/assemblyline_ui/helper/search.py b/assemblyline_ui/helper/search.py index 8e31b377..e4b240ca 100644 --- a/assemblyline_ui/helper/search.py +++ b/assemblyline_ui/helper/search.py @@ -7,6 +7,7 @@ 'result': STORAGE.result, 'signature': STORAGE.signature, 'submission': STORAGE.submission, + 'whitelist': STORAGE.whitelist, 'workflow': STORAGE.workflow } @@ -17,6 +18,7 @@ 'result': "created desc", 'signature': "type asc", 'submission': "times.submitted desc", + 'whitelist': "date desc", 'workflow': "last_seen desc" } diff --git a/test/test_search.py b/test/test_search.py index 5ec9f295..c47cf758 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -9,12 +9,13 @@ from assemblyline.odm.models.file import File from assemblyline.odm.models.result import Result from assemblyline.odm.models.submission import Submission +from assemblyline.odm.models.whitelist import Whitelist from assemblyline.odm.models.workflow import Workflow from assemblyline.odm.randomizer import random_model_obj from assemblyline.odm.random_data import create_users, wipe_users, create_signatures TEST_SIZE = 10 -collections = ['alert', 'file', 'heuristic', 'result', 'signature', 'submission', 'workflow'] +collections = ['alert', 'file', 'heuristic', 'result', 'signature', 'submission', 'whitelist', 'workflow'] file_list = [] signatures = [] @@ -59,6 +60,12 @@ def datastore(datastore_connection): ds.heuristic.save(h.heur_id, h) ds.heuristic.commit() + for _ in range(TEST_SIZE): + w_id = get_random_id() + w = random_model_obj(Whitelist) + ds.whitelist.save(w_id, w) + ds.whitelist.commit() + for _ in range(TEST_SIZE): w_id = get_random_id() w = random_model_obj(Workflow) @@ -73,6 +80,7 @@ def datastore(datastore_connection): ds.signature.wipe() ds.submission.wipe() ds.heuristic.wipe() + ds.whitelist.wipe() ds.workflow.wipe() wipe_users(ds) @@ -129,6 +137,7 @@ def test_histogram_search(datastore, login_session): 'heuristic': False, 'signature': 'last_modified', 'submission': 'times.submitted', + 'whitelist': 'date', 'workflow': 'last_edit' } @@ -148,6 +157,7 @@ def test_histogram_search(datastore, login_session): 'signature': 'order', 'submission': 'file_count', 'heuristic': False, + 'whitelist': False, 'workflow': 'hit_count' } @@ -191,6 +201,7 @@ def test_stats_search(datastore, login_session): 'signature': 'order', 'submission': 'file_count', 'heuristic': False, + 'whitelist': False, 'workflow': 'hit_count' } From 688c563771e9023535d4aa709d7311cbd1e887f4 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 3 Jun 2021 21:27:32 +0000 Subject: [PATCH 02/31] Added whitelist API --- assemblyline_ui/api/v4/whitelist.py | 35 +++++++++++++++++++++++++++++ assemblyline_ui/app.py | 2 ++ test/test_whitelist.py | 31 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 assemblyline_ui/api/v4/whitelist.py create mode 100644 test/test_whitelist.py diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/whitelist.py new file mode 100644 index 00000000..2dd797a2 --- /dev/null +++ b/assemblyline_ui/api/v4/whitelist.py @@ -0,0 +1,35 @@ + +from assemblyline_ui.api.base import make_subapi_blueprint, make_api_response, api_login +from assemblyline_ui.config import STORAGE + +SUB_API = 'whitelist' +whitelist_api = make_subapi_blueprint(SUB_API, api_version=4) +whitelist_api._doc = "Perform operations on whitelisted hashes" + + +@whitelist_api.route("//", methods=["GET"]) +@api_login() +def exists(sha256, _): + """ + Check if a hash exists in the whitelist. + + Variables: + sha256 => Hash to check + + Arguments: + None + + Data Block: + None + + API call example: + GET /api/v1/whitelist/123456...654321/ + + Result example: + + """ + whitelist = STORAGE.whitelist.get_if_exists(sha256) + if whitelist: + return make_api_response(whitelist) + + return make_api_response({}, "The hash was not found in the whitelist.", 404) diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index 87b49067..546fb5b2 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -28,6 +28,7 @@ from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api +from assemblyline_ui.api.v4.whitelist import whitelist_api from assemblyline_ui.api.v4.workflow import workflow_api from assemblyline_ui.error import errors from assemblyline_ui.healthz import healthz @@ -77,6 +78,7 @@ app.register_blueprint(ui_api) app.register_blueprint(user_api) app.register_blueprint(webauthn_api) +app.register_blueprint(whitelist_api) app.register_blueprint(workflow_api) diff --git a/test/test_whitelist.py b/test/test_whitelist.py new file mode 100644 index 00000000..37ecb1bf --- /dev/null +++ b/test/test_whitelist.py @@ -0,0 +1,31 @@ + +import pytest +import random + +from conftest import get_api_data + +from assemblyline.odm.random_data import create_users, create_whitelists, wipe_users, wipe_whitelist + + +workflow_list = [] + + +@pytest.fixture(scope="module") +def datastore(datastore_connection): + try: + create_users(datastore_connection) + create_whitelists(datastore_connection) + yield datastore_connection + finally: + wipe_users(datastore_connection) + wipe_whitelist(datastore_connection) + + +# noinspection PyUnusedLocal +def test_check_whitelist(datastore, login_session): + _, session, host = login_session + + hash = random.choice(datastore.whitelist.search("id:*", fl='id', rows=100, as_obj=False)['items'])['id'] + + resp = get_api_data(session, f"{host}/api/v4/whitelist/{hash}/") + assert resp == datastore.whitelise.get(hash, as_obj=False) From 5b021ec3d0a07b7bdb7314ce92f9774ac6ddba39 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 13:14:04 +0000 Subject: [PATCH 03/31] Fix tests and API --- assemblyline_ui/api/v4/whitelist.py | 6 +++--- test/conftest.py | 20 +++++++++++--------- test/test_whitelist.py | 19 +++++++++++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/whitelist.py index 2dd797a2..5d9b1258 100644 --- a/assemblyline_ui/api/v4/whitelist.py +++ b/assemblyline_ui/api/v4/whitelist.py @@ -9,7 +9,7 @@ @whitelist_api.route("//", methods=["GET"]) @api_login() -def exists(sha256, _): +def exists(sha256, **_): """ Check if a hash exists in the whitelist. @@ -28,8 +28,8 @@ def exists(sha256, _): Result example: """ - whitelist = STORAGE.whitelist.get_if_exists(sha256) + whitelist = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) if whitelist: return make_api_response(whitelist) - return make_api_response({}, "The hash was not found in the whitelist.", 404) + return make_api_response(None, "The hash was not found in the whitelist.", 404) diff --git a/test/conftest.py b/test/conftest.py index 1975f98f..618661d1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -93,15 +93,17 @@ def host(redis_connection): for the failed address to become available. """ errors = {} - for host in POSSIBLE_HOSTS: - try: - result = requests.get(f"{host}/api/v4/auth/login", verify=False) - if result.status_code == 200: - return host - result.raise_for_status() - errors[host] = str(result.status_code) - except requests.RequestException as err: - errors[host] = str(err) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for host in POSSIBLE_HOSTS: + try: + result = requests.get(f"{host}/api/v4/auth/login", verify=False) + if result.status_code == 200: + return host + result.raise_for_status() + errors[host] = str(result.status_code) + except requests.RequestException as err: + errors[host] = str(err) pytest.skip("Couldn't find the API server, can't test against it.\n" + '\n'.join(k + ' ' + v for k, v in errors.items())) diff --git a/test/test_whitelist.py b/test/test_whitelist.py index 37ecb1bf..3316f8ed 100644 --- a/test/test_whitelist.py +++ b/test/test_whitelist.py @@ -2,14 +2,11 @@ import pytest import random -from conftest import get_api_data +from conftest import get_api_data, APIError from assemblyline.odm.random_data import create_users, create_whitelists, wipe_users, wipe_whitelist -workflow_list = [] - - @pytest.fixture(scope="module") def datastore(datastore_connection): try: @@ -22,10 +19,20 @@ def datastore(datastore_connection): # noinspection PyUnusedLocal -def test_check_whitelist(datastore, login_session): +def test_whitelist_exist(datastore, login_session): _, session, host = login_session hash = random.choice(datastore.whitelist.search("id:*", fl='id', rows=100, as_obj=False)['items'])['id'] resp = get_api_data(session, f"{host}/api/v4/whitelist/{hash}/") - assert resp == datastore.whitelise.get(hash, as_obj=False) + assert resp == datastore.whitelist.get(hash, as_obj=False) + + +# noinspection PyUnusedLocal +def test_whitelist_missing(datastore, login_session): + _, session, host = login_session + + hash = "DOES NOT EXISTS" + + with pytest.raises(APIError): + get_api_data(session, f"{host}/api/v4/whitelist/{hash}/") From 7abb1dbf74ae5d8830dc2bcef0195f870cc086fc Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 15:20:27 +0000 Subject: [PATCH 04/31] Fix tests and search --- assemblyline_ui/helper/search.py | 2 +- test/conftest.py | 4 ++-- test/test_search.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assemblyline_ui/helper/search.py b/assemblyline_ui/helper/search.py index e4b240ca..0d337334 100644 --- a/assemblyline_ui/helper/search.py +++ b/assemblyline_ui/helper/search.py @@ -18,7 +18,7 @@ 'result': "created desc", 'signature': "type asc", 'submission': "times.submitted desc", - 'whitelist': "date desc", + 'whitelist': "added desc", 'workflow': "last_seen desc" } diff --git a/test/conftest.py b/test/conftest.py index 618661d1..d13da81f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -97,8 +97,8 @@ def host(redis_connection): warnings.simplefilter('ignore') for host in POSSIBLE_HOSTS: try: - result = requests.get(f"{host}/api/v4/auth/login", verify=False) - if result.status_code == 200: + result = requests.get(f"{host}/api/v4/auth/login/", verify=False) + if result.status_code == 401: return host result.raise_for_status() errors[host] = str(result.status_code) diff --git a/test/test_search.py b/test/test_search.py index c47cf758..1231e78f 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -137,7 +137,7 @@ def test_histogram_search(datastore, login_session): 'heuristic': False, 'signature': 'last_modified', 'submission': 'times.submitted', - 'whitelist': 'date', + 'whitelist': 'added', 'workflow': 'last_edit' } From d1c737edb20f65e99bc833929c5b851c6f37f27c Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 19:02:58 +0000 Subject: [PATCH 05/31] Add add_or_update whitelist API --- assemblyline_ui/api/v4/whitelist.py | 128 +++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/whitelist.py index 5d9b1258..e7ffcd1c 100644 --- a/assemblyline_ui/api/v4/whitelist.py +++ b/assemblyline_ui/api/v4/whitelist.py @@ -1,12 +1,133 @@ -from assemblyline_ui.api.base import make_subapi_blueprint, make_api_response, api_login -from assemblyline_ui.config import STORAGE +from flask import request + +from assemblyline.common.isotime import now_as_iso +from assemblyline.remote.datatypes.lock import Lock +from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint +from assemblyline_ui.config import CLASSIFICATION, STORAGE SUB_API = 'whitelist' whitelist_api = make_subapi_blueprint(SUB_API, api_version=4) whitelist_api._doc = "Perform operations on whitelisted hashes" +@whitelist_api.route("//", methods=["PUT"]) +@api_login(require_type=['user', 'signature_importer']) +def add_or_update(sha256, **kwargs): + """ + Add a hash in the whitelist if it does not exist or update its list of sources if it does + + Variables: + sha256 => Hash to check + + Arguments: + None + + Data Block: + { + "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional + "fileinfo": { # Information about the file - Optional + "md5": "123...321", # MD5 hash of the file + "sha1": "1234...4321", # SHA1 hash of the file + "sha256": "12345....54321", # SHA256 of the file (default: sha256 variable) + "size": 12345, # Size of the file + "type": "document/text"}, # Type of the file + "sources": [ # List of sources for why the file is whitelisted, dedupped on name - Required + {"name": "NSRL", # Name of external source or user who whitelisted it - Required + "reason": [ # List of reasons why the source is whitelisted - Required + "Found as test.txt on default windows 10 CD", + "Found as install.txt on default windows XP CD" + ], + "type": "external"}, # Type or source (external or user) - Required + {"name": "admin", + "reason": ["We've seen this file many times and it leads to False positives"], + "type": "user"} + ] + } + + Result example: + { + "success": true, # Was the hash successfully added + "op": "add" # Was it added to the system or updated + } + """ + # Validate hash lenght + if len(sha256) != 64: + return make_api_response(None, "Invalid sha256 hash length", 400) + + # Load data + data = request.json + if not data: + return make_api_response({}, "No data provided", 400) + user = kwargs['user'] + + # Set defaults + data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) + data.setdefault('fileinfo', {}) + data['fileinfo']['sha256'] = sha256 + data['added'] = data['updated'] = now_as_iso() + + # Validate sources + src_map = {} + for src in data['sources']: + if src['type'] == 'user': + if src['name'] != user['uname']: + return make_api_response( + {}, f"You cannot add a source for another user. {src['name']} != {user['uname']}", 400) + else: + if 'signature_importer' not in user['type']: + return make_api_response( + {}, "You do not have sufficient priviledges to add an external source.", 403) + + src_map[src['name']] = src + + with Lock(f'add_or_update-whitelist-{sha256}', 30): + old = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) + if old: + try: + # Use old added date + data['added'] = old['added'] + + # Use minimal classification + data['classification'] = CLASSIFICATION.min_classification( + data['classification'], old['classification']) + + # Merge file info (keep new values) + for k, v in old['fileinfo'].items(): + if k not in data['fileinfo']: + data['fileinfo'][k] = v + + # Merge sources together + old_src_map = {x['name']: x for x in old['sources']} + for name, src in src_map.items(): + if name not in old_src_map: + old_src_map[name] = src + else: + old_src = old_src_map[name] + if old_src['type'] != src['type']: + return make_api_response( + {}, f"Source type conflict for {name}: {old_src['type']} != {src['type']}", 400) + + for reason in src['reason']: + if reason not in old_src['reason']: + old_src['reason'].append(reason) + + data['sources'] = old_src_map.values() + + # Save data to the DB + STORAGE.whitelist.save(sha256, data) + return make_api_response({'success': True, "op": "update"}) + except Exception as e: + return make_api_response({}, f"Invalid data provided: {str(e)}", 400) + else: + try: + data['sources'] = src_map.values() + STORAGE.whitelist.save(sha256, data) + return make_api_response({'success': True, "op": "add"}) + except Exception as e: + return make_api_response({}, f"Invalid data provided: {str(e)}", 400) + + @whitelist_api.route("//", methods=["GET"]) @api_login() def exists(sha256, **_): @@ -28,6 +149,9 @@ def exists(sha256, **_): Result example: """ + if len(sha256) != 64: + return make_api_response(None, "Invalid sha256 hash length", 400) + whitelist = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) if whitelist: return make_api_response(whitelist) From eb49f065b55177c16268b59449ff91eac6c2a4a5 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 19:03:12 +0000 Subject: [PATCH 06/31] Fix tests and add more --- test/test_search.py | 4 +- test/test_whitelist.py | 173 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 7 deletions(-) diff --git a/test/test_search.py b/test/test_search.py index 1231e78f..71616866 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -11,7 +11,7 @@ from assemblyline.odm.models.submission import Submission from assemblyline.odm.models.whitelist import Whitelist from assemblyline.odm.models.workflow import Workflow -from assemblyline.odm.randomizer import random_model_obj +from assemblyline.odm.randomizer import get_random_hash, random_model_obj from assemblyline.odm.random_data import create_users, wipe_users, create_signatures TEST_SIZE = 10 @@ -61,7 +61,7 @@ def datastore(datastore_connection): ds.heuristic.commit() for _ in range(TEST_SIZE): - w_id = get_random_id() + w_id = "0"+get_random_hash(63) w = random_model_obj(Whitelist) ds.whitelist.save(w_id, w) ds.whitelist.commit() diff --git a/test/test_whitelist.py b/test/test_whitelist.py index 3316f8ed..bdbc5941 100644 --- a/test/test_whitelist.py +++ b/test/test_whitelist.py @@ -1,10 +1,49 @@ -import pytest +import json import random -from conftest import get_api_data, APIError +import pytest +from assemblyline.common.forge import get_classification +from assemblyline.common.isotime import iso_to_epoch from assemblyline.odm.random_data import create_users, create_whitelists, wipe_users, wipe_whitelist +from assemblyline.odm.randomizer import get_random_hash +from conftest import APIError, get_api_data + +add_hash = "10" + get_random_hash(62) +add_error_hash = "11" + get_random_hash(62) +update_hash = "12" + get_random_hash(62) +update_conflict_hash = "13" + get_random_hash(62) + +NSRL_SOURCE = { + "name": "NSRL", + "reason": [ + "Found as test.txt on default windows 10 CD", + "Found as install.txt on default windows XP CD" + ], + "type": "external"} + +NSRL2_SOURCE = { + "name": "NSRL2", + "reason": [ + "File contains only AAAAs..." + ], + "type": "external"} + +ADMIN_SOURCE = { + "name": "admin", + "reason": [ + "Generates a lot of FPs", + ], + "type": "user"} + +USER_SOURCE = { + "name": "user", + "reason": [ + "I just feel like it!", + "I just feel like it!", + ], + "type": "user"} @pytest.fixture(scope="module") @@ -19,6 +58,119 @@ def datastore(datastore_connection): # noinspection PyUnusedLocal +def test_whitelist_add(datastore, login_session): + _, session, host = login_session + + # Generate a random whitelist + wl_data = { + 'fileinfo': {'md5': get_random_hash(32), + 'sha1': get_random_hash(40), + 'sha256': add_hash, + 'size': random.randint(128, 4096), + 'type': 'document/text'}, + 'sources': [NSRL_SOURCE, ADMIN_SOURCE], + } + + # Insert it and test return value + resp = get_api_data(session, f"{host}/api/v4/whitelist/{add_hash}/", method="PUT", data=json.dumps(wl_data)) + assert resp['success'] + assert resp['op'] == 'add' + + # Load inserted data from DB + ds_wl = datastore.whitelist.get(add_hash, as_obj=False) + + # Test dates + added = ds_wl.pop('added', None) + updated = ds_wl.pop('updated', None) + assert added == updated + assert added is not None and updated is not None + + # Test classification + classification = ds_wl.pop('classification', None) + assert classification is not None + + # Test rest + assert ds_wl == wl_data + + +def test_whitelist_add_invalid(datastore, login_session): + _, session, host = login_session + + # Generate a random whitelist + wl_data = {'sources': [USER_SOURCE]} + + # Insert it and test return value + with pytest.raises(APIError) as conflict_exc: + get_api_data(session, f"{host}/api/v4/whitelist/{add_error_hash}/", method="PUT", data=json.dumps(wl_data)) + + assert 'for another user' in conflict_exc.value.args[0] + + +def test_whitelist_update(datastore, login_session): + _, session, host = login_session + cl_eng = get_classification() + + # Generate a random whitelist + wl_data = { + 'classification': cl_eng.RESTRICTED, + 'fileinfo': {'md5': get_random_hash(32), + 'sha1': get_random_hash(40), + 'sha256': update_hash, + 'size': random.randint(128, 4096), + 'type': 'document/text'}, + 'sources': [NSRL_SOURCE], + } + + # Insert it and test return value + resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_hash}/", method="PUT", data=json.dumps(wl_data)) + assert resp['success'] + assert resp['op'] == 'add' + + # Load inserted data from DB + ds_wl = datastore.whitelist.get(update_hash, as_obj=False) + + # Test rest + assert {k: v for k, v in ds_wl.items() if k not in ['added', 'updated']} == wl_data + + u_data = {'sources': [NSRL2_SOURCE]} + + # Insert it and test return value + resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_hash}/", method="PUT", data=json.dumps(u_data)) + assert resp['success'] + assert resp['op'] == 'update' + + # Load inserted data from DB + ds_u = datastore.whitelist.get(update_hash, as_obj=False) + + assert ds_u['added'] == ds_wl['added'] + assert iso_to_epoch(ds_u['updated']) > iso_to_epoch(ds_wl['updated']) + assert ds_u['classification'] == cl_eng.UNRESTRICTED + assert len(ds_u['sources']) == 2 + assert NSRL2_SOURCE in ds_u['sources'] + assert NSRL_SOURCE in ds_u['sources'] + + +def test_whitelist_update_conflict(datastore, login_session): + _, session, host = login_session + + # Generate a random whitelist + wl_data = {'sources': [ADMIN_SOURCE]} + + # Insert it and test return value + resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_conflict_hash}/", + method="PUT", data=json.dumps(wl_data)) + assert resp['success'] + assert resp['op'] == 'add' + + # Insert the same source with a different type + wl_data['sources'][0]['type'] = 'external' + with pytest.raises(APIError) as conflict_exc: + get_api_data(session, f"{host}/api/v4/whitelist/{update_conflict_hash}/", + method="PUT", data=json.dumps(wl_data)) + + assert 'Source type conflict' in conflict_exc.value.args[0] + + def test_whitelist_exist(datastore, login_session): _, session, host = login_session @@ -28,11 +180,22 @@ def test_whitelist_exist(datastore, login_session): assert resp == datastore.whitelist.get(hash, as_obj=False) +# noinspection PyUnusedLocal +def test_whitelist_invalid(datastore, login_session): + _, session, host = login_session + + with pytest.raises(APIError) as invalid_exc: + get_api_data(session, f"{host}/api/v4/whitelist/{get_random_hash(32)}/") + + assert 'hash length' in invalid_exc.value.args[0] + + # noinspection PyUnusedLocal def test_whitelist_missing(datastore, login_session): _, session, host = login_session - hash = "DOES NOT EXISTS" + missing_hash = "f" + get_random_hash(63) + with pytest.raises(APIError) as missing_exc: + get_api_data(session, f"{host}/api/v4/whitelist/{missing_hash}/") - with pytest.raises(APIError): - get_api_data(session, f"{host}/api/v4/whitelist/{hash}/") + assert 'not found' in missing_exc.value.args[0] From bdc0b8ab603c79bdf2ddcc096a9783c8f3fe66cb Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 19:07:19 +0000 Subject: [PATCH 07/31] Make sure the user can check the whitelist on hashes it has access --- assemblyline_ui/api/v4/whitelist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/whitelist.py index e7ffcd1c..69480dce 100644 --- a/assemblyline_ui/api/v4/whitelist.py +++ b/assemblyline_ui/api/v4/whitelist.py @@ -130,7 +130,7 @@ def add_or_update(sha256, **kwargs): @whitelist_api.route("//", methods=["GET"]) @api_login() -def exists(sha256, **_): +def exists(sha256, **kwargs): """ Check if a hash exists in the whitelist. @@ -153,7 +153,7 @@ def exists(sha256, **_): return make_api_response(None, "Invalid sha256 hash length", 400) whitelist = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) - if whitelist: + if whitelist and CLASSIFICATION.is_accessible(kwargs['user']['classification'], whitelist['classification']): return make_api_response(whitelist) return make_api_response(None, "The hash was not found in the whitelist.", 404) From f43a3302b79071a6d265fa42dbe2cd44d91dc3c1 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 4 Jun 2021 19:10:16 +0000 Subject: [PATCH 08/31] Use max classification because someone could put classified info in the reason --- assemblyline_ui/api/v4/whitelist.py | 2 +- test/test_whitelist.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/whitelist.py index 69480dce..f59e487d 100644 --- a/assemblyline_ui/api/v4/whitelist.py +++ b/assemblyline_ui/api/v4/whitelist.py @@ -89,7 +89,7 @@ def add_or_update(sha256, **kwargs): data['added'] = old['added'] # Use minimal classification - data['classification'] = CLASSIFICATION.min_classification( + data['classification'] = CLASSIFICATION.max_classification( data['classification'], old['classification']) # Merge file info (keep new values) diff --git a/test/test_whitelist.py b/test/test_whitelist.py index bdbc5941..ee1eaada 100644 --- a/test/test_whitelist.py +++ b/test/test_whitelist.py @@ -112,7 +112,6 @@ def test_whitelist_update(datastore, login_session): # Generate a random whitelist wl_data = { - 'classification': cl_eng.RESTRICTED, 'fileinfo': {'md5': get_random_hash(32), 'sha1': get_random_hash(40), 'sha256': update_hash, @@ -130,9 +129,12 @@ def test_whitelist_update(datastore, login_session): ds_wl = datastore.whitelist.get(update_hash, as_obj=False) # Test rest - assert {k: v for k, v in ds_wl.items() if k not in ['added', 'updated']} == wl_data + assert {k: v for k, v in ds_wl.items() if k not in ['added', 'updated', 'classification']} == wl_data - u_data = {'sources': [NSRL2_SOURCE]} + u_data = { + 'classification': cl_eng.RESTRICTED, + 'sources': [NSRL2_SOURCE] + } # Insert it and test return value resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_hash}/", method="PUT", data=json.dumps(u_data)) @@ -144,7 +146,7 @@ def test_whitelist_update(datastore, login_session): assert ds_u['added'] == ds_wl['added'] assert iso_to_epoch(ds_u['updated']) > iso_to_epoch(ds_wl['updated']) - assert ds_u['classification'] == cl_eng.UNRESTRICTED + assert ds_u['classification'] == cl_eng.RESTRICTED assert len(ds_u['sources']) == 2 assert NSRL2_SOURCE in ds_u['sources'] assert NSRL_SOURCE in ds_u['sources'] From 081f1cccdbed0e3055552c30ef0b94a3ff0b55b6 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Mon, 7 Jun 2021 17:16:18 +0000 Subject: [PATCH 09/31] Be more inclusive --- .../api/v4/{whitelist.py => safelist.py} | 67 +++++++++++++------ assemblyline_ui/app.py | 4 +- assemblyline_ui/helper/search.py | 4 +- test/{test_whitelist.py => test_safelist.py} | 56 ++++++++-------- test/test_search.py | 18 ++--- 5 files changed, 88 insertions(+), 61 deletions(-) rename assemblyline_ui/api/v4/{whitelist.py => safelist.py} (75%) rename test/{test_whitelist.py => test_safelist.py} (69%) diff --git a/assemblyline_ui/api/v4/whitelist.py b/assemblyline_ui/api/v4/safelist.py similarity index 75% rename from assemblyline_ui/api/v4/whitelist.py rename to assemblyline_ui/api/v4/safelist.py index f59e487d..bb734bfc 100644 --- a/assemblyline_ui/api/v4/whitelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -6,16 +6,16 @@ from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import CLASSIFICATION, STORAGE -SUB_API = 'whitelist' -whitelist_api = make_subapi_blueprint(SUB_API, api_version=4) -whitelist_api._doc = "Perform operations on whitelisted hashes" +SUB_API = 'safelist' +safelist_api = make_subapi_blueprint(SUB_API, api_version=4) +safelist_api._doc = "Perform operations on safelisted hashes" -@whitelist_api.route("//", methods=["PUT"]) +@safelist_api.route("//", methods=["PUT"]) @api_login(require_type=['user', 'signature_importer']) def add_or_update(sha256, **kwargs): """ - Add a hash in the whitelist if it does not exist or update its list of sources if it does + Add a hash in the safelist if it does not exist or update its list of sources if it does Variables: sha256 => Hash to check @@ -32,9 +32,9 @@ def add_or_update(sha256, **kwargs): "sha256": "12345....54321", # SHA256 of the file (default: sha256 variable) "size": 12345, # Size of the file "type": "document/text"}, # Type of the file - "sources": [ # List of sources for why the file is whitelisted, dedupped on name - Required - {"name": "NSRL", # Name of external source or user who whitelisted it - Required - "reason": [ # List of reasons why the source is whitelisted - Required + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"name": "NSRL", # Name of external source or user who safelisted it - Required + "reason": [ # List of reasons why the source is safelisted - Required "Found as test.txt on default windows 10 CD", "Found as install.txt on default windows XP CD" ], @@ -81,8 +81,8 @@ def add_or_update(sha256, **kwargs): src_map[src['name']] = src - with Lock(f'add_or_update-whitelist-{sha256}', 30): - old = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) + with Lock(f'add_or_update-safelist-{sha256}', 30): + old = STORAGE.safelist.get_if_exists(sha256, as_obj=False) if old: try: # Use old added date @@ -115,24 +115,24 @@ def add_or_update(sha256, **kwargs): data['sources'] = old_src_map.values() # Save data to the DB - STORAGE.whitelist.save(sha256, data) + STORAGE.safelist.save(sha256, data) return make_api_response({'success': True, "op": "update"}) except Exception as e: return make_api_response({}, f"Invalid data provided: {str(e)}", 400) else: try: data['sources'] = src_map.values() - STORAGE.whitelist.save(sha256, data) + STORAGE.safelist.save(sha256, data) return make_api_response({'success': True, "op": "add"}) except Exception as e: return make_api_response({}, f"Invalid data provided: {str(e)}", 400) -@whitelist_api.route("//", methods=["GET"]) +@safelist_api.route("//", methods=["GET"]) @api_login() def exists(sha256, **kwargs): """ - Check if a hash exists in the whitelist. + Check if a hash exists in the safelist. Variables: sha256 => Hash to check @@ -144,16 +144,43 @@ def exists(sha256, **kwargs): None API call example: - GET /api/v1/whitelist/123456...654321/ + GET /api/v1/safelist/123456...654321/ Result example: - + """ if len(sha256) != 64: return make_api_response(None, "Invalid sha256 hash length", 400) - whitelist = STORAGE.whitelist.get_if_exists(sha256, as_obj=False) - if whitelist and CLASSIFICATION.is_accessible(kwargs['user']['classification'], whitelist['classification']): - return make_api_response(whitelist) + safelist = STORAGE.safelist.get_if_exists(sha256, as_obj=False) + if safelist and CLASSIFICATION.is_accessible(kwargs['user']['classification'], safelist['classification']): + return make_api_response(safelist) - return make_api_response(None, "The hash was not found in the whitelist.", 404) + return make_api_response(None, "The hash was not found in the safelist.", 404) + + +@safelist_api.route("//", methods=["DELETE"]) +@api_login() +def delete(sha256, **kwargs): + """ + Delete a hash from the safelist + + Variables: + sha256 => Hash to check + + Arguments: + None + + Data Block: + None + + API call example: + DELET /api/v1/safelist/123456...654321/ + + Result example: + {"success": True} + """ + if len(sha256) != 64: + return make_api_response(None, "Invalid sha256 hash length", 400) + + return make_api_response(STORAGE.safelist.delete(sha256)) diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index 546fb5b2..b7acb57b 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -28,7 +28,7 @@ from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api -from assemblyline_ui.api.v4.whitelist import whitelist_api +from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.workflow import workflow_api from assemblyline_ui.error import errors from assemblyline_ui.healthz import healthz @@ -78,7 +78,7 @@ app.register_blueprint(ui_api) app.register_blueprint(user_api) app.register_blueprint(webauthn_api) -app.register_blueprint(whitelist_api) +app.register_blueprint(safelist_api) app.register_blueprint(workflow_api) diff --git a/assemblyline_ui/helper/search.py b/assemblyline_ui/helper/search.py index 0d337334..780218d6 100644 --- a/assemblyline_ui/helper/search.py +++ b/assemblyline_ui/helper/search.py @@ -7,7 +7,7 @@ 'result': STORAGE.result, 'signature': STORAGE.signature, 'submission': STORAGE.submission, - 'whitelist': STORAGE.whitelist, + 'safelist': STORAGE.safelist, 'workflow': STORAGE.workflow } @@ -18,7 +18,7 @@ 'result': "created desc", 'signature': "type asc", 'submission': "times.submitted desc", - 'whitelist': "added desc", + 'safelist': "added desc", 'workflow': "last_seen desc" } diff --git a/test/test_whitelist.py b/test/test_safelist.py similarity index 69% rename from test/test_whitelist.py rename to test/test_safelist.py index ee1eaada..d971403b 100644 --- a/test/test_whitelist.py +++ b/test/test_safelist.py @@ -6,7 +6,7 @@ from assemblyline.common.forge import get_classification from assemblyline.common.isotime import iso_to_epoch -from assemblyline.odm.random_data import create_users, create_whitelists, wipe_users, wipe_whitelist +from assemblyline.odm.random_data import create_users, create_safelists, wipe_users, wipe_safelist from assemblyline.odm.randomizer import get_random_hash from conftest import APIError, get_api_data @@ -50,18 +50,18 @@ def datastore(datastore_connection): try: create_users(datastore_connection) - create_whitelists(datastore_connection) + create_safelists(datastore_connection) yield datastore_connection finally: wipe_users(datastore_connection) - wipe_whitelist(datastore_connection) + wipe_safelist(datastore_connection) # noinspection PyUnusedLocal -def test_whitelist_add(datastore, login_session): +def test_safelist_add(datastore, login_session): _, session, host = login_session - # Generate a random whitelist + # Generate a random safelist wl_data = { 'fileinfo': {'md5': get_random_hash(32), 'sha1': get_random_hash(40), @@ -72,12 +72,12 @@ def test_whitelist_add(datastore, login_session): } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/whitelist/{add_hash}/", method="PUT", data=json.dumps(wl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/{add_hash}/", method="PUT", data=json.dumps(wl_data)) assert resp['success'] assert resp['op'] == 'add' # Load inserted data from DB - ds_wl = datastore.whitelist.get(add_hash, as_obj=False) + ds_wl = datastore.safelist.get(add_hash, as_obj=False) # Test dates added = ds_wl.pop('added', None) @@ -93,24 +93,24 @@ def test_whitelist_add(datastore, login_session): assert ds_wl == wl_data -def test_whitelist_add_invalid(datastore, login_session): +def test_safelist_add_invalid(datastore, login_session): _, session, host = login_session - # Generate a random whitelist + # Generate a random safelist wl_data = {'sources': [USER_SOURCE]} # Insert it and test return value with pytest.raises(APIError) as conflict_exc: - get_api_data(session, f"{host}/api/v4/whitelist/{add_error_hash}/", method="PUT", data=json.dumps(wl_data)) + get_api_data(session, f"{host}/api/v4/safelist/{add_error_hash}/", method="PUT", data=json.dumps(wl_data)) assert 'for another user' in conflict_exc.value.args[0] -def test_whitelist_update(datastore, login_session): +def test_safelist_update(datastore, login_session): _, session, host = login_session cl_eng = get_classification() - # Generate a random whitelist + # Generate a random safelist wl_data = { 'fileinfo': {'md5': get_random_hash(32), 'sha1': get_random_hash(40), @@ -121,12 +121,12 @@ def test_whitelist_update(datastore, login_session): } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_hash}/", method="PUT", data=json.dumps(wl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(wl_data)) assert resp['success'] assert resp['op'] == 'add' # Load inserted data from DB - ds_wl = datastore.whitelist.get(update_hash, as_obj=False) + ds_wl = datastore.safelist.get(update_hash, as_obj=False) # Test rest assert {k: v for k, v in ds_wl.items() if k not in ['added', 'updated', 'classification']} == wl_data @@ -137,12 +137,12 @@ def test_whitelist_update(datastore, login_session): } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_hash}/", method="PUT", data=json.dumps(u_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(u_data)) assert resp['success'] assert resp['op'] == 'update' # Load inserted data from DB - ds_u = datastore.whitelist.get(update_hash, as_obj=False) + ds_u = datastore.safelist.get(update_hash, as_obj=False) assert ds_u['added'] == ds_wl['added'] assert iso_to_epoch(ds_u['updated']) > iso_to_epoch(ds_wl['updated']) @@ -152,14 +152,14 @@ def test_whitelist_update(datastore, login_session): assert NSRL_SOURCE in ds_u['sources'] -def test_whitelist_update_conflict(datastore, login_session): +def test_safelist_update_conflict(datastore, login_session): _, session, host = login_session - # Generate a random whitelist + # Generate a random safelist wl_data = {'sources': [ADMIN_SOURCE]} # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/whitelist/{update_conflict_hash}/", + resp = get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", method="PUT", data=json.dumps(wl_data)) assert resp['success'] assert resp['op'] == 'add' @@ -167,37 +167,37 @@ def test_whitelist_update_conflict(datastore, login_session): # Insert the same source with a different type wl_data['sources'][0]['type'] = 'external' with pytest.raises(APIError) as conflict_exc: - get_api_data(session, f"{host}/api/v4/whitelist/{update_conflict_hash}/", + get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", method="PUT", data=json.dumps(wl_data)) assert 'Source type conflict' in conflict_exc.value.args[0] -def test_whitelist_exist(datastore, login_session): +def test_safelist_exist(datastore, login_session): _, session, host = login_session - hash = random.choice(datastore.whitelist.search("id:*", fl='id', rows=100, as_obj=False)['items'])['id'] + hash = random.choice(datastore.safelist.search("id:*", fl='id', rows=100, as_obj=False)['items'])['id'] - resp = get_api_data(session, f"{host}/api/v4/whitelist/{hash}/") - assert resp == datastore.whitelist.get(hash, as_obj=False) + resp = get_api_data(session, f"{host}/api/v4/safelist/{hash}/") + assert resp == datastore.safelist.get(hash, as_obj=False) # noinspection PyUnusedLocal -def test_whitelist_invalid(datastore, login_session): +def test_safelist_invalid(datastore, login_session): _, session, host = login_session with pytest.raises(APIError) as invalid_exc: - get_api_data(session, f"{host}/api/v4/whitelist/{get_random_hash(32)}/") + get_api_data(session, f"{host}/api/v4/safelist/{get_random_hash(32)}/") assert 'hash length' in invalid_exc.value.args[0] # noinspection PyUnusedLocal -def test_whitelist_missing(datastore, login_session): +def test_safelist_missing(datastore, login_session): _, session, host = login_session missing_hash = "f" + get_random_hash(63) with pytest.raises(APIError) as missing_exc: - get_api_data(session, f"{host}/api/v4/whitelist/{missing_hash}/") + get_api_data(session, f"{host}/api/v4/safelist/{missing_hash}/") assert 'not found' in missing_exc.value.args[0] diff --git a/test/test_search.py b/test/test_search.py index 71616866..5b76feb6 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -9,13 +9,13 @@ from assemblyline.odm.models.file import File from assemblyline.odm.models.result import Result from assemblyline.odm.models.submission import Submission -from assemblyline.odm.models.whitelist import Whitelist +from assemblyline.odm.models.safelist import Safelist from assemblyline.odm.models.workflow import Workflow from assemblyline.odm.randomizer import get_random_hash, random_model_obj from assemblyline.odm.random_data import create_users, wipe_users, create_signatures TEST_SIZE = 10 -collections = ['alert', 'file', 'heuristic', 'result', 'signature', 'submission', 'whitelist', 'workflow'] +collections = ['alert', 'file', 'heuristic', 'result', 'signature', 'submission', 'safelist', 'workflow'] file_list = [] signatures = [] @@ -62,9 +62,9 @@ def datastore(datastore_connection): for _ in range(TEST_SIZE): w_id = "0"+get_random_hash(63) - w = random_model_obj(Whitelist) - ds.whitelist.save(w_id, w) - ds.whitelist.commit() + w = random_model_obj(Safelist) + ds.safelist.save(w_id, w) + ds.safelist.commit() for _ in range(TEST_SIZE): w_id = get_random_id() @@ -80,7 +80,7 @@ def datastore(datastore_connection): ds.signature.wipe() ds.submission.wipe() ds.heuristic.wipe() - ds.whitelist.wipe() + ds.safelist.wipe() ds.workflow.wipe() wipe_users(ds) @@ -137,7 +137,7 @@ def test_histogram_search(datastore, login_session): 'heuristic': False, 'signature': 'last_modified', 'submission': 'times.submitted', - 'whitelist': 'added', + 'safelist': 'added', 'workflow': 'last_edit' } @@ -157,7 +157,7 @@ def test_histogram_search(datastore, login_session): 'signature': 'order', 'submission': 'file_count', 'heuristic': False, - 'whitelist': False, + 'safelist': False, 'workflow': 'hit_count' } @@ -201,7 +201,7 @@ def test_stats_search(datastore, login_session): 'signature': 'order', 'submission': 'file_count', 'heuristic': False, - 'whitelist': False, + 'safelist': False, 'workflow': 'hit_count' } From a4dd901a749ae3be680a09f083d7b8560fe27a50 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Tue, 8 Jun 2021 14:58:55 +0000 Subject: [PATCH 10/31] Added support for other hashes then just sha256 --- assemblyline_ui/api/v4/safelist.py | 47 +++++++++++++++++------------- test/test_safelist.py | 2 +- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index bb734bfc..17211bf6 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -11,14 +11,14 @@ safelist_api._doc = "Perform operations on safelisted hashes" -@safelist_api.route("//", methods=["PUT"]) +@safelist_api.route("//", methods=["PUT"]) @api_login(require_type=['user', 'signature_importer']) -def add_or_update(sha256, **kwargs): +def add_or_update(qhash, **kwargs): """ Add a hash in the safelist if it does not exist or update its list of sources if it does Variables: - sha256 => Hash to check + qhash => Hash to save informations about (either md5, sha1 or sha256) Arguments: None @@ -52,8 +52,8 @@ def add_or_update(sha256, **kwargs): } """ # Validate hash lenght - if len(sha256) != 64: - return make_api_response(None, "Invalid sha256 hash length", 400) + if len(qhash) not in [64, 40, 32]: + return make_api_response(None, "Invalid hash length", 400) # Load data data = request.json @@ -64,7 +64,12 @@ def add_or_update(sha256, **kwargs): # Set defaults data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) data.setdefault('fileinfo', {}) - data['fileinfo']['sha256'] = sha256 + if len(qhash) == 64: + data['fileinfo']['sha256'] = qhash + elif len(qhash) == 40: + data['fileinfo']['sha1'] = qhash + elif len(qhash) == 32: + data['fileinfo']['md5'] = qhash data['added'] = data['updated'] = now_as_iso() # Validate sources @@ -81,8 +86,8 @@ def add_or_update(sha256, **kwargs): src_map[src['name']] = src - with Lock(f'add_or_update-safelist-{sha256}', 30): - old = STORAGE.safelist.get_if_exists(sha256, as_obj=False) + with Lock(f'add_or_update-safelist-{qhash}', 30): + old = STORAGE.safelist.get_if_exists(qhash, as_obj=False) if old: try: # Use old added date @@ -115,27 +120,27 @@ def add_or_update(sha256, **kwargs): data['sources'] = old_src_map.values() # Save data to the DB - STORAGE.safelist.save(sha256, data) + STORAGE.safelist.save(qhash, data) return make_api_response({'success': True, "op": "update"}) except Exception as e: return make_api_response({}, f"Invalid data provided: {str(e)}", 400) else: try: data['sources'] = src_map.values() - STORAGE.safelist.save(sha256, data) + STORAGE.safelist.save(qhash, data) return make_api_response({'success': True, "op": "add"}) except Exception as e: return make_api_response({}, f"Invalid data provided: {str(e)}", 400) -@safelist_api.route("//", methods=["GET"]) +@safelist_api.route("//", methods=["GET"]) @api_login() -def exists(sha256, **kwargs): +def exists(qhash, **kwargs): """ Check if a hash exists in the safelist. Variables: - sha256 => Hash to check + qhash => Hash to check is exist (either md5, sha1 or sha256) Arguments: None @@ -149,19 +154,19 @@ def exists(sha256, **kwargs): Result example: """ - if len(sha256) != 64: - return make_api_response(None, "Invalid sha256 hash length", 400) + if len(qhash) not in [64, 40, 32]: + return make_api_response(None, "Invalid hash length", 400) - safelist = STORAGE.safelist.get_if_exists(sha256, as_obj=False) + safelist = STORAGE.safelist.get_if_exists(qhash, as_obj=False) if safelist and CLASSIFICATION.is_accessible(kwargs['user']['classification'], safelist['classification']): return make_api_response(safelist) return make_api_response(None, "The hash was not found in the safelist.", 404) -@safelist_api.route("//", methods=["DELETE"]) +@safelist_api.route("//", methods=["DELETE"]) @api_login() -def delete(sha256, **kwargs): +def delete(qhash, **_): """ Delete a hash from the safelist @@ -180,7 +185,7 @@ def delete(sha256, **kwargs): Result example: {"success": True} """ - if len(sha256) != 64: - return make_api_response(None, "Invalid sha256 hash length", 400) + if len(qhash) not in [64, 40, 32]: + return make_api_response(None, "Invalid hash length", 400) - return make_api_response(STORAGE.safelist.delete(sha256)) + return make_api_response(STORAGE.safelist.delete(qhash)) diff --git a/test/test_safelist.py b/test/test_safelist.py index d971403b..6d06d008 100644 --- a/test/test_safelist.py +++ b/test/test_safelist.py @@ -187,7 +187,7 @@ def test_safelist_invalid(datastore, login_session): _, session, host = login_session with pytest.raises(APIError) as invalid_exc: - get_api_data(session, f"{host}/api/v4/safelist/{get_random_hash(32)}/") + get_api_data(session, f"{host}/api/v4/safelist/{get_random_hash(12)}/") assert 'hash length' in invalid_exc.value.args[0] From 5c8274ac4660ce56722018b6e163f40bc9541ec4 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 9 Jun 2021 14:40:08 +0000 Subject: [PATCH 11/31] Added a add_update_many function for hashes --- assemblyline_ui/api/v4/safelist.py | 90 ++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 17211bf6..4aa74cd8 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -11,8 +11,8 @@ safelist_api._doc = "Perform operations on safelisted hashes" -@safelist_api.route("//", methods=["PUT"]) -@api_login(require_type=['user', 'signature_importer']) +@safelist_api.route("//", methods=["PUT", "POST"]) +@api_login(require_type=['user', 'signature_importer'], allow_readonly=False, required_priv=["W"]) def add_or_update(qhash, **kwargs): """ Add a hash in the safelist if it does not exist or update its list of sources if it does @@ -51,7 +51,7 @@ def add_or_update(qhash, **kwargs): "op": "add" # Was it added to the system or updated } """ - # Validate hash lenght + # Validate hash length if len(qhash) not in [64, 40, 32]: return make_api_response(None, "Invalid hash length", 400) @@ -133,8 +133,88 @@ def add_or_update(qhash, **kwargs): return make_api_response({}, f"Invalid data provided: {str(e)}", 400) +@safelist_api.route("/add_update_many/", methods=["POST", "PUT"]) +@api_login(audit=False, required_priv=['W'], allow_readonly=False, require_type=['signature_importer']) +def add_update_many_hashes(**_): + """ + Add or Update a list of the safe hashes + + Variables: + None + + Arguments: + None + + Data Block (REQUIRED): + [ # List of Safe hash blocks + { + "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional + "fileinfo": { # Information about the file - Optional + "md5": "123...321", # MD5 hash of the file + "sha1": "1234...4321", # SHA1 hash of the file + "sha256": "12345....54321", # SHA256 of the file (default: sha256 variable) + "size": 12345, # Size of the file + "type": "document/text"}, # Type of the file + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"name": "NSRL", # Name of external source or user who safelisted it - Required + "reason": [ # List of reasons why the source is safelisted - Required + "Found as test.txt on default windows 10 CD", + "Found as install.txt on default windows XP CD" + ], + "type": "external"}, # Type or source (external or user) - Required + {"name": "admin", + "reason": ["We've seen this file many times and it leads to False positives"], + "type": "user"} + ] + } + ... + ] + + Result example: + {"success": 23, # Number of hashes that succeeded + "errors": [], # List of hashes that failed + "skipped": [], # List of skipped hashes, they already exist + """ + data = request.json + + if not isinstance(data, list): + return make_api_response("", "Could not get the list of hashes", 400) + + new_data = {} + for hash_data in data: + hash_data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) + fileinfo = hash_data.get('fileinfo', {}) + key = fileinfo.get('sha256', fileinfo.get('sha1', fileinfo.get('md5', None))) + if not key: + return make_api_response("", f"Invalid hash block: {str(hash_data)}", 400) + new_data[key] = hash_data + + old_data = STORAGE.safelist.multiget(list(new_data.keys()), as_dictionary=True, as_obj=False, + error_on_missing=False) + + # Test signature names + plan = STORAGE.safelist.get_bulk_plan() + for key, val in new_data.items(): + old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED, 'fileinfo': {}, 'sources': []}) + old_val['classification'] = CLASSIFICATION.max_classification( + old_val['classification'], val['classification']) + old_val['updated'] = now_as_iso() + old_val['fileinfo'].update(val['fileinfo']) + for source in val['sources']: + if source not in old_val['sources']: + old_val['sources'].append(source) + + plan.add_upsert_operation(key, old_val) + + if not plan.empty: + res = STORAGE.safelist.bulk(plan) + return make_api_response({"success": len(res['items']), "errors": res['errors']}) + + return make_api_response({"success": 0, "errors": []}) + + @safelist_api.route("//", methods=["GET"]) -@api_login() +@api_login(required_priv=["R"]) def exists(qhash, **kwargs): """ Check if a hash exists in the safelist. @@ -165,7 +245,7 @@ def exists(qhash, **kwargs): @safelist_api.route("//", methods=["DELETE"]) -@api_login() +@api_login(allow_readonly=False) def delete(qhash, **_): """ Delete a hash from the safelist From 7523a6ac31133d58efa737884998452ecb98d5a7 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 9 Jun 2021 17:45:13 +0000 Subject: [PATCH 12/31] Docs improvements --- assemblyline_ui/api/v4/safelist.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 4aa74cd8..ceabefd6 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -172,8 +172,7 @@ def add_update_many_hashes(**_): Result example: {"success": 23, # Number of hashes that succeeded - "errors": [], # List of hashes that failed - "skipped": [], # List of skipped hashes, they already exist + "errors": []} # List of hashes that failed """ data = request.json @@ -232,7 +231,26 @@ def exists(qhash, **kwargs): GET /api/v1/safelist/123456...654321/ Result example: - + { + "classification": "TLP:W", # Classification of the file + "fileinfo": { # Information about the file + "md5": "123...321", # MD5 hash of the file + "sha1": "1234...4321", # SHA1 hash of the file + "sha256": "12345....54321", # SHA256 of the file + "size": 12345, # Size of the file + "type": "document/text"}, # Type of the file + "sources": [ # List of sources for why the file is safelisted, dedupped on name + {"name": "NSRL", # Name of external source or user who safelisted it + "reason": [ # List of reasons why the source is safelisted + "Found as test.txt on default windows 10 CD", + "Found as install.txt on default windows XP CD" + ], + "type": "external"}, # Type or source (external or user) + {"name": "admin", + "reason": ["We've seen this file many times and it leads to False positives"], + "type": "user"} + ] + } """ if len(qhash) not in [64, 40, 32]: return make_api_response(None, "Invalid hash length", 400) From 3409317753163706b8eb8b8ec6f0d02b3f015131 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 9 Jun 2021 17:49:39 +0000 Subject: [PATCH 13/31] Update function names --- assemblyline_ui/api/v4/safelist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index ceabefd6..d03b3cbe 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -13,7 +13,7 @@ @safelist_api.route("//", methods=["PUT", "POST"]) @api_login(require_type=['user', 'signature_importer'], allow_readonly=False, required_priv=["W"]) -def add_or_update(qhash, **kwargs): +def add_or_update_hash(qhash, **kwargs): """ Add a hash in the safelist if it does not exist or update its list of sources if it does @@ -214,7 +214,7 @@ def add_update_many_hashes(**_): @safelist_api.route("//", methods=["GET"]) @api_login(required_priv=["R"]) -def exists(qhash, **kwargs): +def check_hash_exists(qhash, **kwargs): """ Check if a hash exists in the safelist. @@ -264,7 +264,7 @@ def exists(qhash, **kwargs): @safelist_api.route("//", methods=["DELETE"]) @api_login(allow_readonly=False) -def delete(qhash, **_): +def delete_hash(qhash, **_): """ Delete a hash from the safelist From 5cf06457dddfd9e170eae7440dc54bd99f1b921a Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 9 Jun 2021 18:18:10 +0000 Subject: [PATCH 14/31] Added the safe category for heuristics --- assemblyline_ui/api/v4/file.py | 4 +++- assemblyline_ui/api/v4/submission.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/api/v4/file.py b/assemblyline_ui/api/v4/file.py index 61b3d38a..954f0c58 100644 --- a/assemblyline_ui/api/v4/file.py +++ b/assemblyline_ui/api/v4/file.py @@ -491,7 +491,9 @@ def get_file_results(sha256, **kwargs): if sec.get('heuristic', False): # Get the heuristics data - if sec['heuristic']['score'] < 100: + if sec['heuristic']['score'] < 0: + h_type = "safe" + elif sec['heuristic']['score'] < 100: h_type = "info" elif sec['heuristic']['score'] < 1000: h_type = "suspicious" diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index f2f4d16a..d923f330 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -149,7 +149,9 @@ def get_file_submission_results(sid, sha256, **kwargs): h_type = "info" if sec.get('heuristic', False): # Get the heuristics data - if sec['heuristic']['score'] < 100: + if sec['heuristic']['score'] < 0: + h_type = "safe" + elif sec['heuristic']['score'] < 100: h_type = "info" elif sec['heuristic']['score'] < 1000: h_type = "suspicious" From 2f6d9e86e700e8f7a34232fdd79533cbaad8850c Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 9 Jun 2021 18:35:35 +0000 Subject: [PATCH 15/31] Added comments and simplyfied adding hashes --- assemblyline_ui/api/v4/safelist.py | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index d03b3cbe..c8822f7b 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -11,25 +11,22 @@ safelist_api._doc = "Perform operations on safelisted hashes" -@safelist_api.route("//", methods=["PUT", "POST"]) +@safelist_api.route("/", methods=["PUT", "POST"]) @api_login(require_type=['user', 'signature_importer'], allow_readonly=False, required_priv=["W"]) -def add_or_update_hash(qhash, **kwargs): +def add_or_update_hash(**kwargs): """ Add a hash in the safelist if it does not exist or update its list of sources if it does - Variables: - qhash => Hash to save informations about (either md5, sha1 or sha256) - Arguments: None Data Block: { "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional - "fileinfo": { # Information about the file - Optional + "fileinfo": { # Information about the file - At least one hash required "md5": "123...321", # MD5 hash of the file "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file (default: sha256 variable) + "sha256": "12345....54321", # SHA256 of the file "size": 12345, # Size of the file "type": "document/text"}, # Type of the file "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required @@ -51,10 +48,6 @@ def add_or_update_hash(qhash, **kwargs): "op": "add" # Was it added to the system or updated } """ - # Validate hash length - if len(qhash) not in [64, 40, 32]: - return make_api_response(None, "Invalid hash length", 400) - # Load data data = request.json if not data: @@ -64,14 +57,14 @@ def add_or_update_hash(qhash, **kwargs): # Set defaults data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) data.setdefault('fileinfo', {}) - if len(qhash) == 64: - data['fileinfo']['sha256'] = qhash - elif len(qhash) == 40: - data['fileinfo']['sha1'] = qhash - elif len(qhash) == 32: - data['fileinfo']['md5'] = qhash data['added'] = data['updated'] = now_as_iso() + # Find the best hash to use for the key + qhash = data['fileinfo'].get('sha256', data['fileinfo'].get('sha1', data['fileinfo'].get('md5', None))) + # Validate hash length + if not qhash: + return make_api_response(None, "No valid hash found", 400) + # Validate sources src_map = {} for src in data['sources']: @@ -181,31 +174,46 @@ def add_update_many_hashes(**_): new_data = {} for hash_data in data: + # Set a classification if None hash_data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) + + # Find the hash used for the key fileinfo = hash_data.get('fileinfo', {}) key = fileinfo.get('sha256', fileinfo.get('sha1', fileinfo.get('md5', None))) if not key: return make_api_response("", f"Invalid hash block: {str(hash_data)}", 400) + + # Save the new hash_block new_data[key] = hash_data + # Get already existing hashes old_data = STORAGE.safelist.multiget(list(new_data.keys()), as_dictionary=True, as_obj=False, error_on_missing=False) # Test signature names plan = STORAGE.safelist.get_bulk_plan() for key, val in new_data.items(): + # Use maximum classification old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED, 'fileinfo': {}, 'sources': []}) old_val['classification'] = CLASSIFICATION.max_classification( old_val['classification'], val['classification']) + + # Update updated time old_val['updated'] = now_as_iso() + + # Update fileinfo old_val['fileinfo'].update(val['fileinfo']) + + # Merge sources for source in val['sources']: if source not in old_val['sources']: old_val['sources'].append(source) + # Add upsert operation plan.add_upsert_operation(key, old_val) if not plan.empty: + # Execute plan res = STORAGE.safelist.bulk(plan) return make_api_response({"success": len(res['items']), "errors": res['errors']}) @@ -278,7 +286,7 @@ def delete_hash(qhash, **_): None API call example: - DELET /api/v1/safelist/123456...654321/ + DELETE /api/v1/safelist/123456...654321/ Result example: {"success": True} From 199aef2c5c02758e24279d2706c0e84f04ce0164 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 10 Jun 2021 01:12:04 +0000 Subject: [PATCH 16/31] Merge sources in a better way --- assemblyline_ui/api/v4/safelist.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index c8822f7b..75357178 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -104,7 +104,7 @@ def add_or_update_hash(**kwargs): old_src = old_src_map[name] if old_src['type'] != src['type']: return make_api_response( - {}, f"Source type conflict for {name}: {old_src['type']} != {src['type']}", 400) + {}, f"Source {name} has a type conflict: {old_src['type']} != {src['type']}", 400) for reason in src['reason']: if reason not in old_src['reason']: @@ -205,9 +205,24 @@ def add_update_many_hashes(**_): old_val['fileinfo'].update(val['fileinfo']) # Merge sources - for source in val['sources']: - if source not in old_val['sources']: - old_val['sources'].append(source) + src_map = {x['name']: x for x in val['sources'] if x['type'] == 'external'} + if not src_map: + make_api_response({}, f"No valid source found for {key}", 400) + + old_src_map = {x['name']: x for x in old_val['sources']} + for name, src in src_map.items(): + if name not in old_src_map: + old_src_map[name] = src + else: + old_src = old_src_map[name] + if old_src['type'] != src['type']: + return make_api_response( + {}, f"Hash {key} source {name} has a type conflict: {old_src['type']} != {src['type']}", 400) + + for reason in src['reason']: + if reason not in old_src['reason']: + old_src['reason'].append(reason) + old_val['sources'] = old_src_map.values() # Add upsert operation plan.add_upsert_operation(key, old_val) From c4f95b892e10f07618cb5ac84506074f1fa0d235 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 10 Jun 2021 01:23:17 +0000 Subject: [PATCH 17/31] Allow total hit tracking via search API --- assemblyline_ui/api/v4/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/api/v4/search.py b/assemblyline_ui/api/v4/search.py index 0b534512..20885335 100644 --- a/assemblyline_ui/api/v4/search.py +++ b/assemblyline_ui/api/v4/search.py @@ -55,7 +55,7 @@ def search(bucket, **kwargs): return make_api_response("", f"Not a valid bucket to search in: {bucket}", 400) user = kwargs['user'] - fields = ["offset", "rows", "sort", "fl", "timeout", "deep_paging_id"] + fields = ["offset", "rows", "sort", "fl", "timeout", "deep_paging_id", 'track_total_hits'] multi_fields = ['filters'] boolean_fields = ['use_archive'] From fc1985fe5f5e23ba966b019c77db13a11971ac9a Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 10 Jun 2021 16:00:48 +0000 Subject: [PATCH 18/31] Fully delete can only be done by sig manager, admins or user if they are the only source --- assemblyline_ui/api/v4/safelist.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 75357178..c2f5420e 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -287,7 +287,7 @@ def check_hash_exists(qhash, **kwargs): @safelist_api.route("//", methods=["DELETE"]) @api_login(allow_readonly=False) -def delete_hash(qhash, **_): +def delete_hash(qhash, **kwargs): """ Delete a hash from the safelist @@ -306,7 +306,20 @@ def delete_hash(qhash, **_): Result example: {"success": True} """ + user = kwargs['user'] + if len(qhash) not in [64, 40, 32]: return make_api_response(None, "Invalid hash length", 400) - return make_api_response(STORAGE.safelist.delete(qhash)) + if 'admin' in user['type'] or 'signature_manager' in user['type']: + return make_api_response({'success': STORAGE.safelist.delete(qhash)}) + else: + safe_hash = STORAGE.safelist.get_if_exists(qhash, as_obj=False) + if safe_hash: + safe_hash['sources'] = [x for x in safe_hash['sources'] if x['name'] != user['uname']] + if len(safe_hash['sources']) == 0: + return make_api_response({'success': STORAGE.safelist.delete(qhash)}) + else: + return make_api_response({'success': STORAGE.safelist.save(qhash, safe_hash)}) + + return make_api_response({'success': False}) From 3183959b615e1c16c4f320e4bdbba04794d3a429 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 11 Jun 2021 04:25:51 +0000 Subject: [PATCH 19/31] Add support for tag and disabling --- assemblyline_ui/api/v4/safelist.py | 46 +++++++++++++++++++++++++++--- test/test_safelist.py | 38 ++++++++++++------------ 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index c2f5420e..80fe6c66 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -60,7 +60,7 @@ def add_or_update_hash(**kwargs): data['added'] = data['updated'] = now_as_iso() # Find the best hash to use for the key - qhash = data['fileinfo'].get('sha256', data['fileinfo'].get('sha1', data['fileinfo'].get('md5', None))) + qhash = data['hashes'].get('sha256', data['hashes'].get('sha1', data['hashes'].get('md5', None))) # Validate hash length if not qhash: return make_api_response(None, "No valid hash found", 400) @@ -77,6 +77,10 @@ def add_or_update_hash(**kwargs): return make_api_response( {}, "You do not have sufficient priviledges to add an external source.", 403) + src_cl = src.get('classification', None) + if src_cl: + data['classification'] = CLASSIFICATION.max_classification(data['classification'], src_cl) + src_map[src['name']] = src with Lock(f'add_or_update-safelist-{qhash}', 30): @@ -178,8 +182,7 @@ def add_update_many_hashes(**_): hash_data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) # Find the hash used for the key - fileinfo = hash_data.get('fileinfo', {}) - key = fileinfo.get('sha256', fileinfo.get('sha1', fileinfo.get('md5', None))) + key = hash_data['hashes'].get('sha256', hash_data['hashes'].get('sha1', hash_data['hashes'].get('md5', None))) if not key: return make_api_response("", f"Invalid hash block: {str(hash_data)}", 400) @@ -211,6 +214,10 @@ def add_update_many_hashes(**_): old_src_map = {x['name']: x for x in old_val['sources']} for name, src in src_map.items(): + src_cl = src.get('classification', None) + if src_cl: + data['classification'] = CLASSIFICATION.max_classification(data['classification'], src_cl) + if name not in old_src_map: old_src_map[name] = src else: @@ -285,6 +292,37 @@ def check_hash_exists(qhash, **kwargs): return make_api_response(None, "The hash was not found in the safelist.", 404) +@safelist_api.route("/enable//", methods=["PUT"]) +@api_login(allow_readonly=False) +def set_hash_status(qhash, **kwargs): + """ + Set the enabled status of a hash + + Variables: + qhash => Hash to change the status + + Arguments: + None + + Data Block: + "true" + + Result example: + {"success": True} + """ + user = kwargs['user'] + data = request.json + + if len(qhash) not in [64, 40, 32]: + return make_api_response(None, "Invalid hash length", 400) + + if 'admin' in user['type'] or 'signature_manager' in user['type']: + return make_api_response({'success': STORAGE.safelist.update( + qhash, [(STORAGE.safelist.UPDATE_SET, 'enabled', data)])}) + + return make_api_response({}, "You are not allowed to change the status", 403) + + @safelist_api.route("//", methods=["DELETE"]) @api_login(allow_readonly=False) def delete_hash(qhash, **kwargs): @@ -292,7 +330,7 @@ def delete_hash(qhash, **kwargs): Delete a hash from the safelist Variables: - sha256 => Hash to check + qhash => Hash to check Arguments: None diff --git a/test/test_safelist.py b/test/test_safelist.py index 6d06d008..184cacb1 100644 --- a/test/test_safelist.py +++ b/test/test_safelist.py @@ -62,7 +62,7 @@ def test_safelist_add(datastore, login_session): _, session, host = login_session # Generate a random safelist - wl_data = { + sl_data = { 'fileinfo': {'md5': get_random_hash(32), 'sha1': get_random_hash(40), 'sha256': add_hash, @@ -72,36 +72,36 @@ def test_safelist_add(datastore, login_session): } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{add_hash}/", method="PUT", data=json.dumps(wl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/{add_hash}/", method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' # Load inserted data from DB - ds_wl = datastore.safelist.get(add_hash, as_obj=False) + ds_sl = datastore.safelist.get(add_hash, as_obj=False) # Test dates - added = ds_wl.pop('added', None) - updated = ds_wl.pop('updated', None) + added = ds_sl.pop('added', None) + updated = ds_sl.pop('updated', None) assert added == updated assert added is not None and updated is not None # Test classification - classification = ds_wl.pop('classification', None) + classification = ds_sl.pop('classification', None) assert classification is not None # Test rest - assert ds_wl == wl_data + assert ds_sl == sl_data def test_safelist_add_invalid(datastore, login_session): _, session, host = login_session # Generate a random safelist - wl_data = {'sources': [USER_SOURCE]} + sl_data = {'sources': [USER_SOURCE]} # Insert it and test return value with pytest.raises(APIError) as conflict_exc: - get_api_data(session, f"{host}/api/v4/safelist/{add_error_hash}/", method="PUT", data=json.dumps(wl_data)) + get_api_data(session, f"{host}/api/v4/safelist/{add_error_hash}/", method="PUT", data=json.dumps(sl_data)) assert 'for another user' in conflict_exc.value.args[0] @@ -111,7 +111,7 @@ def test_safelist_update(datastore, login_session): cl_eng = get_classification() # Generate a random safelist - wl_data = { + sl_data = { 'fileinfo': {'md5': get_random_hash(32), 'sha1': get_random_hash(40), 'sha256': update_hash, @@ -121,15 +121,15 @@ def test_safelist_update(datastore, login_session): } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(wl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' # Load inserted data from DB - ds_wl = datastore.safelist.get(update_hash, as_obj=False) + ds_sl = datastore.safelist.get(update_hash, as_obj=False) # Test rest - assert {k: v for k, v in ds_wl.items() if k not in ['added', 'updated', 'classification']} == wl_data + assert {k: v for k, v in ds_sl.items() if k not in ['added', 'updated', 'classification']} == sl_data u_data = { 'classification': cl_eng.RESTRICTED, @@ -144,8 +144,8 @@ def test_safelist_update(datastore, login_session): # Load inserted data from DB ds_u = datastore.safelist.get(update_hash, as_obj=False) - assert ds_u['added'] == ds_wl['added'] - assert iso_to_epoch(ds_u['updated']) > iso_to_epoch(ds_wl['updated']) + assert ds_u['added'] == ds_sl['added'] + assert iso_to_epoch(ds_u['updated']) > iso_to_epoch(ds_sl['updated']) assert ds_u['classification'] == cl_eng.RESTRICTED assert len(ds_u['sources']) == 2 assert NSRL2_SOURCE in ds_u['sources'] @@ -156,19 +156,19 @@ def test_safelist_update_conflict(datastore, login_session): _, session, host = login_session # Generate a random safelist - wl_data = {'sources': [ADMIN_SOURCE]} + sl_data = {'sources': [ADMIN_SOURCE]} # Insert it and test return value resp = get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", - method="PUT", data=json.dumps(wl_data)) + method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' # Insert the same source with a different type - wl_data['sources'][0]['type'] = 'external' + sl_data['sources'][0]['type'] = 'external' with pytest.raises(APIError) as conflict_exc: get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", - method="PUT", data=json.dumps(wl_data)) + method="PUT", data=json.dumps(sl_data)) assert 'Source type conflict' in conflict_exc.value.args[0] From 31101994158dc0700b7b65f87d7688c34c0c463a Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 11 Jun 2021 13:40:24 +0000 Subject: [PATCH 20/31] Re-use the same merge function --- assemblyline_ui/api/v4/safelist.py | 223 +++++++++++++++++------------ 1 file changed, 129 insertions(+), 94 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 80fe6c66..d9c2e871 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -11,6 +11,66 @@ safelist_api._doc = "Perform operations on safelisted hashes" +class InvalidSafehash(Exception): + pass + + +def _merge_safe_hashes(new, old): + try: + # Check if hash types match + if new['type'] != old['type']: + raise InvalidSafehash(f"Safe hash type mismatch: {new['type']} != {old['type']}") + + # Use max classification + old['classification'] = CLASSIFICATION.max_classification(old['classification'], new['classification']) + + # Update updated time + old['updated'] = now_as_iso() + + # Update hashes + old['hashes'].update(new['hashes']) + + # Update type specific info + if old['type'] == 'file': + old.setdefault('file', {}) + new_names = new.get('file', {}).pop('name', []) + if 'name' in old['file']: + for name in new_names: + if name not in old['file']['name']: + old['file']['name'].append(name) + elif new_names: + old['file']['name'] = new_names + old['file'].update(new.get('file', {})) + elif old['type'] == 'tag': + old['tag'] = new['tag'] + + # Merge sources + src_map = {x['name']: x for x in new['sources']} + if not src_map: + raise InvalidSafehash("No valid source found") + + old_src_map = {x['name']: x for x in old['sources']} + for name, src in src_map.items(): + src_cl = src.get('classification', None) + if src_cl: + old['classification'] = CLASSIFICATION.max_classification(old['classification'], src_cl) + + if name not in old_src_map: + old_src_map[name] = src + else: + old_src = old_src_map[name] + if old_src['type'] != src['type']: + raise InvalidSafehash(f"Source {name} has a type conflict: {old_src['type']} != {src['type']}") + + for reason in src['reason']: + if reason not in old_src['reason']: + old_src['reason'].append(reason) + old['sources'] = old_src_map.values() + return old + except Exception as e: + raise InvalidSafehash(f"Invalid data provided: {str(e)}") + + @safelist_api.route("/", methods=["PUT", "POST"]) @api_login(require_type=['user', 'signature_importer'], allow_readonly=False, required_priv=["W"]) def add_or_update_hash(**kwargs): @@ -23,12 +83,15 @@ def add_or_update_hash(**kwargs): Data Block: { "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional - "fileinfo": { # Information about the file - At least one hash required + "file": { # Information about the file - Only used in file mode + "name": ["file.txt"] # Possible names for the file + "size": 12345, # Size of the file + "type": "document/text"}, # Type of the file + }, + "hashes": { # Information about the file - At least one hash required "md5": "123...321", # MD5 hash of the file "sha1": "1234...4321", # SHA1 hash of the file "sha256": "12345....54321", # SHA256 of the file - "size": 12345, # Size of the file - "type": "document/text"}, # Type of the file "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required {"name": "NSRL", # Name of external source or user who safelisted it - Required "reason": [ # List of reasons why the source is safelisted - Required @@ -39,7 +102,12 @@ def add_or_update_hash(**kwargs): {"name": "admin", "reason": ["We've seen this file many times and it leads to False positives"], "type": "user"} - ] + ], + "tag": { # Tag information - Only used in tag mode + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag + }, + "type": "tag" # Type of safelist hash (tag or file) } Result example: @@ -56,7 +124,11 @@ def add_or_update_hash(**kwargs): # Set defaults data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) - data.setdefault('fileinfo', {}) + data.setdefault('hashes', {}) + if data['type'] == 'tag': + data.pop('file', None) + elif data['type'] == 'file': + data.pop('tag', None) data['added'] = data['updated'] = now_as_iso() # Find the best hash to use for the key @@ -87,40 +159,11 @@ def add_or_update_hash(**kwargs): old = STORAGE.safelist.get_if_exists(qhash, as_obj=False) if old: try: - # Use old added date - data['added'] = old['added'] - - # Use minimal classification - data['classification'] = CLASSIFICATION.max_classification( - data['classification'], old['classification']) - - # Merge file info (keep new values) - for k, v in old['fileinfo'].items(): - if k not in data['fileinfo']: - data['fileinfo'][k] = v - - # Merge sources together - old_src_map = {x['name']: x for x in old['sources']} - for name, src in src_map.items(): - if name not in old_src_map: - old_src_map[name] = src - else: - old_src = old_src_map[name] - if old_src['type'] != src['type']: - return make_api_response( - {}, f"Source {name} has a type conflict: {old_src['type']} != {src['type']}", 400) - - for reason in src['reason']: - if reason not in old_src['reason']: - old_src['reason'].append(reason) - - data['sources'] = old_src_map.values() - # Save data to the DB - STORAGE.safelist.save(qhash, data) + STORAGE.safelist.save(qhash, _merge_safe_hashes(data, old)) return make_api_response({'success': True, "op": "update"}) - except Exception as e: - return make_api_response({}, f"Invalid data provided: {str(e)}", 400) + except InvalidSafehash as e: + return make_api_response({}, str(e), 400) else: try: data['sources'] = src_map.values() @@ -146,12 +189,15 @@ def add_update_many_hashes(**_): [ # List of Safe hash blocks { "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional - "fileinfo": { # Information about the file - Optional - "md5": "123...321", # MD5 hash of the file - "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file (default: sha256 variable) + "file": { # Information about the file - Only used in file mode + "name": ["file.txt"] # Possible names for the file "size": 12345, # Size of the file "type": "document/text"}, # Type of the file + }, + "hashes": { # Information about the file - At least one hash required + "md5": "123...321", # MD5 hash of the file + "sha1": "1234...4321", # SHA1 hash of the file + "sha256": "12345....54321", # SHA256 of the file "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required {"name": "NSRL", # Name of external source or user who safelisted it - Required "reason": [ # List of reasons why the source is safelisted - Required @@ -162,7 +208,12 @@ def add_update_many_hashes(**_): {"name": "admin", "reason": ["We've seen this file many times and it leads to False positives"], "type": "user"} - ] + ], + "tag": { # Tag information - Only used in tag mode + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag + }, + "type": "tag" # Type of safelist hash (tag or file) } ... ] @@ -180,6 +231,10 @@ def add_update_many_hashes(**_): for hash_data in data: # Set a classification if None hash_data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) + if hash_data['type'] == 'tag': + hash_data.pop('file', None) + elif hash_data['type'] == 'file': + hash_data.pop('tag', None) # Find the hash used for the key key = hash_data['hashes'].get('sha256', hash_data['hashes'].get('sha1', hash_data['hashes'].get('md5', None))) @@ -197,42 +252,14 @@ def add_update_many_hashes(**_): plan = STORAGE.safelist.get_bulk_plan() for key, val in new_data.items(): # Use maximum classification - old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED, 'fileinfo': {}, 'sources': []}) - old_val['classification'] = CLASSIFICATION.max_classification( - old_val['classification'], val['classification']) - - # Update updated time - old_val['updated'] = now_as_iso() - - # Update fileinfo - old_val['fileinfo'].update(val['fileinfo']) - - # Merge sources - src_map = {x['name']: x for x in val['sources'] if x['type'] == 'external'} - if not src_map: - make_api_response({}, f"No valid source found for {key}", 400) - - old_src_map = {x['name']: x for x in old_val['sources']} - for name, src in src_map.items(): - src_cl = src.get('classification', None) - if src_cl: - data['classification'] = CLASSIFICATION.max_classification(data['classification'], src_cl) - - if name not in old_src_map: - old_src_map[name] = src - else: - old_src = old_src_map[name] - if old_src['type'] != src['type']: - return make_api_response( - {}, f"Hash {key} source {name} has a type conflict: {old_src['type']} != {src['type']}", 400) - - for reason in src['reason']: - if reason not in old_src['reason']: - old_src['reason'].append(reason) - old_val['sources'] = old_src_map.values() + old_val = old_data.get(key, {'classification': CLASSIFICATION.UNRESTRICTED, + 'hashes': {}, 'sources': [], 'type': val['type']}) # Add upsert operation - plan.add_upsert_operation(key, old_val) + try: + plan.add_upsert_operation(key, _merge_safe_hashes(val, old_val)) + except InvalidSafehash as e: + return make_api_response("", str(e), 400) if not plan.empty: # Execute plan @@ -262,24 +289,32 @@ def check_hash_exists(qhash, **kwargs): Result example: { - "classification": "TLP:W", # Classification of the file - "fileinfo": { # Information about the file - "md5": "123...321", # MD5 hash of the file - "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file - "size": 12345, # Size of the file - "type": "document/text"}, # Type of the file - "sources": [ # List of sources for why the file is safelisted, dedupped on name - {"name": "NSRL", # Name of external source or user who safelisted it - "reason": [ # List of reasons why the source is safelisted - "Found as test.txt on default windows 10 CD", - "Found as install.txt on default windows XP CD" - ], - "type": "external"}, # Type or source (external or user) - {"name": "admin", - "reason": ["We've seen this file many times and it leads to False positives"], - "type": "user"} - ] + "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional + "file": { # Information about the file - Only used in file mode + "name": ["file.txt"] # Possible names for the file + "size": 12345, # Size of the file + "type": "document/text"}, # Type of the file + }, + "hashes": { # Information about the file - At least one hash required + "md5": "123...321", # MD5 hash of the file + "sha1": "1234...4321", # SHA1 hash of the file + "sha256": "12345....54321", # SHA256 of the file + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"name": "NSRL", # Name of external source or user who safelisted it - Required + "reason": [ # List of reasons why the source is safelisted - Required + "Found as test.txt on default windows 10 CD", + "Found as install.txt on default windows XP CD" + ], + "type": "external"}, # Type or source (external or user) - Required + {"name": "admin", + "reason": ["We've seen this file many times and it leads to False positives"], + "type": "user"} + ], + "tag": { # Tag information - Only used in tag mode + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag + }, + "type": "tag" # Type of safelist hash (tag or file) } """ if len(qhash) not in [64, 40, 32]: From 4fa26086e7f06fc3afcc930dcd7f070d40455bae Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Fri, 11 Jun 2021 18:40:47 +0000 Subject: [PATCH 21/31] Compute the tag hashes directly in the API --- assemblyline_ui/api/v4/safelist.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index d9c2e871..0cdb9a15 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -1,4 +1,5 @@ +import hashlib from flask import request from assemblyline.common.isotime import now_as_iso @@ -126,6 +127,10 @@ def add_or_update_hash(**kwargs): data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) data.setdefault('hashes', {}) if data['type'] == 'tag': + hashed_value = f"{data['tag']['type']}: {data['tag']['value']}".encode('utf8') + data['hashes']['md5'] = hashlib.md5(hashed_value).hexdigest() + data['hashes']['sha1'] = hashlib.sha1(hashed_value).hexdigest() + data['hashes']['sha256'] = hashlib.sha256(hashed_value).hexdigest() data.pop('file', None) elif data['type'] == 'file': data.pop('tag', None) From 9de45c323745e4a77766d674b276e6f9e510277f Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Mon, 14 Jun 2021 19:39:19 +0000 Subject: [PATCH 22/31] Added support for safelisted tags --- assemblyline_ui/api/v4/file.py | 2 +- assemblyline_ui/api/v4/submission.py | 21 +++++++++++---------- assemblyline_ui/helper/result.py | 4 +++- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/assemblyline_ui/api/v4/file.py b/assemblyline_ui/api/v4/file.py index 954f0c58..ef7266b6 100644 --- a/assemblyline_ui/api/v4/file.py +++ b/assemblyline_ui/api/v4/file.py @@ -523,7 +523,7 @@ def get_file_results(sha256, **kwargs): # Process tags for t in sec['tags']: output["tags"].setdefault(t['type'], []) - t_item = (t['value'], h_type) + t_item = (t['value'], h_type, t['safelisted']) if t_item not in output["tags"][t['type']]: output["tags"][t['type']].append(t_item) diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index d923f330..863de303 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -183,17 +183,17 @@ def get_file_submission_results(sid, sha256, **kwargs): output["tags"].setdefault(t['type'], {}) current_htype = output["tags"][t['type']].get(t['value'], None) if not current_htype: - output["tags"][t['type']][t['value']] = h_type + output["tags"][t['type']][t['value']] = (h_type, t['safelisted']) else: if current_htype == 'malicious' or h_type == 'malicious': - output["tags"][t['type']][t['value']] = 'malicious' + output["tags"][t['type']][t['value']] = ('malicious', t['safelisted']) elif current_htype == 'suspicious' or h_type == 'suspicious': - output["tags"][t['type']][t['value']] = 'suspicious' + output["tags"][t['type']][t['value']] = ('suspicious', t['safelisted']) else: - output["tags"][t['type']][t['value']] = 'info' + output["tags"][t['type']][t['value']] = ('info', t['safelisted']) for t_type in output["tags"]: - output["tags"][t_type] = [(k, v) for k, v in output['tags'][t_type].items()] + output["tags"][t_type] = [(k, v[0], v[1]) for k, v in output['tags'][t_type].items()] output['signatures'] = list(output['signatures']) @@ -613,18 +613,19 @@ def get_summary(sid, **kwargs): output['tags'][summary_type].setdefault(t['type'], {}) current_htype = output['tags'][summary_type][t['type']].get(t['value'], None) if not current_htype: - output['tags'][summary_type][t['type']][t['value']] = t['h_type'] + output['tags'][summary_type][t['type']][t['value']] = (t['h_type'], t['safelisted']) else: if current_htype == 'malicious' or t['h_type'] == 'malicious': - output['tags'][summary_type][t['type']][t['value']] = 'malicious' + output['tags'][summary_type][t['type']][t['value']] = ('malicious', t['safelisted']) elif current_htype == 'suspicious' or t['h_type'] == 'suspicious': - output['tags'][summary_type][t['type']][t['value']] = 'suspicious' + output['tags'][summary_type][t['type']][t['value']] = ('suspicious', t['safelisted']) else: - output['tags'][summary_type][t['type']][t['value']] = 'info' + output['tags'][summary_type][t['type']][t['value']] = ('info', t['safelisted']) for summary_type in output['tags']: for t_type in output['tags'][summary_type]: - output['tags'][summary_type][t_type] = [(k, v) for k, v in output['tags'][summary_type][t_type].items()] + output['tags'][summary_type][t_type] = [(k, v[0], v[1]) + for k, v in output['tags'][summary_type][t_type].items()] return make_api_response(output) else: diff --git a/assemblyline_ui/helper/result.py b/assemblyline_ui/helper/result.py index ee128ca9..689d32e2 100644 --- a/assemblyline_ui/helper/result.py +++ b/assemblyline_ui/helper/result.py @@ -1,4 +1,5 @@ import json +from assemblyline.common.dict_utils import flatten from assemblyline_ui.config import CLASSIFICATION, LOGGER from assemblyline.common.classification import InvalidClassification @@ -58,7 +59,8 @@ def filter_sections(sections, user_classification, min_classification): pass # Changing tags to a list - section['tags'] = tag_dict_to_list(section['tags']) + section['tags'] = tag_dict_to_list(flatten(section['tags']), False) + section['tags'] += tag_dict_to_list(section.pop('safelisted_tags', {}), True) final_sections.append(section) # Telling the user a section was hidden From a32900ab0c9062467aec930e1038b1e56513f570 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Mon, 14 Jun 2021 20:00:32 +0000 Subject: [PATCH 23/31] Strip whitelisted tags from report --- assemblyline_ui/api/v4/submission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index 863de303..f9c49ff2 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -822,7 +822,7 @@ def recurse_get_names(data): name_map = recurse_get_names(tree['tree']) summary = get_or_create_summary(submission_id, submission.pop('results', []), user['classification']) - tags = summary['tags'] + tags = [t for t in summary['tags'] if not t['safelisted']] attack_matrix = summary['attack_matrix'] heuristics = summary['heuristics'] submission['classification'] = Classification.max_classification(submission['classification'], From b4d16a8809991025dbb92aa1cf597f0feb0d44fa Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Tue, 15 Jun 2021 15:25:21 +0000 Subject: [PATCH 24/31] Fix Tests --- test/conftest.py | 2 +- test/test_file.py | 1 - test/test_safelist.py | 119 +++++++++++++++++++++++++++++++++--------- 3 files changed, 96 insertions(+), 26 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d13da81f..3e4f82ff 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -148,7 +148,7 @@ def get_api_data(session, url, params=None, data=None, method="GET", raw=False, try: res_data = res.json() return res_data['api_response'] - except JSONDecodeError: + except Exception: raise APIError(f'{res.status_code}: {res.content or None}') else: try: diff --git a/test/test_file.py b/test/test_file.py index 0e65f1c2..119907b2 100644 --- a/test/test_file.py +++ b/test/test_file.py @@ -2,7 +2,6 @@ import pytest -from assemblyline.common import forge from assemblyline.common.dict_utils import unflatten from assemblyline.common.tagging import tag_list_to_dict from assemblyline.odm.models.file import File diff --git a/test/test_safelist.py b/test/test_safelist.py index 184cacb1..facf6ebb 100644 --- a/test/test_safelist.py +++ b/test/test_safelist.py @@ -1,4 +1,5 @@ +import hashlib import json import random @@ -10,12 +11,13 @@ from assemblyline.odm.randomizer import get_random_hash from conftest import APIError, get_api_data -add_hash = "10" + get_random_hash(62) +add_hash_file = "10" + get_random_hash(62) add_error_hash = "11" + get_random_hash(62) update_hash = "12" + get_random_hash(62) update_conflict_hash = "13" + get_random_hash(62) NSRL_SOURCE = { + "classification": 'TLP:W', "name": "NSRL", "reason": [ "Found as test.txt on default windows 10 CD", @@ -24,6 +26,7 @@ "type": "external"} NSRL2_SOURCE = { + "classification": 'TLP:W', "name": "NSRL2", "reason": [ "File contains only AAAAs..." @@ -31,6 +34,7 @@ "type": "external"} ADMIN_SOURCE = { + "classification": 'TLP:W', "name": "admin", "reason": [ "Generates a lot of FPs", @@ -38,6 +42,7 @@ "type": "user"} USER_SOURCE = { + "classification": 'TLP:W', "name": "user", "reason": [ "I just feel like it!", @@ -58,26 +63,28 @@ def datastore(datastore_connection): # noinspection PyUnusedLocal -def test_safelist_add(datastore, login_session): +def test_safelist_add_file(datastore, login_session): _, session, host = login_session # Generate a random safelist sl_data = { - 'fileinfo': {'md5': get_random_hash(32), - 'sha1': get_random_hash(40), - 'sha256': add_hash, - 'size': random.randint(128, 4096), - 'type': 'document/text'}, + 'hashes': {'md5': get_random_hash(32), + 'sha1': get_random_hash(40), + 'sha256': add_hash_file}, + 'file': {'name': ['file.txt'], + 'size': random.randint(128, 4096), + 'type': 'document/text'}, 'sources': [NSRL_SOURCE, ADMIN_SOURCE], + 'type': 'file' } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{add_hash}/", method="PUT", data=json.dumps(sl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' # Load inserted data from DB - ds_sl = datastore.safelist.get(add_hash, as_obj=False) + ds_sl = datastore.safelist.get(add_hash_file, as_obj=False) # Test dates added = ds_sl.pop('added', None) @@ -85,10 +92,66 @@ def test_safelist_add(datastore, login_session): assert added == updated assert added is not None and updated is not None + # Make sure tag is none + tag = ds_sl.pop('tag', {}) + assert tag is None + # Test classification classification = ds_sl.pop('classification', None) assert classification is not None + # Test enabled + enabled = ds_sl.pop('enabled', None) + assert enabled + + # Test rest + assert ds_sl == sl_data + + +def test_safelist_add_tag(datastore, login_session): + _, session, host = login_session + + tag_type = 'network.static.ip' + tag_value = '127.0.0.1' + hashed_value = f"{tag_type}: {tag_value}".encode('utf8') + + # Generate a random safelist + sl_data = { + 'hashes': {'md5': hashlib.md5(hashed_value).hexdigest(), + 'sha1': hashlib.sha1(hashed_value).hexdigest(), + 'sha256': hashlib.sha256(hashed_value).hexdigest()}, + 'tag': {'type': tag_type, + 'value': tag_value}, + 'sources': [NSRL_SOURCE, ADMIN_SOURCE], + 'type': 'tag' + } + + # Insert it and test return value + resp = get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) + assert resp['success'] + assert resp['op'] == 'add' + + # Load inserted data from DB + ds_sl = datastore.safelist.get(hashlib.sha256(hashed_value).hexdigest(), as_obj=False) + + # Test dates + added = ds_sl.pop('added', None) + updated = ds_sl.pop('updated', None) + assert added == updated + assert added is not None and updated is not None + + # Make sure file is none + file = ds_sl.pop('file', {}) + assert file is None + + # Test classification + classification = ds_sl.pop('classification', None) + assert classification is not None + + # Test enabled + enabled = ds_sl.pop('enabled', None) + assert enabled + # Test rest assert ds_sl == sl_data @@ -97,11 +160,14 @@ def test_safelist_add_invalid(datastore, login_session): _, session, host = login_session # Generate a random safelist - sl_data = {'sources': [USER_SOURCE]} + sl_data = { + 'hashes': {'sha256': add_error_hash}, + 'sources': [USER_SOURCE], + 'type': 'file'} # Insert it and test return value with pytest.raises(APIError) as conflict_exc: - get_api_data(session, f"{host}/api/v4/safelist/{add_error_hash}/", method="PUT", data=json.dumps(sl_data)) + get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) assert 'for another user' in conflict_exc.value.args[0] @@ -112,16 +178,18 @@ def test_safelist_update(datastore, login_session): # Generate a random safelist sl_data = { - 'fileinfo': {'md5': get_random_hash(32), - 'sha1': get_random_hash(40), - 'sha256': update_hash, - 'size': random.randint(128, 4096), - 'type': 'document/text'}, + 'hashes': {'md5': get_random_hash(32), + 'sha1': get_random_hash(40), + 'sha256': update_hash}, + 'file': {'name': [], + 'size': random.randint(128, 4096), + 'type': 'document/text'}, 'sources': [NSRL_SOURCE], + 'type': 'file' } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(sl_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' @@ -129,15 +197,18 @@ def test_safelist_update(datastore, login_session): ds_sl = datastore.safelist.get(update_hash, as_obj=False) # Test rest - assert {k: v for k, v in ds_sl.items() if k not in ['added', 'updated', 'classification']} == sl_data + assert {k: v for k, v in ds_sl.items() + if k not in ['added', 'updated', 'classification', 'enabled', 'tag']} == sl_data u_data = { 'classification': cl_eng.RESTRICTED, - 'sources': [NSRL2_SOURCE] + 'hashes': {'sha256': update_hash}, + 'sources': [NSRL2_SOURCE], + 'type': 'file' } # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{update_hash}/", method="PUT", data=json.dumps(u_data)) + resp = get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(u_data)) assert resp['success'] assert resp['op'] == 'update' @@ -156,10 +227,10 @@ def test_safelist_update_conflict(datastore, login_session): _, session, host = login_session # Generate a random safelist - sl_data = {'sources': [ADMIN_SOURCE]} + sl_data = {'hashes': {'sha256': update_conflict_hash}, 'file': {}, 'sources': [ADMIN_SOURCE], 'type': 'file'} # Insert it and test return value - resp = get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", + resp = get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) assert resp['success'] assert resp['op'] == 'add' @@ -167,10 +238,10 @@ def test_safelist_update_conflict(datastore, login_session): # Insert the same source with a different type sl_data['sources'][0]['type'] = 'external' with pytest.raises(APIError) as conflict_exc: - get_api_data(session, f"{host}/api/v4/safelist/{update_conflict_hash}/", + get_api_data(session, f"{host}/api/v4/safelist/", method="PUT", data=json.dumps(sl_data)) - assert 'Source type conflict' in conflict_exc.value.args[0] + assert 'has a type conflict:' in conflict_exc.value.args[0] def test_safelist_exist(datastore, login_session): From c4a34bb6c9363728edf50506eeb56d4d941ed0b1 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Tue, 15 Jun 2021 15:35:16 +0000 Subject: [PATCH 25/31] Better safelist validation --- assemblyline_ui/api/v4/safelist.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 0cdb9a15..1df18bab 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -127,13 +127,20 @@ def add_or_update_hash(**kwargs): data.setdefault('classification', CLASSIFICATION.UNRESTRICTED) data.setdefault('hashes', {}) if data['type'] == 'tag': - hashed_value = f"{data['tag']['type']}: {data['tag']['value']}".encode('utf8') + tag_data = data.get('tag', None) + if tag_data is None or 'type' not in tag_data or 'value' not in tag_data: + return make_api_response(None, "Tag data not found", 400) + + hashed_value = f"{tag_data['type']}: {tag_data['value']}".encode('utf8') data['hashes']['md5'] = hashlib.md5(hashed_value).hexdigest() data['hashes']['sha1'] = hashlib.sha1(hashed_value).hexdigest() data['hashes']['sha256'] = hashlib.sha256(hashed_value).hexdigest() data.pop('file', None) + elif data['type'] == 'file': data.pop('tag', None) + data.setdefault('file', {}) + data['added'] = data['updated'] = now_as_iso() # Find the best hash to use for the key From 2dba0109002a33a08e6431f5a76364f38e2acc74 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Tue, 15 Jun 2021 15:42:58 +0000 Subject: [PATCH 26/31] Change the API docs --- assemblyline_ui/api/v4/safelist.py | 81 +++++++++++++++++------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/assemblyline_ui/api/v4/safelist.py b/assemblyline_ui/api/v4/safelist.py index 1df18bab..8fb093f8 100644 --- a/assemblyline_ui/api/v4/safelist.py +++ b/assemblyline_ui/api/v4/safelist.py @@ -83,32 +83,35 @@ def add_or_update_hash(**kwargs): Data Block: { - "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional + "classification": "TLP:W", # Classification of the safe hash (Computed for the mix of sources) - Optional + "enabled": true, # Is the safe hash enabled or not "file": { # Information about the file - Only used in file mode "name": ["file.txt"] # Possible names for the file "size": 12345, # Size of the file "type": "document/text"}, # Type of the file }, - "hashes": { # Information about the file - At least one hash required - "md5": "123...321", # MD5 hash of the file - "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file - "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required - {"name": "NSRL", # Name of external source or user who safelisted it - Required + "hashes": { # Information about the safe hash - At least one hash required + "md5": "123...321", # MD5 hash of the safe hash + "sha1": "1234...4321", # SHA1 hash of the safe hash + "sha256": "12345....54321", # SHA256 of the safe hash + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"classification": "TLP:W", # Classification of the source (default: TLP:W) - Optional + "name": "NSRL", # Name of external source or user who safelisted it - Required "reason": [ # List of reasons why the source is safelisted - Required "Found as test.txt on default windows 10 CD", "Found as install.txt on default windows XP CD" ], "type": "external"}, # Type or source (external or user) - Required - {"name": "admin", + {"classification": "TLP:W", + "name": "admin", "reason": ["We've seen this file many times and it leads to False positives"], "type": "user"} ], "tag": { # Tag information - Only used in tag mode - "type": "network.url", # Type of tag - "value": "google.ca" # Value of the tag + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag }, - "type": "tag" # Type of safelist hash (tag or file) + "type": "tag" # Type of safelist hash (tag or file) } Result example: @@ -200,32 +203,35 @@ def add_update_many_hashes(**_): Data Block (REQUIRED): [ # List of Safe hash blocks { - "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional - "file": { # Information about the file - Only used in file mode + "classification": "TLP:W", # Classification of the safe hash (Computed for the mix of sources) - Optional + "enabled": true, # Is the safe hash enabled or not + "file": { # Information about the file - Only used in file mode "name": ["file.txt"] # Possible names for the file "size": 12345, # Size of the file "type": "document/text"}, # Type of the file }, - "hashes": { # Information about the file - At least one hash required - "md5": "123...321", # MD5 hash of the file - "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file - "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required - {"name": "NSRL", # Name of external source or user who safelisted it - Required + "hashes": { # Information about the safe hash - At least one hash required + "md5": "123...321", # MD5 hash of the safe hash + "sha1": "1234...4321", # SHA1 hash of the safe hash + "sha256": "12345....54321", # SHA256 of the safe hash + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"classification": "TLP:W", # Classification of the source (default: TLP:W) - Optional + "name": "NSRL", # Name of external source or user who safelisted it - Required "reason": [ # List of reasons why the source is safelisted - Required "Found as test.txt on default windows 10 CD", "Found as install.txt on default windows XP CD" ], - "type": "external"}, # Type or source (external or user) - Required - {"name": "admin", + "type": "external"}, # Type or source (external or user) - Required + {"classification": "TLP:W", + "name": "admin", "reason": ["We've seen this file many times and it leads to False positives"], "type": "user"} ], - "tag": { # Tag information - Only used in tag mode - "type": "network.url", # Type of tag - "value": "google.ca" # Value of the tag + "tag": { # Tag information - Only used in tag mode + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag }, - "type": "tag" # Type of safelist hash (tag or file) + "type": "tag" # Type of safelist hash (tag or file) } ... ] @@ -301,32 +307,35 @@ def check_hash_exists(qhash, **kwargs): Result example: { - "classification": "TLP:W", # Classification of the file (default: TLP:W) - Optional + "classification": "TLP:W", # Classification of the safe hash (Computed for the mix of sources) - Optional + "enabled": true, # Is the safe hash enabled or not "file": { # Information about the file - Only used in file mode "name": ["file.txt"] # Possible names for the file "size": 12345, # Size of the file "type": "document/text"}, # Type of the file }, - "hashes": { # Information about the file - At least one hash required - "md5": "123...321", # MD5 hash of the file - "sha1": "1234...4321", # SHA1 hash of the file - "sha256": "12345....54321", # SHA256 of the file - "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required - {"name": "NSRL", # Name of external source or user who safelisted it - Required + "hashes": { # Information about the safe hash - At least one hash required + "md5": "123...321", # MD5 hash of the safe hash + "sha1": "1234...4321", # SHA1 hash of the safe hash + "sha256": "12345....54321", # SHA256 of the safe hash + "sources": [ # List of sources for why the file is safelisted, dedupped on name - Required + {"classification": "TLP:W", # Classification of the source (default: TLP:W) - Optional + "name": "NSRL", # Name of external source or user who safelisted it - Required "reason": [ # List of reasons why the source is safelisted - Required "Found as test.txt on default windows 10 CD", "Found as install.txt on default windows XP CD" ], "type": "external"}, # Type or source (external or user) - Required - {"name": "admin", + {"classification": "TLP:W", + "name": "admin", "reason": ["We've seen this file many times and it leads to False positives"], "type": "user"} ], "tag": { # Tag information - Only used in tag mode - "type": "network.url", # Type of tag - "value": "google.ca" # Value of the tag + "type": "network.url", # Type of tag + "value": "google.ca" # Value of the tag }, - "type": "tag" # Type of safelist hash (tag or file) + "type": "tag" # Type of safelist hash (tag or file) } """ if len(qhash) not in [64, 40, 32]: From 109d2758f50ce235e8b04a40ee5ed4464e99809c Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Tue, 15 Jun 2021 19:42:52 +0000 Subject: [PATCH 27/31] First stab at tag_safelist APIs --- assemblyline_ui/api/v4/tag_safelist.py | 148 +++++++++++++++++++++++++ assemblyline_ui/app.py | 6 +- 2 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 assemblyline_ui/api/v4/tag_safelist.py diff --git a/assemblyline_ui/api/v4/tag_safelist.py b/assemblyline_ui/api/v4/tag_safelist.py new file mode 100644 index 00000000..ca71e263 --- /dev/null +++ b/assemblyline_ui/api/v4/tag_safelist.py @@ -0,0 +1,148 @@ + +from assemblyline.datastore.exceptions import SearchException +from flask import request + +from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint +from assemblyline_ui.config import STORAGE + +SUB_API = 'tag_safelist' +tag_safelist_api = make_subapi_blueprint(SUB_API, api_version=4) +tag_safelist_api._doc = "Perform operations on safelisted hashes" + + +@tag_safelist_api.route("/", methods=["PUT"]) +@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) +def add_to_tag_safelist(**kwargs): + """ + Add to system tag safelist + + Arguments: + None + + Data Block: + { + + } + + Result example: + { + "success": true, # Was the tag safelist successfully added + } + """ + # Load data + data = request.json + if not data: + return make_api_response({}, "No data provided", 400) + + return make_api_response({"success": True}) + + +@tag_safelist_api.route("//", methods=["GET"]) +@api_login(require_type=['admin'], required_priv=['R']) +def get_tag_safelist(tag_id, **_): + """ + Get the details of a tag safelist item + + Variables: + tag_id => Id of the tag to get + + Arguments: + None + + Data Block: + None + + Result example: + { + TBD + } + """ + return make_api_response(STORAGE.tag_safelist.get(tag_id, as_obj=False)) + + +@tag_safelist_api.route("/list/", methods=["GET"]) +@api_login(require_type=['admin']) +def list_tag_safelist(**_): + """ + List all tag safelist items (per page) + + Variables: + None + + Arguments: + offset => Offset at which we start giving tag safelists + query => Query to apply to the tag safelist + rows => Numbers of tags to return + sort => Sort order + + Data Block: + None + + Result example: + {"total": 201, # Total tags found + "offset": 0, # Offset in the tag safelist + "count": 100, # Number of tags returned + "items": [] # List of tag safelist blocks + } + """ + offset = int(request.args.get('offset', 0)) + rows = int(request.args.get('rows', 100)) + query = request.args.get('query', "id:*") or "id:*" + sort = request.args.get('sort', "created desc") + + try: + return make_api_response(STORAGE.tag_safelist.search(query, offset=offset, rows=rows, as_obj=False, + sort=sort)) + except SearchException as e: + return make_api_response("", f"The specified search query is not valid. ({e})", 400) + + +@tag_safelist_api.route("//", methods=["POST"]) +@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) +def update_tag_in_safelist(tag_id, **kwargs): + """ + Update a system tag safelist + + Arguments: + None + + Data Block: + { + + } + + Result example: + { + "success": true, # Was the tag safelist successfully added + } + """ + # Load data + data = request.json + if not data: + return make_api_response({}, "No data provided", 400) + + return make_api_response({"success": True}) + + +@tag_safelist_api.route("//", methods=["DELETE"]) +@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) +def delete_hash(tag_id, **_): + """ + Delete a system tag from the safelist + + Variables: + tag_id => tag id to delete + + Arguments: + None + + Data Block: + None + + API call example: + DELETE /api/v1/tag_safelist/123456...654321/ + + Result example: + {"success": True} + """ + return make_api_response({'success': STORAGE.safelist.delete(tag_id)}) diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index b7acb57b..7d690133 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -20,6 +20,7 @@ from assemblyline_ui.api.v4.ingest import ingest_api from assemblyline_ui.api.v4.live import live_api from assemblyline_ui.api.v4.result import result_api +from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.search import search_api from assemblyline_ui.api.v4.service import service_api from assemblyline_ui.api.v4.signature import signature_api @@ -28,7 +29,7 @@ from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api -from assemblyline_ui.api.v4.safelist import safelist_api +from assemblyline_ui.api.v4.tag_safelist import tag_safelist_api from assemblyline_ui.api.v4.workflow import workflow_api from assemblyline_ui.error import errors from assemblyline_ui.healthz import healthz @@ -70,15 +71,16 @@ app.register_blueprint(ingest_api) app.register_blueprint(live_api) app.register_blueprint(result_api) +app.register_blueprint(safelist_api) app.register_blueprint(search_api) app.register_blueprint(service_api) app.register_blueprint(signature_api) app.register_blueprint(submission_api) app.register_blueprint(submit_api) +app.register_blueprint(tag_safelist_api) app.register_blueprint(ui_api) app.register_blueprint(user_api) app.register_blueprint(webauthn_api) -app.register_blueprint(safelist_api) app.register_blueprint(workflow_api) From fa0693cd499e8a6bbe88fab9e50bda8de9efa206 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 16 Jun 2021 12:22:14 +0000 Subject: [PATCH 28/31] Revert "First stab at tag_safelist APIs" This reverts commit 109d2758f50ce235e8b04a40ee5ed4464e99809c. --- assemblyline_ui/api/v4/tag_safelist.py | 148 ------------------------- assemblyline_ui/app.py | 6 +- 2 files changed, 2 insertions(+), 152 deletions(-) delete mode 100644 assemblyline_ui/api/v4/tag_safelist.py diff --git a/assemblyline_ui/api/v4/tag_safelist.py b/assemblyline_ui/api/v4/tag_safelist.py deleted file mode 100644 index ca71e263..00000000 --- a/assemblyline_ui/api/v4/tag_safelist.py +++ /dev/null @@ -1,148 +0,0 @@ - -from assemblyline.datastore.exceptions import SearchException -from flask import request - -from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint -from assemblyline_ui.config import STORAGE - -SUB_API = 'tag_safelist' -tag_safelist_api = make_subapi_blueprint(SUB_API, api_version=4) -tag_safelist_api._doc = "Perform operations on safelisted hashes" - - -@tag_safelist_api.route("/", methods=["PUT"]) -@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) -def add_to_tag_safelist(**kwargs): - """ - Add to system tag safelist - - Arguments: - None - - Data Block: - { - - } - - Result example: - { - "success": true, # Was the tag safelist successfully added - } - """ - # Load data - data = request.json - if not data: - return make_api_response({}, "No data provided", 400) - - return make_api_response({"success": True}) - - -@tag_safelist_api.route("//", methods=["GET"]) -@api_login(require_type=['admin'], required_priv=['R']) -def get_tag_safelist(tag_id, **_): - """ - Get the details of a tag safelist item - - Variables: - tag_id => Id of the tag to get - - Arguments: - None - - Data Block: - None - - Result example: - { - TBD - } - """ - return make_api_response(STORAGE.tag_safelist.get(tag_id, as_obj=False)) - - -@tag_safelist_api.route("/list/", methods=["GET"]) -@api_login(require_type=['admin']) -def list_tag_safelist(**_): - """ - List all tag safelist items (per page) - - Variables: - None - - Arguments: - offset => Offset at which we start giving tag safelists - query => Query to apply to the tag safelist - rows => Numbers of tags to return - sort => Sort order - - Data Block: - None - - Result example: - {"total": 201, # Total tags found - "offset": 0, # Offset in the tag safelist - "count": 100, # Number of tags returned - "items": [] # List of tag safelist blocks - } - """ - offset = int(request.args.get('offset', 0)) - rows = int(request.args.get('rows', 100)) - query = request.args.get('query', "id:*") or "id:*" - sort = request.args.get('sort', "created desc") - - try: - return make_api_response(STORAGE.tag_safelist.search(query, offset=offset, rows=rows, as_obj=False, - sort=sort)) - except SearchException as e: - return make_api_response("", f"The specified search query is not valid. ({e})", 400) - - -@tag_safelist_api.route("//", methods=["POST"]) -@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) -def update_tag_in_safelist(tag_id, **kwargs): - """ - Update a system tag safelist - - Arguments: - None - - Data Block: - { - - } - - Result example: - { - "success": true, # Was the tag safelist successfully added - } - """ - # Load data - data = request.json - if not data: - return make_api_response({}, "No data provided", 400) - - return make_api_response({"success": True}) - - -@tag_safelist_api.route("//", methods=["DELETE"]) -@api_login(require_type=['admin'], allow_readonly=False, required_priv=["W"]) -def delete_hash(tag_id, **_): - """ - Delete a system tag from the safelist - - Variables: - tag_id => tag id to delete - - Arguments: - None - - Data Block: - None - - API call example: - DELETE /api/v1/tag_safelist/123456...654321/ - - Result example: - {"success": True} - """ - return make_api_response({'success': STORAGE.safelist.delete(tag_id)}) diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index 7d690133..b7acb57b 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -20,7 +20,6 @@ from assemblyline_ui.api.v4.ingest import ingest_api from assemblyline_ui.api.v4.live import live_api from assemblyline_ui.api.v4.result import result_api -from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.search import search_api from assemblyline_ui.api.v4.service import service_api from assemblyline_ui.api.v4.signature import signature_api @@ -29,7 +28,7 @@ from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api -from assemblyline_ui.api.v4.tag_safelist import tag_safelist_api +from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.workflow import workflow_api from assemblyline_ui.error import errors from assemblyline_ui.healthz import healthz @@ -71,16 +70,15 @@ app.register_blueprint(ingest_api) app.register_blueprint(live_api) app.register_blueprint(result_api) -app.register_blueprint(safelist_api) app.register_blueprint(search_api) app.register_blueprint(service_api) app.register_blueprint(signature_api) app.register_blueprint(submission_api) app.register_blueprint(submit_api) -app.register_blueprint(tag_safelist_api) app.register_blueprint(ui_api) app.register_blueprint(user_api) app.register_blueprint(webauthn_api) +app.register_blueprint(safelist_api) app.register_blueprint(workflow_api) From 06635a18badbab246303b688c06d97a5f8d8f83f Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Wed, 16 Jun 2021 19:54:03 +0000 Subject: [PATCH 29/31] Added APIs for the tag_safelister --- assemblyline_ui/api/v4/admin.py | 93 +++++++++++++++++++++++++++++++++ assemblyline_ui/app.py | 6 ++- 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 assemblyline_ui/api/v4/admin.py diff --git a/assemblyline_ui/api/v4/admin.py b/assemblyline_ui/api/v4/admin.py new file mode 100644 index 00000000..96e7cc4e --- /dev/null +++ b/assemblyline_ui/api/v4/admin.py @@ -0,0 +1,93 @@ + +from flask import request + +from assemblyline.common import forge +from assemblyline.common.str_utils import safe_str +from assemblyline.odm.models.tagging import Tagging + +from assemblyline_ui.config import STORAGE +from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint +import yaml + + +Classification = forge.get_classification() +config = forge.get_config() + +SUB_API = 'admin' +admin_api = make_subapi_blueprint(SUB_API, api_version=4) +admin_api._doc = "Perform administrative actions" + +ADMIN_FILE_TTL = 60 * 60 * 24 * 365 * 100 # Just keep the file for 100 years... + + +@admin_api.route("/tag_safelist/", methods=["GET"]) +@api_login(require_type=['admin'], required_priv=['R']) +def get_tag_safelist(**_): + """ + Get the current tag_safelist + + Variables: + None + + Arguments: + None + + Data Block: + None + + Result example: + + """ + with forge.get_cachestore('assemblyline_ui', config=config, datastore=STORAGE) as cache: + tag_safelist_yml = cache.get('tag_safelist_yml') + if not tag_safelist_yml: + yml_data = forge.get_tag_safelist_data() + if yml_data: + return make_api_response(yaml.safe_dump(yml_data)) + + return make_api_response(None, "Could not find the tag_safelist.yml file", 404) + + return make_api_response(safe_str(tag_safelist_yml)) + + +@admin_api.route("/tag_safelist/", methods=["PUT"]) +@api_login(require_type=['admin'], allow_readonly=False, required_priv=['W']) +def put_tag_safelist(**_): + """ + Save a new version of the tag_safelist file + + Variables: + None + + Arguments: + None + + Data Block: + + + Result example: + {"success": true} + """ + tag_safelist_yml = request.json + + try: + yml_data = yaml.safe_load(tag_safelist_yml) + for key in yml_data.keys(): + if key not in ['match', 'regex']: + raise Exception('Invalid key found.') + + fields = Tagging.flat_fields() + for tag_type in ['match', 'regex']: + for key, value in yml_data[tag_type].items(): + if key not in fields: + raise Exception(f'{key} is not a valid tag type') + + if not isinstance(value, list): + raise Exception(f'Value for {key} should be a list of strings') + except Exception as e: + return make_api_response(None, f"Invalid tag_safelist.yml file submitted: {str(e)}", 400) + + with forge.get_cachestore('assemblyline_ui', config=config, datastore=STORAGE) as cache: + cache.save('tag_safelist_yml', tag_safelist_yml.encode('utf-8'), ttl=ADMIN_FILE_TTL, force=True) + + return make_api_response({'success': True}) diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index b7acb57b..54ff7997 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -9,6 +9,7 @@ from assemblyline_ui.api.base import api from assemblyline_ui.api.v4 import apiv4 +from assemblyline_ui.api.v4.admin import admin_api from assemblyline_ui.api.v4.alert import alert_api from assemblyline_ui.api.v4.authentication import auth_api from assemblyline_ui.api.v4.bundle import bundle_api @@ -20,6 +21,7 @@ from assemblyline_ui.api.v4.ingest import ingest_api from assemblyline_ui.api.v4.live import live_api from assemblyline_ui.api.v4.result import result_api +from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.search import search_api from assemblyline_ui.api.v4.service import service_api from assemblyline_ui.api.v4.signature import signature_api @@ -28,7 +30,6 @@ from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api -from assemblyline_ui.api.v4.safelist import safelist_api from assemblyline_ui.api.v4.workflow import workflow_api from assemblyline_ui.error import errors from assemblyline_ui.healthz import healthz @@ -58,8 +59,9 @@ app.register_blueprint(healthz) app.register_blueprint(api) app.register_blueprint(apiv4) -app.register_blueprint(auth_api) +app.register_blueprint(admin_api) app.register_blueprint(alert_api) +app.register_blueprint(auth_api) app.register_blueprint(bundle_api) app.register_blueprint(errors) app.register_blueprint(error_api) From 741d05a7dd98cf3bcbfd5f329cae25fc8fcdbfcd Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 17 Jun 2021 14:58:03 +0000 Subject: [PATCH 30/31] Change cache name --- assemblyline_ui/api/v4/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/api/v4/admin.py b/assemblyline_ui/api/v4/admin.py index 96e7cc4e..a38c28a2 100644 --- a/assemblyline_ui/api/v4/admin.py +++ b/assemblyline_ui/api/v4/admin.py @@ -38,7 +38,7 @@ def get_tag_safelist(**_): Result example: """ - with forge.get_cachestore('assemblyline_ui', config=config, datastore=STORAGE) as cache: + with forge.get_cachestore('system', config=config, datastore=STORAGE) as cache: tag_safelist_yml = cache.get('tag_safelist_yml') if not tag_safelist_yml: yml_data = forge.get_tag_safelist_data() @@ -87,7 +87,7 @@ def put_tag_safelist(**_): except Exception as e: return make_api_response(None, f"Invalid tag_safelist.yml file submitted: {str(e)}", 400) - with forge.get_cachestore('assemblyline_ui', config=config, datastore=STORAGE) as cache: + with forge.get_cachestore('system', config=config, datastore=STORAGE) as cache: cache.save('tag_safelist_yml', tag_safelist_yml.encode('utf-8'), ttl=ADMIN_FILE_TTL, force=True) return make_api_response({'success': True}) From 932b9aa292ea7b82a29b90d969810e85dd1f24e7 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Thu, 17 Jun 2021 18:25:12 +0000 Subject: [PATCH 31/31] move API to system --- assemblyline_ui/api/v4/{admin.py => system.py} | 10 +++++----- assemblyline_ui/app.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) rename assemblyline_ui/api/v4/{admin.py => system.py} (91%) diff --git a/assemblyline_ui/api/v4/admin.py b/assemblyline_ui/api/v4/system.py similarity index 91% rename from assemblyline_ui/api/v4/admin.py rename to assemblyline_ui/api/v4/system.py index a38c28a2..fce18fd1 100644 --- a/assemblyline_ui/api/v4/admin.py +++ b/assemblyline_ui/api/v4/system.py @@ -13,14 +13,14 @@ Classification = forge.get_classification() config = forge.get_config() -SUB_API = 'admin' -admin_api = make_subapi_blueprint(SUB_API, api_version=4) -admin_api._doc = "Perform administrative actions" +SUB_API = 'system' +system_api = make_subapi_blueprint(SUB_API, api_version=4) +system_api._doc = "Perform system actions" ADMIN_FILE_TTL = 60 * 60 * 24 * 365 * 100 # Just keep the file for 100 years... -@admin_api.route("/tag_safelist/", methods=["GET"]) +@system_api.route("/tag_safelist/", methods=["GET"]) @api_login(require_type=['admin'], required_priv=['R']) def get_tag_safelist(**_): """ @@ -50,7 +50,7 @@ def get_tag_safelist(**_): return make_api_response(safe_str(tag_safelist_yml)) -@admin_api.route("/tag_safelist/", methods=["PUT"]) +@system_api.route("/tag_safelist/", methods=["PUT"]) @api_login(require_type=['admin'], allow_readonly=False, required_priv=['W']) def put_tag_safelist(**_): """ diff --git a/assemblyline_ui/app.py b/assemblyline_ui/app.py index 54ff7997..57497b74 100644 --- a/assemblyline_ui/app.py +++ b/assemblyline_ui/app.py @@ -9,7 +9,6 @@ from assemblyline_ui.api.base import api from assemblyline_ui.api.v4 import apiv4 -from assemblyline_ui.api.v4.admin import admin_api from assemblyline_ui.api.v4.alert import alert_api from assemblyline_ui.api.v4.authentication import auth_api from assemblyline_ui.api.v4.bundle import bundle_api @@ -27,6 +26,7 @@ from assemblyline_ui.api.v4.signature import signature_api from assemblyline_ui.api.v4.submission import submission_api from assemblyline_ui.api.v4.submit import submit_api +from assemblyline_ui.api.v4.system import system_api from assemblyline_ui.api.v4.ui import ui_api from assemblyline_ui.api.v4.user import user_api from assemblyline_ui.api.v4.webauthn import webauthn_api @@ -59,7 +59,6 @@ app.register_blueprint(healthz) app.register_blueprint(api) app.register_blueprint(apiv4) -app.register_blueprint(admin_api) app.register_blueprint(alert_api) app.register_blueprint(auth_api) app.register_blueprint(bundle_api) @@ -77,6 +76,7 @@ app.register_blueprint(signature_api) app.register_blueprint(submission_api) app.register_blueprint(submit_api) +app.register_blueprint(system_api) app.register_blueprint(ui_api) app.register_blueprint(user_api) app.register_blueprint(webauthn_api)