From 22e473d05a3351d74efda8ab0e16160196a58385 Mon Sep 17 00:00:00 2001 From: Eric Fossum Date: Wed, 13 Dec 2023 04:44:40 +0000 Subject: [PATCH 1/5] Swapping os.path with Path in check version. Should help with delete logic. --- htpclient/binarydownload.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/htpclient/binarydownload.py b/htpclient/binarydownload.py index d6a487e..6736078 100644 --- a/htpclient/binarydownload.py +++ b/htpclient/binarydownload.py @@ -28,8 +28,7 @@ def get_version(self): def check_client_version(self): if self.args.disable_update: return - if os.path.isfile("old.zip"): - os.unlink("old.zip") # cleanup old version + Path("old.zip").unlink(missing_ok=True) # cleanup old version query = copy_and_set_token(dict_checkVersion, self.config.get_value('token')) query['version'] = Initialize.get_version_number() req = JsonRequest(query) @@ -47,17 +46,14 @@ def check_client_version(self): logging.warning("Got empty URL for client update!") else: logging.info("New client version available!") - if os.path.isfile("update.zip"): - os.unlink("update.zip") + Path("update.zip").unlink(missing_ok=True) Download.download(url, "update.zip") if os.path.isfile("update.zip") and os.path.getsize("update.zip"): - if os.path.isfile("old.zip"): - os.unlink("old.zip") - os.rename("hashtopolis.zip", "old.zip") - os.rename("update.zip", "hashtopolis.zip") + Path("old.zip").unlink(missing_ok=True) + Path("hashtopolis.zip").unlink(missing_ok=True) + Path("update.zip").rename("hashtopolis.zip") logging.info("Update received, restarting client...") - if os.path.exists("lock.pid"): - os.unlink("lock.pid") + Path("lock.pid").unlink(missing_ok=True) os.execl(sys.executable, sys.executable, "hashtopolis.zip") exit(0) @@ -133,7 +129,7 @@ def check_prince(self): os.rmdir("temp") logging.debug("PRINCE downloaded and extracted") return True - + def check_preprocessor(self, task): logging.debug("Checking if requested preprocessor is present...") path = Path(self.config.get_value('preprocessors-path'), str(task.get_task()['preprocessor'])) @@ -208,7 +204,7 @@ def check_version(self, cracker_id): # Linux cmd = f"./7zr{Initialize.get_os_extension()} x -o'{temp_folder}' '{zip_file}'" os.system(cmd) - + # Clean up 7zip os.unlink(zip_file) From 1e8e01232c6b00f48c4fff7390332309640eacb8 Mon Sep 17 00:00:00 2001 From: Eric Fossum Date: Wed, 13 Dec 2023 05:00:29 +0000 Subject: [PATCH 2/5] Remove increment options from benchmark execution. --- htpclient/hashcat_cracker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/htpclient/hashcat_cracker.py b/htpclient/hashcat_cracker.py index c656496..b9f5c1c 100644 --- a/htpclient/hashcat_cracker.py +++ b/htpclient/hashcat_cracker.py @@ -5,8 +5,8 @@ from pathlib import Path from time import sleep from queue import Queue, Empty +import re from threading import Thread, Lock - import time from htpclient.config import Config @@ -651,6 +651,9 @@ def run_speed_benchmark(self, task): # Replace #HL# with the real hashlist attackcmd = attackcmd.replace(task['hashlistAlias'], f'"{hashlist_path}"') + attackcmd = re.sub(r'--increment(\s+|$)', '', attackcmd) + attackcmd = re.sub(r'--increment-(max|min)(=\S+|\s+\S+)?\s*', '', attackcmd) + attackcmd = attackcmd.replace('--', f'"{hashlist_path}"') args.append(attackcmd) args.append(task['cmdpars']) From 5392726ed185d4bfdf28ec7108b3d3e9f5898df6 Mon Sep 17 00:00:00 2001 From: Eric Fossum Date: Fri, 15 Dec 2023 21:26:36 +0000 Subject: [PATCH 3/5] Added ignore cert option to cmd line. --- __main__.py | 5 +++-- htpclient/initialize.py | 6 +++--- htpclient/jsonRequest.py | 8 ++++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/__main__.py b/__main__.py index 83bfe3c..6933ba3 100644 --- a/__main__.py +++ b/__main__.py @@ -300,7 +300,7 @@ def de_register(): if ans is None: logging.error("De-registration failed!") elif ans['response'] != 'SUCCESS': - logging.error("Error on de-registration: " + str(ans)) + logging.error("Error on de-registration: %s", str(ans)) else: logging.info("Successfully de-registered!") # cleanup @@ -334,6 +334,7 @@ def de_register(): parser.add_argument('--preprocessors-path', type=str, required=False, help='Use given folder path as preprocessors location') parser.add_argument('--zaps-path', type=str, required=False, help='Use given folder path as zaps location') parser.add_argument('--cpu-only', action='store_true', help='Force client to register as CPU only and also only reading out CPU information') + parser.add_argument('--ignore-cert', action='store_true', help="Ignore the validity of the server's certificate.") args = parser.parse_args() if args.version: @@ -370,7 +371,7 @@ def de_register(): logging.info("Ignoring lock.pid file because PID is not existent anymore or not running python!") # create lock file - with open("lock.pid", 'w') as f: + with open("lock.pid", 'w', encoding=sys.getdefaultencoding()) as f: f.write(str(os.getpid())) f.close() diff --git a/htpclient/initialize.py b/htpclient/initialize.py index 214e7d3..20591d2 100644 --- a/htpclient/initialize.py +++ b/htpclient/initialize.py @@ -161,7 +161,7 @@ def __check_token(self, args): query['cpu-only'] = True req = JsonRequest(query) - ans = req.execute() + ans = req.execute(args.ignore_cert) if ans is None: logging.error("Request failed!") self.__check_token(args) @@ -181,7 +181,7 @@ def __check_cert(self, args): cert = os.path.abspath(args.cert) logging.debug("Setting cert to: " + cert) self.config.set_value('cert', cert) - + if cert is not None: Session().s.cert = cert logging.debug("Configuration session cert to: " + cert) @@ -210,7 +210,7 @@ def __check_url(self, args): self.__check_url(args) else: logging.debug("Connection test successful!") - + if args.cpu_only is not None and args.cpu_only: logging.debug("Setting agent to be CPU only..") self.config.set_value('cpu-only', True) diff --git a/htpclient/jsonRequest.py b/htpclient/jsonRequest.py index 1d68e45..c8155a6 100644 --- a/htpclient/jsonRequest.py +++ b/htpclient/jsonRequest.py @@ -10,10 +10,14 @@ def __init__(self, data): self.config = Config() self.session = Session().s - def execute(self): + def execute(self, ignore_certificate: bool = True): try: logging.debug(self.data) - r = self.session.post(self.config.get_value('url'), json=self.data, timeout=30) + r = self.session.post( + self.config.get_value('url'), + json=self.data, + timeout=30, + verify=not ignore_certificate) if r.status_code != 200: logging.error("Status code from server: " + str(r.status_code)) return None From f4936632abbab2dfc7efa1c6b2975acc086227af Mon Sep 17 00:00:00 2001 From: Eric Fossum Date: Fri, 15 Dec 2023 21:55:12 +0000 Subject: [PATCH 4/5] Allow redirects for reverse proxy. --- htpclient/download.py | 7 ++-- htpclient/hashcat_cracker.py | 76 ++++++++++++++++++------------------ htpclient/jsonRequest.py | 8 ++-- htpclient/session.py | 9 ++++- tests/hashtopolis.py | 30 +++++++------- 5 files changed, 67 insertions(+), 63 deletions(-) diff --git a/htpclient/download.py b/htpclient/download.py index 36bbfd3..8b81f39 100644 --- a/htpclient/download.py +++ b/htpclient/download.py @@ -1,11 +1,10 @@ import logging +import os +import sys from time import sleep import requests -import sys -import os -from htpclient.initialize import Initialize from htpclient.session import Session @@ -17,7 +16,7 @@ def download(url, output, no_header=False): # Check header if not no_header: - head = session.head(url) + head = session.head(url, allow_redirects=True) # not sure if we only should allow 200/301/302, but then it's present for sure if head.status_code not in [200, 301, 302]: logging.error("File download header reported wrong status code: " + str(head.status_code)) diff --git a/htpclient/hashcat_cracker.py b/htpclient/hashcat_cracker.py index c656496..8db69aa 100644 --- a/htpclient/hashcat_cracker.py +++ b/htpclient/hashcat_cracker.py @@ -1,18 +1,18 @@ -import string import logging -import subprocess -import psutil +import os from pathlib import Path -from time import sleep +import psutil from queue import Queue, Empty +import string +import subprocess from threading import Thread, Lock - import time + from htpclient.config import Config from htpclient.hashcat_status import HashcatStatus from htpclient.initialize import Initialize -from htpclient.jsonRequest import JsonRequest, os +from htpclient.jsonRequest import JsonRequest from htpclient.helpers import send_error, update_files, kill_hashcat, get_bit, print_speed, get_rules_and_hl, get_wordlist, escape_ansi from htpclient.dicts import * @@ -43,13 +43,13 @@ def __init__(self, cracker_id, binary_download): self.callPath = f"'./{self.executable_name}'" cmd = [str(self.executable_path), "--version"] - + try: logging.debug(f"CALL: {' '.join(cmd)}") output = subprocess.check_output(cmd, cwd=self.cracker_path) except subprocess.CalledProcessError as e: logging.error("Error during version detection: " + str(e)) - sleep(5) + time.sleep(5) self.version_string = output.decode().replace('v', '') self.lock = Lock() @@ -103,27 +103,27 @@ def build_command(self, task, chunk): args.append('-p "\t"') args.append(f"-s {chunk['skip']}") args.append(f"-l {chunk['length']}") - + if 'useBrain' in task and task['useBrain']: # when using brain we set the according parameters args.append('--brain-client') args.append(f"--brain-host {task['brainHost']}") args.append(f"--brain-port {task['brainPort']}") args.append(f"--brain-password {task['brainPass']}") - + if 'brainFeatures' in task: args.append(f"--brain-client-features {task['brainFeatures']}") else: # remove should only be used if we run without brain args.append('--potfile-disable') args.append('--remove') args.append(f"--remove-timer={task['statustimer']}") - + files = update_files(task['attackcmd']) files = files.replace(task['hashlistAlias'], f'"{hashlist_file}"') args.append(files) args.append(task['cmdpars']) - - + + full_cmd = ' '.join(args) full_cmd = f'{self.callPath} {full_cmd}' @@ -187,7 +187,7 @@ def build_preprocessor_command(self, task, chunk, preprocessor): skip_length = chunk['skip'] + chunk['length'] pre_args.append(f"| head -n {skip_length}") pre_args.append(f"| tail -n {chunk['length']}") - + zaps_file = Path(self.config.get_value('zaps-path'), f"hashlist_{task['hashlistId']}") output_file = Path(self.config.get_value('hashlists-path'), f"{task['hashlistId']}.out") hashlist_file = Path(self.config.get_value('hashlists-path'), str(task['hashlistId'])) @@ -236,23 +236,23 @@ def run_chunk(self, task, chunk, preprocessor): full_cmd = self.build_command(task, chunk) self.statusCount = 0 self.wasStopped = False - + # Set paths outfile_path = Path(self.config.get_value('hashlists-path'), f"{task['hashlistId']}.out") outfile_backup_path = Path(self.config.get_value('hashlists-path'), f"{task['hashlistId']}_{time.time()}.out") zapfile_path = Path(self.config.get_value('zaps-path'), f"hashlist_{task['hashlistId']}") - + # clear old found file - earlier we deleted them, but just in case, we just move it to a unique filename if configured so if os.path.exists(outfile_path): if self.config.get_value('outfile-history'): os.rename(outfile_path, outfile_backup_path) else: os.unlink(outfile_path) - + # create zap folder if not os.path.exists(zapfile_path): os.mkdir(zapfile_path) - + # Call command logging.debug("CALL: " + full_cmd) if Initialize.get_os() != 1: @@ -393,7 +393,7 @@ def run_loop(self, proc, chunk, task): ans = req.execute() if ans is None: logging.error("Failed to send solve!") - sleep(1) + time.sleep(1) elif ans['response'] != 'SUCCESS': self.wasStopped = True logging.error("Error from server on solve: " + str(ans)) @@ -401,7 +401,7 @@ def run_loop(self, proc, chunk, task): kill_hashcat(proc.pid, Initialize.get_os()) except ProcessLookupError: pass - sleep(5) + time.sleep(5) return elif 'agent' in ans.keys() and ans['agent'] == 'stop': # server set agent to stop @@ -411,7 +411,7 @@ def run_loop(self, proc, chunk, task): kill_hashcat(proc.pid, Initialize.get_os()) except ProcessLookupError: pass - sleep(5) + time.sleep(5) return else: cracks_count = len(self.cracks) @@ -436,7 +436,7 @@ def run_loop(self, proc, chunk, task): if msg and str(msg) != '^C': # this is maybe not the fanciest way, but as ctrl+c is sent to the underlying process it reports it to stderr logging.error("HC error: " + msg) send_error(msg, self.config.get_value('token'), task['taskId'], chunk['chunkId']) - sleep(0.1) # we set a minimal sleep to avoid overreaction of the client sending a huge number of errors, but it should not be slowed down too much, in case the errors are not critical and the agent can continue + time.sleep(0.1) # we set a minimal sleep to avoid overreaction of the client sending a huge number of errors, but it should not be slowed down too much, in case the errors are not critical and the agent can continue def measure_keyspace(self, task, chunk): if 'usePrince' in task.get_task() and task.get_task()['usePrince']: @@ -446,9 +446,9 @@ def measure_keyspace(self, task, chunk): task = task.get_task() # TODO: refactor this to be better code files = update_files(task['attackcmd']) files = files.replace(task['hashlistAlias'] + " ", "") - + full_cmd = f"{self.callPath} --keyspace --quiet {files} {task['cmdpars']}" - + if 'useBrain' in task and task['useBrain']: full_cmd = f"{full_cmd} -S" @@ -459,7 +459,7 @@ def measure_keyspace(self, task, chunk): except subprocess.CalledProcessError as e: logging.error("Error during keyspace measure: " + str(e) + " Output: " + output.decode(encoding='utf-8')) send_error("Keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) - sleep(5) + time.sleep(5) return False output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") ks = 0 @@ -489,7 +489,7 @@ def prince_keyspace(self, task, chunk): except subprocess.CalledProcessError: logging.error("Error during PRINCE keyspace measure") send_error("PRINCE keyspace measure failed!", self.config.get_value('token'), task['taskId'], None) - sleep(5) + time.sleep(5) return False output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") keyspace = "0" @@ -513,14 +513,14 @@ def preprocessor_keyspace(self, task, chunk): if not os.path.isfile(binary): split = binary.split(".") binary = '.'.join(split[:-1]) + get_bit() + "." + split[-1] - + if Initialize.get_os() == 1: # Windows binary = f'"{binary}"' else: # Mac / Linux binary = f'"./{binary}"' - + args = [] args.append(preprocessor['keyspaceCommand']) args.append(update_files(task.get_task()['preprocessorCommand'])) @@ -534,7 +534,7 @@ def preprocessor_keyspace(self, task, chunk): except subprocess.CalledProcessError: logging.error("Error during preprocessor keyspace measure") send_error("Preprocessor keyspace measure failed!", self.config.get_value('token'), task.get_task()['taskId'], None) - sleep(5) + time.sleep(5) return False output = output.decode(encoding='utf-8').replace("\r\n", "\n").split("\n") keyspace = "0" @@ -556,15 +556,15 @@ def run_benchmark(self, task): args.append('--machine-readable') args.append('--quiet') args.append(f"--runtime={task['bench']}") - + args.append('--restore-disable') args.append('--potfile-disable') args.append('--session=hashtopolis') args.append('-p') args.append('"\t"') - - - + + + hashlist_path = Path(self.config.get_value('hashlists-path'), str(task['hashlistId'])) hashlist_out_path = Path(self.config.get_value('hashlists-path'), f"{str(task['hashlistId'])}.out") @@ -579,7 +579,7 @@ def run_benchmark(self, task): full_cmd = ' '.join(args) full_cmd = f"{self.callPath} {full_cmd}" - + logging.debug(f"CALL: {full_cmd}") proc = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cracker_path) output, error = proc.communicate() @@ -626,13 +626,13 @@ def run_speed_benchmark(self, task): args.append('--machine-readable') args.append('--quiet') args.append('--progress-only') - + args.append('--restore-disable') args.append('--potfile-disable') args.append('--session=hashtopolis') args.append('-p') args.append('"\t"') - + hashlist_path = Path(self.config.get_value('hashlists-path'), str(task['hashlistId'])) hashlist_out_path = Path(self.config.get_value('hashlists-path'), f"{str(task['hashlistId'])}.out") @@ -640,7 +640,7 @@ def run_speed_benchmark(self, task): attackcmd = get_rules_and_hl(update_files(task['attackcmd'])) # Replace #HL# with the real hashlist attackcmd = attackcmd.replace(task['hashlistAlias'], f'"{hashlist_path}"') - + args.append(attackcmd) # This dict is purely used for benchmarking with prince @@ -661,7 +661,7 @@ def run_speed_benchmark(self, task): args.append('-o') args.append(f'"{hashlist_out_path}"') - + full_cmd = ' '.join(args) full_cmd = f"{self.callPath} {full_cmd}" diff --git a/htpclient/jsonRequest.py b/htpclient/jsonRequest.py index c8155a6..01a9418 100644 --- a/htpclient/jsonRequest.py +++ b/htpclient/jsonRequest.py @@ -1,10 +1,11 @@ import logging -from htpclient.config import * -from htpclient.session import * +from htpclient.config import Config +from htpclient.session import Session class JsonRequest: + def __init__(self, data): self.data = data self.config = Config() @@ -17,7 +18,8 @@ def execute(self, ignore_certificate: bool = True): self.config.get_value('url'), json=self.data, timeout=30, - verify=not ignore_certificate) + verify=not ignore_certificate, + allow_redirects=True) if r.status_code != 200: logging.error("Status code from server: " + str(r.status_code)) return None diff --git a/htpclient/session.py b/htpclient/session.py index 5552125..fb21760 100644 --- a/htpclient/session.py +++ b/htpclient/session.py @@ -1,11 +1,16 @@ + +from __future__ import annotations + import requests class Session: - __instance = None + __instance: Session = None + s: requests.Session - def __new__(cls, s=None): + def __new__(cls, s: requests.Session | None = None): if Session.__instance is None: Session.__instance = object.__new__(cls) + assert isinstance(s, requests.Session) Session.__instance.s = s return Session.__instance diff --git a/tests/hashtopolis.py b/tests/hashtopolis.py index ef2a061..5ed3c8d 100644 --- a/tests/hashtopolis.py +++ b/tests/hashtopolis.py @@ -48,8 +48,6 @@ def __init__(self): self.username = self._cfg['username'] self.password = self._cfg['password'] - - class HashtopolisConnector(object): # Cache authorisation token per endpoint @@ -62,14 +60,14 @@ def __init__(self, model_uri, config): self._hashtopolis_uri = config._hashtopolis_uri self.config = config - def authenticate(self): + def authenticate(self): if not self._api_endpoint in HashtopolisConnector.token: # Request access TOKEN, used throughout the test logger.info("Start authentication") auth_uri = self._api_endpoint + '/auth/token' auth = (self.config.username, self.config.password) - r = requests.post(auth_uri, auth=auth) + r = requests.post(auth_uri, auth=auth, allow_redirects=True) HashtopolisConnector.token[self._api_endpoint] = r.json()['token'] HashtopolisConnector.token_expires[self._api_endpoint] = r.json()['token'] @@ -93,7 +91,7 @@ def filter(self, expand, filter): '__gt': '>', '__gte': '>=', '__lt': '<', - '__lte': '<=', + '__lte': '<=', } for k,v in filter.items(): l = None @@ -104,7 +102,7 @@ def filter(self, expand, filter): # Default to equal assignment if l == None: l = f'{k}={v}' - filter_list.append(l) + filter_list.append(l) payload = { 'filter': filter_list, @@ -146,7 +144,7 @@ def create(self, obj): headers = self._headers payload = dict([(k,v[1]) for (k,v) in obj.diff().items()]) - r = requests.post(uri, headers=headers, data=json.dumps(payload)) + r = requests.post(uri, headers=headers, data=json.dumps(payload), allow_redirects=True) if r.status_code != 201: logger.exception("Creation of object failed: %s", r.text) @@ -222,7 +220,7 @@ def get(cls, expand=None, **kwargs): return objs[0] @classmethod - def filter(cls, expand=None, **kwargs): + def filter(cls, expand=None, **kwargs): # Get all objects api_objs = cls.get_conn().filter(expand, kwargs) @@ -242,7 +240,7 @@ def __new__(cls, clsname, bases, attrs, uri=None, **kwargs): return super().__new__(cls, clsname, bases, attrs) new_class = super().__new__(cls, clsname, bases, attrs) - + setattr(new_class, 'objects', type('Manager', (ManagerBase,), {'_model_uri': uri})) setattr(new_class.objects, '_model', new_class) cls_registry[clsname] = new_class @@ -251,10 +249,10 @@ def __new__(cls, clsname, bases, attrs, uri=None, **kwargs): class Model(metaclass=ModelBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): self.set_initial(kwargs) super().__init__() - + def _dict2obj(self, dict): # Function to convert a dict to an object. uri = dict.get('_self') @@ -264,7 +262,7 @@ def _dict2obj(self, dict): model_uri = model.objects._model_uri # Check if part of the uri is in the model uri if model_uri in uri: - + obj = model() # Set all the attributes of the object. @@ -287,12 +285,12 @@ def set_initial(self, kv): # Create attribute values for k,v in kv.items(): - + # In case expand is true, there can be a attribute which also is an object. # Example: Users in AccessGroups. This part will convert the returned data. # Into proper objects. if type(v) is list and len(v) > 0: - + # Many-to-Many relation obj_list = [] # Loop through all the values in the list and convert them to objects. @@ -373,7 +371,7 @@ def __repr__(self): class CrackerType(Model, uri="/ui/crackertypes"): def __repr__(self): return self._self - + class Config(Model, uri="/ui/configs"): def __repr__(self): return self._self @@ -389,7 +387,7 @@ def __init__(self): def __repr__(self): return self._self - + def do_upload(self, filename, file_stream): self.authenticate() From 8de5ecd46399880d0d93f6f63fadee6e9cf038d6 Mon Sep 17 00:00:00 2001 From: Eric Fossum Date: Sun, 24 Dec 2023 16:06:08 -0800 Subject: [PATCH 5/5] Adding forgotten requests import --- __main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/__main__.py b/__main__.py index 6933ba3..28fc71b 100644 --- a/__main__.py +++ b/__main__.py @@ -1,4 +1,5 @@ import glob +import requests import shutil import signal import sys