Skip to content

Commit

Permalink
Remove task_id param from memo functions, as whole task record is ava…
Browse files Browse the repository at this point in the history
…ilable (#2080)

When these functions need the task id, they can extract it from
the task record.

see #2014

Co-authored-by: Yadu Nand Babuji <[email protected]>
  • Loading branch information
benclifford and yadudoc authored Jun 10, 2021
1 parent 20e4543 commit 1449ea7
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 5 deletions.
4 changes: 2 additions & 2 deletions parsl/dataflow/dflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ def handle_app_update(self, task_record, future):
if not task_record['app_fu'] == future:
logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))

self.memoizer.update_memo(task_id, task_record, future)
self.memoizer.update_memo(task_record, future)

if self.checkpoint_mode == 'task_exit':
self.checkpoint(tasks=[task_id])
Expand Down Expand Up @@ -557,7 +557,7 @@ def launch_task(self, task_record, executable, *args, **kwargs):
task_id = task_record['id']
task_record['try_time_launched'] = datetime.datetime.now()

memo_fu = self.memoizer.check_memo(task_id, task_record)
memo_fu = self.memoizer.check_memo(task_record)
if memo_fu:
logger.info("Reusing cached result for task {}".format(task_id))
task_record['from_memo'] = True
Expand Down
11 changes: 8 additions & 3 deletions parsl/dataflow/memoization.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def make_hash(self, task):
hashedsum = hashlib.md5(x).hexdigest()
return hashedsum

def check_memo(self, task_id, task):
def check_memo(self, task):
"""Create a hash of the task and its inputs and check the lookup table for this hash.
If present, the results are returned. The result is a tuple indicating whether a memo
Expand All @@ -221,6 +221,9 @@ def check_memo(self, task_id, task):
This call will also set task['hashsum'] to the unique hashsum for the func+inputs.
"""

task_id = task['id']

if not self.memoize or not task['memoize']:
task['hashsum'] = None
logger.debug("Task {} will not be memoized".format(task_id))
Expand Down Expand Up @@ -254,11 +257,10 @@ def hash_lookup(self, hashsum):
"""
return self.memo_lookup_table[hashsum]

def update_memo(self, task_id, task, r):
def update_memo(self, task, r):
"""Updates the memoization lookup table with the result from a task.
Args:
- task_id (int): Integer task id
- task (dict) : A task dict from dfk.tasks
- r (Result future): Result future
Expand All @@ -267,6 +269,9 @@ def update_memo(self, task_id, task, r):
"""
# TODO: could use typeguard
assert isinstance(r, Future)

task_id = task['id']

if not self.memoize or not task['memoize'] or 'hashsum' not in task:
return

Expand Down

0 comments on commit 1449ea7

Please sign in to comment.