|
1 | 1 | import random
|
2 | 2 | import time
|
3 | 3 | from dataclasses import dataclass
|
4 |
| -from typing import Iterable, List, Optional, Union |
| 4 | +from typing import Dict, Iterable, List, Optional, Union |
5 | 5 |
|
6 | 6 | import datahub.metadata.schema_classes as models
|
7 | 7 | from datahub.api.entities.datajob import DataFlow, DataJob
|
@@ -29,6 +29,13 @@ class ContainerKeyWithId(ContainerKey):
|
29 | 29 | id: str
|
30 | 30 |
|
31 | 31 |
|
| 32 | +@dataclass |
| 33 | +class RunInfo: |
| 34 | + start_time: int |
| 35 | + duration: int |
| 36 | + result: InstanceRunResult |
| 37 | + |
| 38 | + |
32 | 39 | @dataclass
|
33 | 40 | class Container:
|
34 | 41 | key: ContainerKeyWithId
|
@@ -222,32 +229,32 @@ def generate_pipeline(
|
222 | 229 | start_time = end_time - (30 * 24 * 60 * 60 * 1000)
|
223 | 230 | run_timestamps = [start_time + (i * 5 * 24 * 60 * 60 * 1000) for i in range(5)]
|
224 | 231 |
|
225 |
| - run_dict = { |
226 |
| - "run_1": { |
227 |
| - "start_time": run_timestamps[0], |
228 |
| - "duration": 45, |
229 |
| - "result": InstanceRunResult.SUCCESS, |
230 |
| - }, |
231 |
| - "run_2": { |
232 |
| - "start_time": run_timestamps[1], |
233 |
| - "duration": 60, |
234 |
| - "result": InstanceRunResult.FAILURE, |
235 |
| - }, |
236 |
| - "run_3": { |
237 |
| - "start_time": run_timestamps[2], |
238 |
| - "duration": 55, |
239 |
| - "result": InstanceRunResult.SUCCESS, |
240 |
| - }, |
241 |
| - "run_4": { |
242 |
| - "start_time": run_timestamps[3], |
243 |
| - "duration": 70, |
244 |
| - "result": InstanceRunResult.SUCCESS, |
245 |
| - }, |
246 |
| - "run_5": { |
247 |
| - "start_time": run_timestamps[4], |
248 |
| - "duration": 50, |
249 |
| - "result": InstanceRunResult.FAILURE, |
250 |
| - }, |
| 232 | + run_dict: Dict[str, RunInfo] = { |
| 233 | + "run_1": RunInfo( |
| 234 | + start_time=run_timestamps[0], |
| 235 | + duration=45, |
| 236 | + result=InstanceRunResult.SUCCESS, |
| 237 | + ), |
| 238 | + "run_2": RunInfo( |
| 239 | + start_time=run_timestamps[1], |
| 240 | + duration=60, |
| 241 | + result=InstanceRunResult.FAILURE, |
| 242 | + ), |
| 243 | + "run_3": RunInfo( |
| 244 | + start_time=run_timestamps[2], |
| 245 | + duration=55, |
| 246 | + result=InstanceRunResult.SUCCESS, |
| 247 | + ), |
| 248 | + "run_4": RunInfo( |
| 249 | + start_time=run_timestamps[3], |
| 250 | + duration=70, |
| 251 | + result=InstanceRunResult.SUCCESS, |
| 252 | + ), |
| 253 | + "run_5": RunInfo( |
| 254 | + start_time=run_timestamps[4], |
| 255 | + duration=50, |
| 256 | + result=InstanceRunResult.FAILURE, |
| 257 | + ), |
251 | 258 | }
|
252 | 259 |
|
253 | 260 | for i, (model_name, model_description) in enumerate(
|
@@ -340,25 +347,24 @@ def generate_pipeline(
|
340 | 347 | )
|
341 | 348 |
|
342 | 349 | # Generate start and end events
|
343 |
| - start_time_millis = int(run_dict[run_id]["start_time"]) |
344 |
| - duration_minutes = int(run_dict[run_id]["duration"]) |
| 350 | + run_info = run_dict[run_id] |
| 351 | + start_time_millis = run_info.start_time |
| 352 | + duration_minutes = run_info.duration |
345 | 353 | end_time_millis = start_time_millis + (duration_minutes * 60000)
|
346 |
| - result = run_dict[run_id]["result"] |
347 |
| - if not isinstance(result, InstanceRunResult): |
348 |
| - raise TypeError(f"Expected InstanceRunResult, got {type(result)}") |
| 354 | + result = run_info.result |
349 | 355 |
|
350 | 356 | result_type = (
|
351 | 357 | "SUCCESS" if result == InstanceRunResult.SUCCESS else "FAILURE"
|
352 | 358 | )
|
353 | 359 |
|
354 | 360 | yield from data_process_instance.start_event_mcp(
|
355 |
| - start_timestamp_millis=int(start_time_millis) |
| 361 | + start_timestamp_millis=start_time_millis |
356 | 362 | )
|
357 | 363 | yield from data_process_instance.end_event_mcp(
|
358 |
| - end_timestamp_millis=int(end_time_millis), |
| 364 | + end_timestamp_millis=end_time_millis, |
359 | 365 | result=result,
|
360 | 366 | result_type=result_type,
|
361 |
| - start_timestamp_millis=int(start_time_millis), |
| 367 | + start_timestamp_millis=start_time_millis, |
362 | 368 | )
|
363 | 369 |
|
364 | 370 | # Model
|
|
0 commit comments