Skip to content
This repository has been archived by the owner on Jul 21, 2023. It is now read-only.

Commit

Permalink
Add a workaround for table referencing another from different DAG
Browse files Browse the repository at this point in the history
This replaces node's parent to link to root, instead of failing to lookup table from another DAG
Also references issue created on Github
  • Loading branch information
MikolajBalcerek committed Aug 2, 2019
1 parent f1385e6 commit 14fe6ab
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions dashboard/model/tables_data.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import csv
from concurrent.futures import ThreadPoolExecutor
from typing import Dict

from dashboard.models import Period, Table
from dashboard.utils import clean_dag_id, handle_resource, simple_state
Expand Down Expand Up @@ -43,7 +44,6 @@ class DagTableProgress:

class TableDataProvider:


@staticmethod
def _format_table_id(table):
id = table['db'] + '.' + table['name']
Expand Down Expand Up @@ -75,8 +75,8 @@ def get_tables(self):
def history(self, table):
return self.__get_detailed_view_data(self.tables[table], DETAILED_CHART_DAYS_NUM)

def get_tables_by_dag(self, dag_name):
return [table for table in self.tables.values() if table.dag_id == dag_name]
def get_tables_by_dag(self, dag_name) -> Dict[str, Table]:
return {id: table for id, table in self.tables.items() if table.dag_id == dag_name}

def get_tables_graph(self, dag_id, execution_date):
name_without_version = clean_dag_id(dag_id)
Expand All @@ -89,7 +89,7 @@ def get_tables_graph(self, dag_id, execution_date):
state=self.airflow.get_dag_state(dag_id, execution_date))

# tables
for table in dag_tables:
for table in dag_tables.values():
yield GraphVertex(
id=table.id,
name=table.name + (' ({})'.format(table.period.name) if table.period else ''),
Expand All @@ -98,7 +98,9 @@ def get_tables_graph(self, dag_id, execution_date):
dag_progress[table.task_id].end_date,
dag_progress[table.task_id].duration
),
parent=table.get_parent()
# workaround for table.uses not being able to reference table managed by other DAG
# see https://github.com/Wikia/discreETLy/issues/22
parent='main' if table.uses is None or table.uses not in dag_tables.keys() else table.uses
)

@handle_resource('influx')
Expand Down

0 comments on commit 14fe6ab

Please sign in to comment.