Skip to content

Commit 24a9070

Browse files
yoonhyejinyoonhyejin
committed
update golden files for dags
1 parent 01f17c3 commit 24a9070

File tree

3 files changed

+832
-260
lines changed

3 files changed

+832
-260
lines changed

‎metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json

+21-9
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@
125125
"downstream_task_ids": "[]",
126126
"inlets": "[]",
127127
"outlets": "[]",
128-
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
128+
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
129129
},
130130
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
131131
"name": "transform_cost_table",
@@ -302,16 +302,16 @@
302302
"state": "running",
303303
"operator": "AthenaOperator",
304304
"priority_weight": "1",
305-
"log_url": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
305+
"log_url": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
306306
"orchestrator": "airflow",
307307
"dag_id": "athena_operator",
308308
"task_id": "transform_cost_table"
309309
},
310-
"externalUrl": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
310+
"externalUrl": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
311311
"name": "athena_operator_transform_cost_table_manual_run_test",
312312
"type": "BATCH_AD_HOC",
313313
"created": {
314-
"time": 1732719433576,
314+
"time": 1737519321050,
315315
"actor": "urn:li:corpuser:datahub"
316316
}
317317
}
@@ -329,6 +329,18 @@
329329
}
330330
}
331331
},
332+
{
333+
"entityType": "dataProcessInstance",
334+
"entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
335+
"changeType": "UPSERT",
336+
"aspectName": "dataPlatformInstance",
337+
"aspect": {
338+
"json": {
339+
"platform": "urn:li:dataPlatform:airflow",
340+
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:airflow,prod)"
341+
}
342+
}
343+
},
332344
{
333345
"entityType": "dataProcessInstance",
334346
"entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad",
@@ -388,7 +400,7 @@
388400
"aspectName": "dataProcessInstanceRunEvent",
389401
"aspect": {
390402
"json": {
391-
"timestampMillis": 1732719433576,
403+
"timestampMillis": 1737519321050,
392404
"partitionSpec": {
393405
"partition": "FULL_TABLE_SNAPSHOT",
394406
"type": "FULL_TABLE"
@@ -405,14 +417,14 @@
405417
"aspectName": "operation",
406418
"aspect": {
407419
"json": {
408-
"timestampMillis": 1732719433736,
420+
"timestampMillis": 1737519321422,
409421
"partitionSpec": {
410422
"partition": "FULL_TABLE_SNAPSHOT",
411423
"type": "FULL_TABLE"
412424
},
413425
"actor": "urn:li:corpuser:airflow",
414426
"operationType": "CREATE",
415-
"lastUpdatedTimestamp": 1732719433736
427+
"lastUpdatedTimestamp": 1737519321422
416428
}
417429
}
418430
},
@@ -435,7 +447,7 @@
435447
"downstream_task_ids": "[]",
436448
"inlets": "[]",
437449
"outlets": "[]",
438-
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
450+
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
439451
},
440452
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table",
441453
"name": "transform_cost_table",
@@ -656,7 +668,7 @@
656668
"aspectName": "dataProcessInstanceRunEvent",
657669
"aspect": {
658670
"json": {
659-
"timestampMillis": 1732719433747,
671+
"timestampMillis": 1737519321471,
660672
"partitionSpec": {
661673
"partition": "FULL_TABLE_SNAPSHOT",
662674
"type": "FULL_TABLE"

‎metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_snowflake_operator.json

+84-8
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,17 @@
2323
}
2424
}
2525
},
26+
{
27+
"entityType": "dataFlow",
28+
"entityUrn": "urn:li:dataFlow:(airflow,snowflake_operator,prod)",
29+
"changeType": "UPSERT",
30+
"aspectName": "status",
31+
"aspect": {
32+
"json": {
33+
"removed": false
34+
}
35+
}
36+
},
2637
{
2738
"entityType": "dataFlow",
2839
"entityUrn": "urn:li:dataFlow:(airflow,snowflake_operator,prod)",
@@ -115,7 +126,7 @@
115126
"downstream_task_ids": "[]",
116127
"inlets": "[]",
117128
"outlets": "[]",
118-
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
129+
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
119130
},
120131
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=snowflake_operator&_flt_3_task_id=transform_cost_table",
121132
"name": "transform_cost_table",
@@ -126,6 +137,17 @@
126137
}
127138
}
128139
},
140+
{
141+
"entityType": "dataJob",
142+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)",
143+
"changeType": "UPSERT",
144+
"aspectName": "status",
145+
"aspect": {
146+
"json": {
147+
"removed": false
148+
}
149+
}
150+
},
129151
{
130152
"entityType": "dataJob",
131153
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)",
@@ -281,16 +303,16 @@
281303
"state": "running",
282304
"operator": "SnowflakeOperator",
283305
"priority_weight": "1",
284-
"log_url": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
306+
"log_url": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
285307
"orchestrator": "airflow",
286308
"dag_id": "snowflake_operator",
287309
"task_id": "transform_cost_table"
288310
},
289-
"externalUrl": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs",
311+
"externalUrl": "http://airflow.example.com/dags/snowflake_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
290312
"name": "snowflake_operator_transform_cost_table_manual_run_test",
291313
"type": "BATCH_AD_HOC",
292314
"created": {
293-
"time": 1718733682840,
315+
"time": 1737519137480,
294316
"actor": "urn:li:corpuser:datahub"
295317
}
296318
}
@@ -308,6 +330,18 @@
308330
}
309331
}
310332
},
333+
{
334+
"entityType": "dataProcessInstance",
335+
"entityUrn": "urn:li:dataProcessInstance:3161034cc84e16a7c5e1906225734747",
336+
"changeType": "UPSERT",
337+
"aspectName": "dataPlatformInstance",
338+
"aspect": {
339+
"json": {
340+
"platform": "urn:li:dataPlatform:airflow",
341+
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:airflow,prod)"
342+
}
343+
}
344+
},
311345
{
312346
"entityType": "dataProcessInstance",
313347
"entityUrn": "urn:li:dataProcessInstance:3161034cc84e16a7c5e1906225734747",
@@ -367,7 +401,7 @@
367401
"aspectName": "dataProcessInstanceRunEvent",
368402
"aspect": {
369403
"json": {
370-
"timestampMillis": 1717179684292,
404+
"timestampMillis": 1737519137480,
371405
"partitionSpec": {
372406
"partition": "FULL_TABLE_SNAPSHOT",
373407
"type": "FULL_TABLE"
@@ -384,14 +418,56 @@
384418
"aspectName": "operation",
385419
"aspect": {
386420
"json": {
387-
"timestampMillis": 1717179684935,
421+
"timestampMillis": 1737519138203,
388422
"partitionSpec": {
389423
"partition": "FULL_TABLE_SNAPSHOT",
390424
"type": "FULL_TABLE"
391425
},
392426
"actor": "urn:li:corpuser:airflow",
393427
"operationType": "CREATE",
394-
"lastUpdatedTimestamp": 1717179684935
428+
"lastUpdatedTimestamp": 1737519138203
429+
}
430+
}
431+
},
432+
{
433+
"entityType": "dataJob",
434+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)",
435+
"changeType": "UPSERT",
436+
"aspectName": "dataJobInfo",
437+
"aspect": {
438+
"json": {
439+
"customProperties": {
440+
"depends_on_past": "False",
441+
"email": "None",
442+
"label": "'transform_cost_table'",
443+
"execution_timeout": "None",
444+
"sla": "None",
445+
"sql": "'\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n '",
446+
"task_id": "'transform_cost_table'",
447+
"trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
448+
"wait_for_downstream": "False",
449+
"downstream_task_ids": "[]",
450+
"inlets": "[]",
451+
"outlets": "[]",
452+
"openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}"
453+
},
454+
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=snowflake_operator&_flt_3_task_id=transform_cost_table",
455+
"name": "transform_cost_table",
456+
"type": {
457+
"string": "COMMAND"
458+
},
459+
"env": "PROD"
460+
}
461+
}
462+
},
463+
{
464+
"entityType": "dataJob",
465+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,snowflake_operator,prod),transform_cost_table)",
466+
"changeType": "UPSERT",
467+
"aspectName": "status",
468+
"aspect": {
469+
"json": {
470+
"removed": false
395471
}
396472
}
397473
},
@@ -538,7 +614,7 @@
538614
"aspectName": "dataProcessInstanceRunEvent",
539615
"aspect": {
540616
"json": {
541-
"timestampMillis": 1717179685374,
617+
"timestampMillis": 1737519138302,
542618
"partitionSpec": {
543619
"partition": "FULL_TABLE_SNAPSHOT",
544620
"type": "FULL_TABLE"

0 commit comments

Comments
 (0)