Skip to content

Commit 01f17c3

Browse files
yoonhyejinyoonhyejin
committed
update golden files for airflow
1 parent 534e3b4 commit 01f17c3

File tree

4 files changed

+401
-91
lines changed

4 files changed

+401
-91
lines changed

Diff for: ‎metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_basic_iolets.json

+85-10
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,17 @@
2323
}
2424
}
2525
},
26+
{
27+
"entityType": "dataFlow",
28+
"entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)",
29+
"changeType": "UPSERT",
30+
"aspectName": "status",
31+
"aspect": {
32+
"json": {
33+
"removed": false
34+
}
35+
}
36+
},
2637
{
2738
"entityType": "dataFlow",
2839
"entityUrn": "urn:li:dataFlow:(airflow,basic_iolets,prod)",
@@ -114,7 +125,7 @@
114125
"downstream_task_ids": "[]",
115126
"inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
116127
"outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
117-
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
128+
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"multiple_outputs\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"<<non-serializable: _DownstreamPriorityWeightStrategy>>\"}, \"type\": \"operator\"}]}"
118129
},
119130
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
120131
"name": "run_data_task",
@@ -125,6 +136,17 @@
125136
}
126137
}
127138
},
139+
{
140+
"entityType": "dataJob",
141+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
142+
"changeType": "UPSERT",
143+
"aspectName": "status",
144+
"aspect": {
145+
"json": {
146+
"removed": false
147+
}
148+
}
149+
},
128150
{
129151
"entityType": "dataJob",
130152
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
@@ -281,16 +303,16 @@
281303
"state": "running",
282304
"operator": "BashOperator",
283305
"priority_weight": "1",
284-
"log_url": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&map_index=-1&tab=logs",
306+
"log_url": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
285307
"orchestrator": "airflow",
286308
"dag_id": "basic_iolets",
287309
"task_id": "run_data_task"
288310
},
289-
"externalUrl": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&map_index=-1&tab=logs",
311+
"externalUrl": "http://airflow.example.com/dags/basic_iolets/grid?dag_run_id=manual_run_test&task_id=run_data_task&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs",
290312
"name": "basic_iolets_run_data_task_manual_run_test",
291313
"type": "BATCH_AD_HOC",
292314
"created": {
293-
"time": 1718733614956,
315+
"time": 1737510387004,
294316
"actor": "urn:li:corpuser:datahub"
295317
}
296318
}
@@ -308,6 +330,18 @@
308330
}
309331
}
310332
},
333+
{
334+
"entityType": "dataProcessInstance",
335+
"entityUrn": "urn:li:dataProcessInstance:5d666eaf9015a31b3e305e8bc2dba078",
336+
"changeType": "UPSERT",
337+
"aspectName": "dataPlatformInstance",
338+
"aspect": {
339+
"json": {
340+
"platform": "urn:li:dataPlatform:airflow",
341+
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:airflow,prod)"
342+
}
343+
}
344+
},
311345
{
312346
"entityType": "dataProcessInstance",
313347
"entityUrn": "urn:li:dataProcessInstance:5d666eaf9015a31b3e305e8bc2dba078",
@@ -423,7 +457,7 @@
423457
"aspectName": "dataProcessInstanceRunEvent",
424458
"aspect": {
425459
"json": {
426-
"timestampMillis": 1717179624988,
460+
"timestampMillis": 1737510387004,
427461
"partitionSpec": {
428462
"partition": "FULL_TABLE_SNAPSHOT",
429463
"type": "FULL_TABLE"
@@ -440,14 +474,14 @@
440474
"aspectName": "operation",
441475
"aspect": {
442476
"json": {
443-
"timestampMillis": 1717179625524,
477+
"timestampMillis": 1737510387613,
444478
"partitionSpec": {
445479
"partition": "FULL_TABLE_SNAPSHOT",
446480
"type": "FULL_TABLE"
447481
},
448482
"actor": "urn:li:corpuser:airflow",
449483
"operationType": "CREATE",
450-
"lastUpdatedTimestamp": 1717179625524
484+
"lastUpdatedTimestamp": 1737510387613
451485
}
452486
}
453487
},
@@ -458,14 +492,55 @@
458492
"aspectName": "operation",
459493
"aspect": {
460494
"json": {
461-
"timestampMillis": 1717179625547,
495+
"timestampMillis": 1737510387702,
462496
"partitionSpec": {
463497
"partition": "FULL_TABLE_SNAPSHOT",
464498
"type": "FULL_TABLE"
465499
},
466500
"actor": "urn:li:corpuser:airflow",
467501
"operationType": "CREATE",
468-
"lastUpdatedTimestamp": 1717179625547
502+
"lastUpdatedTimestamp": 1737510387702
503+
}
504+
}
505+
},
506+
{
507+
"entityType": "dataJob",
508+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
509+
"changeType": "UPSERT",
510+
"aspectName": "dataJobInfo",
511+
"aspect": {
512+
"json": {
513+
"customProperties": {
514+
"depends_on_past": "False",
515+
"email": "None",
516+
"label": "'run_data_task'",
517+
"execution_timeout": "None",
518+
"sla": "None",
519+
"task_id": "'run_data_task'",
520+
"trigger_rule": "<TriggerRule.ALL_SUCCESS: 'all_success'>",
521+
"wait_for_downstream": "False",
522+
"downstream_task_ids": "[]",
523+
"inlets": "[Dataset(platform='snowflake', name='mydb.schema.tableA', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableB', env='DEV', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableC', env='PROD', platform_instance='cloud'), Urn(_urn='urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)'), Urn(_urn='urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)')]",
524+
"outlets": "[Dataset(platform='snowflake', name='mydb.schema.tableD', env='PROD', platform_instance=None), Dataset(platform='snowflake', name='mydb.schema.tableE', env='PROD', platform_instance=None)]",
525+
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.25.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"BashOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"multiple_outputs\": false, \"operator_class\": \"airflow.operators.bash.BashOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"run_data_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"<<non-serializable: _DownstreamPriorityWeightStrategy>>\"}, \"type\": \"operator\"}]}"
526+
},
527+
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task",
528+
"name": "run_data_task",
529+
"type": {
530+
"string": "COMMAND"
531+
},
532+
"env": "PROD"
533+
}
534+
}
535+
},
536+
{
537+
"entityType": "dataJob",
538+
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)",
539+
"changeType": "UPSERT",
540+
"aspectName": "status",
541+
"aspect": {
542+
"json": {
543+
"removed": false
469544
}
470545
}
471546
},
@@ -613,7 +688,7 @@
613688
"aspectName": "dataProcessInstanceRunEvent",
614689
"aspect": {
615690
"json": {
616-
"timestampMillis": 1717179625632,
691+
"timestampMillis": 1737510387814,
617692
"partitionSpec": {
618693
"partition": "FULL_TABLE_SNAPSHOT",
619694
"type": "FULL_TABLE"

0 commit comments

Comments
 (0)