Skip to content

Commit 07b4630

Browse files
charlie-costanzocharlie-costanzo
authored
Payments: create Littlepay feed v3 external tables (#3762)
* Payments: create Littlepay feed v3 external tables * rename create external tables directory for consistency * rename external table schema and tables to follow prior pattern
1 parent 88e3a4e commit 07b4630

11 files changed

+532
-0
lines changed

‎airflow/dags/create_external_tables/payments/littlepay_v3/authorisations.yml

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "authorisations/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.authorisations"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "authorisations/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: participant_id
14+
type: STRING
15+
- name: aggregation_id
16+
type: STRING
17+
- name: acquirer_code # renamed in v3 from acquirer_id
18+
type: STRING
19+
- name: request_type
20+
type: STRING # (Card_Check, Authorisation, Debt Recovery)
21+
- name: amount # renamed in v3 from transaction_amount
22+
type: STRING
23+
- name: currency_code
24+
type: STRING # ISO 4217 Numeric Code
25+
- name: retrieval_reference_number
26+
type: STRING
27+
- name: littlepay_reference_number
28+
type: STRING
29+
- name: external_reference_number
30+
type: STRING
31+
- name: response_code
32+
type: STRING
33+
- name: status
34+
type: STRING # (Authorised, Declined, Failed, Invalid, Lost, Stolen, Unavailable, Unknown, Verified)
35+
- name: authorisation_timestamp_utc # renamed in v3 from authorisation_date_time_utc
36+
type: STRING
37+
- name: request_created_timestamp_utc # added in v3
38+
type: TIMESTAMP
39+
- name: record_updated_timestamp_utc # added in v3
40+
type: TIMESTAMP
41+
- name: channel # added in v3
42+
type: STRING
43+
- name: _line_number
44+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/customer_funding_sources.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "customer-funding-sources/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.customer_funding_sources"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "customer-funding-sources/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: cfs_id
14+
type: STRING
15+
- name: funding_source_id # renamed in v3 from funding_source_vault_id
16+
type: STRING
17+
- name: customer_id
18+
type: STRING
19+
- name: bin
20+
type: STRING
21+
- name: masked_pan
22+
type: STRING
23+
- name: card_scheme
24+
type: STRING
25+
- name: issuer_country
26+
type: STRING
27+
- name: form_factor
28+
type: STRING
29+
- name: principal_customer_id
30+
type: STRING
31+
- name: participant_id
32+
type: STRING
33+
- name: record_updated_timestamp_utc
34+
type: TIMESTAMP
35+
- name: channel
36+
type: STRING
37+
- name: _line_number
38+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/device_transaction_purchases.yml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "device-transaction-purchases/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.device_transaction_purchases"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "device-transaction-purchases/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: littlepay_transaction_id
14+
type: STRING
15+
- name: purchase_id
16+
type: STRING
17+
- name: correlated_purchase_id
18+
type: STRING
19+
- name: product_id
20+
type: STRING
21+
- name: description
22+
type: STRING
23+
- name: indicative_amount
24+
type: STRING
25+
- name: transaction_timestamp_utc # renamed in v3 from transaction_time
26+
type: TIMESTAMP
27+
- name: record_updated_timestamp_utc
28+
type: TIMESTAMP
29+
- name: participant_id
30+
type: STRING
31+
- name: channel
32+
type: STRING
33+
- name: _line_number
34+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/device_transactions.yml

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "device-transactions/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.device_transactions"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "device-transactions/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: participant_id
14+
type: STRING
15+
- name: customer_id
16+
type: STRING
17+
- name: device_transaction_id
18+
type: STRING
19+
- name: littlepay_transaction_id
20+
type: STRING
21+
- name: device_id
22+
type: STRING
23+
- name: device_id_issuer
24+
type: STRING
25+
- name: type
26+
type: STRING
27+
- name: transaction_outcome
28+
type: STRING
29+
- name: transaction_timestamp_utc # renamed in v3 from transaction_date_time_utc
30+
type: TIMESTAMP
31+
- name: location_id
32+
type: STRING
33+
- name: location_name
34+
type: STRING
35+
- name: zone_id
36+
type: STRING
37+
- name: route_id
38+
type: STRING
39+
- name: mode
40+
type: STRING
41+
- name: direction
42+
type: STRING
43+
- name: latitude
44+
type: STRING
45+
- name: longitude
46+
type: STRING
47+
- name: vehicle_id
48+
type: STRING
49+
- name: granted_zone_ids
50+
type: STRING
51+
- name: onward_zone_ids
52+
type: STRING
53+
- name: processed_timestamp_utc
54+
type: STRING
55+
- name: trip_attributes
56+
type: STRING
57+
- name: record_updated_timestamp_utc
58+
type: TIMESTAMP
59+
- name: funding_source_id
60+
type: STRING
61+
- name: channel
62+
type: STRING
63+
- name: _line_number
64+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/micropayment_adjustments.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "micropayment-adjustments/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.micropayment_adjustments"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "micropayment-adjustments/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: micropayment_id
14+
type: STRING
15+
- name: adjustment_id
16+
type: STRING
17+
- name: participant_id
18+
type: STRING
19+
- name: customer_id
20+
type: STRING
21+
- name: product_id
22+
type: STRING
23+
- name: type
24+
type: STRING # Daily_Cap, Weekly_Cap
25+
- name: description
26+
type: STRING
27+
- name: amount
28+
type: STRING
29+
- name: time_period_type
30+
type: STRING # Peak, Off Peak
31+
- name: applied
32+
type: STRING
33+
- name: zone_ids_used
34+
type: STRING
35+
- name: incentive_product_id
36+
type: STRING
37+
- name: record_updated_timestamp_utc
38+
type: TIMESTAMP
39+
- name: currency_code
40+
type: STRING
41+
- name: funding_source_id
42+
type: STRING
43+
- name: channel
44+
type: STRING
45+
- name: _line_number
46+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/micropayment_device_transactions.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "micropayment-device-transactions/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.micropayment_device_transactions"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "micropayment-device-transactions/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: littlepay_transaction_id
14+
type: STRING
15+
- name: micropayment_id
16+
type: STRING
17+
- name: record_updated_timestamp_utc
18+
type: TIMESTAMP
19+
- name: participant_id
20+
type: STRING
21+
- name: channel
22+
type: STRING
23+
- name: _line_number
24+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/micropayments.yml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "micropayments/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.micropayments"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "micropayments/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: micropayment_id
14+
type: STRING
15+
- name: aggregation_id
16+
type: STRING
17+
- name: participant_id
18+
type: STRING
19+
- name: customer_id
20+
type: STRING
21+
- name: funding_source_id # renamed in v3 from funding_source_vault_id
22+
type: STRING
23+
- name: payment_liability
24+
type: STRING
25+
- name: charge_amount
26+
type: STRING
27+
- name: nominal_amount
28+
type: STRING
29+
- name: currency_code
30+
type: STRING
31+
- name: type
32+
type: STRING
33+
- name: charge_type
34+
type: STRING
35+
- name: record_updated_timestamp_utc
36+
type: TIMESTAMP
37+
- name: channel
38+
type: STRING
39+
- name: _line_number
40+
type: STRING

‎airflow/dags/create_external_tables/payments/littlepay_v3/products.yml

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
operator: operators.ExternalTable
2+
bucket: "{{ env_var('CALITP_BUCKET__LITTLEPAY_PARSED_V3') }}"
3+
source_objects:
4+
- "products/*.jsonl.gz"
5+
destination_project_dataset_table: "external_littlepay_v3.products"
6+
source_format: NEWLINE_DELIMITED_JSON
7+
use_bq_client: true
8+
hive_options:
9+
mode: CUSTOM
10+
require_partition_filter: false
11+
source_uri_prefix: "products/{instance:STRING}/{extract_filename:STRING}/{ts:TIMESTAMP}/"
12+
schema_fields:
13+
- name: participant_id
14+
type: STRING
15+
- name: product_id
16+
type: STRING
17+
- name: previous_version_id
18+
type: STRING
19+
- name: original_version_id
20+
type: STRING
21+
- name: product_code
22+
type: STRING
23+
- name: product_description
24+
type: STRING
25+
- name: product_type
26+
type: STRING # (Capping, Discount)
27+
- name: activation_type
28+
type: STRING # (Customer, Card)
29+
- name: product_status
30+
type: STRING # (Active, Inactive)
31+
- name: superseded
32+
type: STRING
33+
- name: product_created_timestamp_utc # renamed in v3 from created_date
34+
type: STRING
35+
- name: capping_type
36+
type: STRING # (Daily_cap, Weekly_cap, Multiday_cap, Timebased_cap)
37+
- name: multi_operator
38+
type: STRING
39+
- name: capping_start_time
40+
type: STRING
41+
- name: capping_end_time
42+
type: STRING
43+
- name: rules_transaction_types
44+
type: STRING # (Autoscan, Flat, Variable)
45+
- name: rules_default_limit
46+
type: STRING
47+
- name: rules_max_fare_value
48+
type: STRING
49+
- name: scheduled_start_timestamp_utc # renamed in v3 from scheduled_start_date_time
50+
type: TIMESTAMP
51+
- name: scheduled_end_timestamp_utc # renamed in v3 from scheduled_end_date_time
52+
type: TIMESTAMP
53+
- name: all_day
54+
type: BOOLEAN
55+
- name: weekly_cap_start_day
56+
type: STRING # (Monday, Sunday)
57+
- name: weekly_cap_end_day
58+
type: STRING
59+
- name: number_of_days_in_cap_window
60+
type: STRING
61+
- name: capping_duration
62+
type: STRING
63+
- name: number_of_transfer
64+
type: STRING
65+
- name: capping_time_zone
66+
type: STRING
67+
- name: capping_overlap_time
68+
type: STRING
69+
- name: capping_application_level
70+
type: STRING # (Customer, Card)
71+
- name: route_capping_enabled
72+
type: STRING
73+
- name: routes
74+
type: STRING
75+
- name: zoned_capping_enabled
76+
type: BOOLEAN
77+
- name: zoned_capping_mode
78+
type: STRING
79+
- name: zoned_capping_pricing_type
80+
type: STRING
81+
- name: on_peak_zones
82+
type: STRING
83+
- name: off_peak_zones
84+
type: STRING
85+
- name: incentive_enabled
86+
type: BOOLEAN
87+
- name: incentive_type
88+
type: STRING
89+
- name: discount_qualifier
90+
type: STRING
91+
- name: configuration
92+
type: STRING
93+
- name: record_updated_timestamp_utc
94+
type: TIMESTAMP
95+
- name: channel
96+
type: STRING
97+
- name: _line_number
98+
type: STRING

0 commit comments

Comments
 (0)