Skip to content

Commit 1165cad

Browse files
cch-kcch-k
authored
initial checkin (#1)
1 parent 4a5853f commit 1165cad

File tree

8 files changed

+530
-0
lines changed

8 files changed

+530
-0
lines changed

‎.github/workflows/fetch.yaml

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Fetch Data
2+
3+
on:
4+
workflow_dispatch:
5+
6+
defaults:
7+
run:
8+
shell: bash
9+
10+
jobs:
11+
on_schedule:
12+
name: Scheduled Run
13+
14+
runs-on: ubuntu-22.04
15+
16+
steps:
17+
- name: Checkout source
18+
uses: actions/checkout@v4
19+
id: checkout-source
20+
21+
22+
- uses: actions/setup-python@v5
23+
with:
24+
python-version: '3.11'
25+
26+
27+
- uses: 'google-github-actions/auth@v2'
28+
with:
29+
credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
30+
31+
32+
- name: execute
33+
run: |
34+
python src/weather_data_loader.py
35+

‎.gitignore

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
# Byte-compiled / optimized / DLL files
2+
**/__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
pip-wheel-metadata/
24+
share/python-wheels/
25+
*.egg-info/
26+
.installed.cfg
27+
*.egg
28+
MANIFEST
29+
30+
# PyInstaller
31+
# Usually these files are written by a python script from a template
32+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
33+
*.manifest
34+
*.spec
35+
36+
# Installer logs
37+
pip-log.txt
38+
pip-delete-this-directory.txt
39+
40+
# Unit test / coverage reports
41+
htmlcov/
42+
.tox/
43+
.nox/
44+
.coverage
45+
.coverage.*
46+
.cache
47+
nosetests.xml
48+
coverage.xml
49+
*.cover
50+
*.py,cover
51+
.hypothesis/
52+
.pytest_cache/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
target/
76+
77+
# Jupyter Notebook
78+
.ipynb_checkpoints
79+
80+
# IPython
81+
profile_default/
82+
ipython_config.py
83+
84+
# pyenv
85+
.python-version
86+
87+
# pipenv
88+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
90+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
91+
# install all needed dependencies.
92+
#Pipfile.lock
93+
94+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
95+
__pypackages__/
96+
97+
# Celery stuff
98+
celerybeat-schedule
99+
celerybeat.pid
100+
101+
# SageMath parsed files
102+
*.sage.py
103+
104+
# Environments
105+
# .env
106+
.venv
107+
env/
108+
venv/
109+
ENV/
110+
env.bak/
111+
venv.bak/
112+
113+
**/venv/
114+
115+
# Spyder project settings
116+
.spyderproject
117+
.spyproject
118+
119+
# Rope project settings
120+
.ropeproject
121+
122+
# mkdocs documentation
123+
/site
124+
125+
# mypy
126+
.mypy_cache/
127+
.dmypy.json
128+
dmypy.json
129+
130+
# Pyre type checker
131+
.pyre/
132+
133+
# locust stats files
134+
false_*.csv
135+
true_*.csv
136+
137+
#intellij
138+
*.iml
139+
*.idea
140+
141+
envs/
142+
143+
# OS files
144+
.DS_Store
145+
146+
147+
google_credential.json
148+
application_default_credentials.json
149+
150+
**/.terraform
151+
terraform/.terraform.lock.hcl
152+
terraform/terraform.tfstate
153+
terraform/terraform.tfstate.backup
154+
155+

‎Makefile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
2+
# Variables
3+
TERRAFORM_DIR := terraform
4+
CURRENT_DIR := $(shell pwd)
5+
6+
.PHONY: list
7+
list:
8+
@LC_ALL=C $(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/(^|\n)# Files(\n|$$)/,/(^|\n)# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$'
9+
10+
11+
tf_init:
12+
cd $(TERRAFORM_DIR) && terraform init ; cd $(CURRENT_DIR)
13+
14+
15+
tf_plan:
16+
cd $(TERRAFORM_DIR) && terraform plan ; cd $(CURRENT_DIR)
17+
18+
tf_apply:
19+
cd $(TERRAFORM_DIR) && terraform apply ; cd $(CURRENT_DIR)
20+
21+

‎note.md

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
2+
#
3+
4+
make tf_plan output
5+
6+
7+
```
8+
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
9+
+ create
10+
11+
Terraform will perform the following actions:
12+
13+
# google_bigquery_dataset.project_dataset will be created
14+
+ resource "google_bigquery_dataset" "project_dataset" {
15+
+ creation_time = (known after apply)
16+
+ dataset_id = "project_dataset"
17+
+ default_collation = (known after apply)
18+
+ delete_contents_on_destroy = false
19+
+ effective_labels = (known after apply)
20+
+ etag = (known after apply)
21+
+ id = (known after apply)
22+
+ is_case_insensitive = (known after apply)
23+
+ last_modified_time = (known after apply)
24+
+ location = "US"
25+
+ max_time_travel_hours = (known after apply)
26+
+ project = "axial-gist-411121"
27+
+ self_link = (known after apply)
28+
+ storage_billing_model = (known after apply)
29+
+ terraform_labels = (known after apply)
30+
}
31+
32+
# google_storage_bucket.project-bucket will be created
33+
+ resource "google_storage_bucket" "project-bucket" {
34+
+ effective_labels = (known after apply)
35+
+ force_destroy = true
36+
+ id = (known after apply)
37+
+ location = "US"
38+
+ name = "data-engineering-zoomcamp-2024-project"
39+
+ project = (known after apply)
40+
+ public_access_prevention = (known after apply)
41+
+ self_link = (known after apply)
42+
+ storage_class = "STANDARD"
43+
+ terraform_labels = (known after apply)
44+
+ uniform_bucket_level_access = (known after apply)
45+
+ url = (known after apply)
46+
47+
+ lifecycle_rule {
48+
+ action {
49+
+ type = "AbortIncompleteMultipartUpload"
50+
}
51+
+ condition {
52+
+ age = 1
53+
+ matches_prefix = []
54+
+ matches_storage_class = []
55+
+ matches_suffix = []
56+
+ with_state = (known after apply)
57+
}
58+
}
59+
}
60+
61+
Plan: 2 to add, 0 to change, 0 to destroy.
62+
63+
```
64+
65+
# make tf_apply output
66+
67+
68+
```
69+
Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
70+
+ create
71+
72+
Terraform will perform the following actions:
73+
74+
# google_bigquery_dataset.project_dataset will be created
75+
+ resource "google_bigquery_dataset" "project_dataset" {
76+
+ creation_time = (known after apply)
77+
+ dataset_id = "project_dataset"
78+
+ default_collation = (known after apply)
79+
+ delete_contents_on_destroy = false
80+
+ effective_labels = (known after apply)
81+
+ etag = (known after apply)
82+
+ id = (known after apply)
83+
+ is_case_insensitive = (known after apply)
84+
+ last_modified_time = (known after apply)
85+
+ location = "US"
86+
+ max_time_travel_hours = (known after apply)
87+
+ project = "axial-gist-411121"
88+
+ self_link = (known after apply)
89+
+ storage_billing_model = (known after apply)
90+
+ terraform_labels = (known after apply)
91+
}
92+
93+
# google_storage_bucket.project-bucket will be created
94+
+ resource "google_storage_bucket" "project-bucket" {
95+
+ effective_labels = (known after apply)
96+
+ force_destroy = true
97+
+ id = (known after apply)
98+
+ location = "US"
99+
+ name = "data-engineering-zoomcamp-2024-project"
100+
+ project = (known after apply)
101+
+ public_access_prevention = (known after apply)
102+
+ self_link = (known after apply)
103+
+ storage_class = "STANDARD"
104+
+ terraform_labels = (known after apply)
105+
+ uniform_bucket_level_access = (known after apply)
106+
+ url = (known after apply)
107+
108+
+ lifecycle_rule {
109+
+ action {
110+
+ type = "AbortIncompleteMultipartUpload"
111+
}
112+
+ condition {
113+
+ age = 1
114+
+ matches_prefix = []
115+
+ matches_storage_class = []
116+
+ matches_suffix = []
117+
+ with_state = (known after apply)
118+
}
119+
}
120+
}
121+
122+
Plan: 2 to add, 0 to change, 0 to destroy.
123+
124+
Do you want to perform these actions?
125+
Terraform will perform the actions described above.
126+
Only 'yes' will be accepted to approve.
127+
128+
Enter a value: yes
129+
130+
google_bigquery_dataset.project_dataset: Creating...
131+
google_storage_bucket.project-bucket: Creating...
132+
google_bigquery_dataset.project_dataset: Creation complete after 1s [id=projects/axial-gist-411121/datasets/project_dataset]
133+
google_storage_bucket.project-bucket: Creation complete after 1s [id=data-engineering-zoomcamp-2024-project]
134+
135+
Apply complete! Resources: 2 added, 0 changed, 0 destroyed.
136+
137+
```

‎src/requirements.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
requests==2.31.0
2+
pandas==2.2.1
3+
pyarrow==15.0.2
4+
google-cloud-storage==2.16.0

0 commit comments

Comments
 (0)