Add core app functionality

Author: andrewda

.gcloudignore

@@ -0,0 +1,4 @@
+.git
+.next
+node_modules
+data

.github/workflows/load-weather.yaml

@@ -0,0 +1,15 @@
+on:
+  schedule:
+    - cron: '45 * * * *' # Hourly at 45 minutes past the hour
+
+  workflow_dispatch:
+
+jobs:
+  load-weather:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: pip3 install -r data/requirements.txt
+      - name: Load weather data
+        run: python3 data/load_weather.py

.gitignore

@@ -34,3 +34,10 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# CIFP files
+CIFP_*
+CIFP_*.zip
+
+# Data CSVs
+data/*.csv

README.md

@@ -1,38 +1,3 @@
-This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+# In the Soup ☁️
 
-## Getting Started
-
-First, run the development server:
-
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-```
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
-
-[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
-
-The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
-
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
-
-## Learn More
-
-To learn more about Next.js, take a look at the following resources:
-
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
-
-## Deploy on Vercel
-
-The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
-
-Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.
+Find nearby instrument approaches in IMC.

app.yaml

@@ -0,0 +1,7 @@
+env: standard
+runtime: nodejs18
+service: default
+handlers:
+  - url: /.*
+    secure: always
+    script: auto

data/load_cifp.py

@@ -0,0 +1,115 @@
+"""
+Parse the FAA's CIFP file to extract airport and approach/FAF data.
+"""
+
+import json
+import sys
+
+import arinc424.record as a424
+import google.auth
+import pandas as pd
+import pandas_gbq
+from tqdm import tqdm
+
+credentials, project = google.auth.default()
+
+
+def dms_to_dd(dms):
+  """Convert a DMS string to a decimal degree float.
+  """
+
+  if dms[0] == 'N' or dms[0] == 'E':
+    sign = 1
+  else:
+    sign = -1
+
+  dms = dms[1:].rjust(9, '0')
+
+  d = int(dms[0:3])
+  m = int(dms[3:5])
+  s = int(dms[5:9]) / 100
+
+  return sign * (d + m / 60 + s / 3600)
+
+
+def parse_cifp(file_path):
+  """Parse the CIFP file and return pandas DataFrames containing the airport and FAF data.
+  @param file_path: The path to the CIFP file
+  @return: A tuple containing a pandas DataFrame for the airport data and a pandas DataFrame for the FAF data
+  """
+
+  cifp = open(file_path, 'r').readlines()
+  records = []
+
+  print('Reading CIFP records...')
+
+  for line in tqdm(cifp):
+    record = a424.Record()
+    record.read(line)
+    records.append(record)
+
+  print('Extracting relevant CIFP data...')
+
+  apts = []
+  fafs = []
+
+  for record in tqdm(records):
+    is_apt = False
+    is_faf = False
+
+    for f in record.fields:
+      if f.name == 'Section Code' and f.value == 'PA':
+        is_apt = True
+        break
+
+      if f.name == 'Waypoint Description Code' and f.value == 'E  F':
+        is_faf = True
+        break
+
+    if is_apt:
+      apts.append(json.loads(record.json()))
+
+    if is_faf:
+      fafs.append(json.loads(record.json()))
+
+  df_apt = pd.DataFrame(apts)
+  df_apt = df_apt.apply(lambda x: x.str.strip())
+  df_apt['Latitude'] = df_apt['Airport Reference Pt. Latitude'].apply(lambda x: dms_to_dd(x))
+  df_apt['Longitude'] = df_apt['Airport Reference Pt. Longitude'].apply(lambda x: dms_to_dd(x))
+
+  df_faf = pd.DataFrame(fafs)
+  df_faf = df_faf.apply(lambda x: x.str.strip())
+
+  return df_apt, df_faf
+
+
+if __name__ == '__main__':
+  file_path = sys.argv[1]
+
+  df_apt, df_faf = parse_cifp(file_path)
+
+  # Remove special characters from the column names
+  df_apt.columns = df_apt.columns.str.replace(r'[^a-zA-Z0-9_ ]', '', regex=True)
+  df_faf.columns = df_faf.columns.str.replace(r'[^a-zA-Z0-9_ ]', '', regex=True)
+
+  # Condense multiple spaces in the column names
+  df_apt.columns = df_apt.columns.str.replace(r' +', '_', regex=True)
+  df_faf.columns = df_faf.columns.str.replace(r' +', '_', regex=True)
+
+  # Set column types
+  df_apt = df_apt.mask(df_apt == '')
+  columns = ['Longest_Runway', 'Airport_Elevation', 'Transition_Altitude', 'Transition_Level']
+  df_apt[columns] = df_apt[columns].apply(pd.to_numeric, errors='coerce')
+
+  df_faf = df_faf.mask(df_faf == '')
+  columns = ['RNP', 'Arc_Radius', 'Theta', 'Rho', 'Magnetic_Course', 'Route_Holding_Distance_or_Time', 'Altitude', 'Altitude_2', 'Speed_Limit', 'Transition_Altitude', 'Vertical_Angle']
+  df_faf[columns] = df_faf[columns].apply(pd.to_numeric, errors='coerce')
+
+  df_apt.to_csv('apt.csv', index=False)
+  df_faf.to_csv('faf.csv', index=False)
+
+  print('Uploading to BigQuery...')
+
+  # Upload to BigQuery
+  pandas_gbq.to_gbq(df_apt, 'aeronautical.airport', project, if_exists='replace', credentials=credentials)
+  pandas_gbq.to_gbq(df_faf, 'aeronautical.faf', project, if_exists='replace', credentials=credentials)

data/load_weather.py

@@ -0,0 +1,146 @@
+"""
+Loads weather data from NOAA and uploads it to BigQuery.
+"""
+
+from datetime import datetime, timedelta
+from ftplib import FTP
+from io import StringIO
+
+import google.auth
+import pandas as pd
+import pandas_gbq
+from tqdm import tqdm
+
+credentials, project = google.auth.default()
+
+
+def get_weather_data():
+  """Get weather data from NOAA.
+  @return: Tuple of (nbh, nbs) where nbh is the weather data for the next 24 hours
+  and nbs is the weather data for the next 72 hours (in 3hr increments).
+  """
+
+  print('Accessing NOAA FTP server...')
+
+  ftp = FTP('ftp.ncep.noaa.gov')
+  ftp.login()
+
+  ftp.cwd('pub/data/nccf/com/blend/prod')
+
+  # Get forecast days
+  forecast_days = ftp.nlst()
+  forecast_day = forecast_days[-1]
+  ftp.cwd(forecast_day)
+
+  # Get forecast hours
+  forecast_hours = ftp.nlst()
+  forecast_hour = forecast_hours[-1]
+
+  print(f'Downloading NBH forecast {forecast_day} {forecast_hour}Z...')
+
+  nbh_str = StringIO()
+  ftp.retrlines('RETR {time}/text/blend_nbhtx.t{time}z'.format(time=forecast_hour), lambda line: nbh_str.write(line + '\n'))
+  nbh = nbh_str.getvalue()
+  nbh_str.close()
+
+  print(f'Downloading NBS forecast {forecast_day} {forecast_hour}Z...')
+
+  nbs_str = StringIO()
+  ftp.retrlines('RETR {time}/text/blend_nbstx.t{time}z'.format(time=forecast_hour), lambda line: nbs_str.write(line + '\n'))
+  nbs = nbs_str.getvalue()
+  nbs_str.close()
+
+  return nbh, nbs
+
+
+def parse_weather_data(data, fmt):
+  """Parse weather data for a specific location
+  @param data: The weather data to parse
+  @param fmt: The format of the data. Either 'nbh' or 'nbs'
+  @return: A pandas DataFrame containing the weather data
+  """
+
+  location = data.strip().split('\n')[0].split(' ')[0]
+
+  forecast_date = None
+  if fmt == 'nbh':
+    forecast_date_str = data.strip().split('\n')[0].strip().split(' ')
+    forecast_date_str = list(filter(len, forecast_date_str))[-3:]
+    forecast_date_str = ' '.join(forecast_date_str)
+    forecast_date = datetime.strptime(forecast_date_str, '%m/%d/%Y %H%M %Z')
+    first_date = forecast_date + timedelta(hours=1)
+  elif fmt == 'nbs':
+    forecast_date_str = data.strip().split('\n')[0].strip().split(' ')
+    forecast_date_str = list(filter(len, forecast_date_str))[-3:]
+    forecast_date_str = ' '.join(forecast_date_str)
+    utc_hour = int(forecast_date_str[-8:-6])
+    forecast_date = datetime.strptime(forecast_date_str, '%m/%d/%Y %H%M %Z')
+    first_date = forecast_date + timedelta(hours=6 - (utc_hour % 3))
+
+  skip_lines = 1 if fmt == 'nbh' else 2
+
+  lines = data.strip().split('\n')[skip_lines:]
+  parsed_data = {}
+
+  max_len = max([len(line) for line in lines])
+  for line in lines:
+    var_name = line[:5].strip()
+    value_str = line[5:]
+
+    # Split the values into a list of integers every 3 characters
+    values = [int(value_str[i:i+3].strip()) if value_str[i:i+3].strip().lstrip('-').isdigit() else None for i in range(0, max_len - 5, 3)]
+
+    parsed_data[var_name] = values
+
+  df = pd.DataFrame(parsed_data)
+  df['Location'] = location
+  df['Forecast_Time'] = forecast_date
+
+  date = first_date
+  dates = []
+
+  prev_hr = None
+  for hr in df['UTC']:
+    date = date.replace(hour=hr)
+
+    if prev_hr is not None and hr < prev_hr:
+      date += timedelta(days=1)
+
+    dates.append(date)
+    prev_hr = hr
+
+  df['Time'] = dates
+
+  return df
+
+
+if __name__ == '__main__':
+  nbh, nbs = get_weather_data()
+
+  # Split the data into separate locations
+  nbh = nbh.strip().split(' ' * 50)[1:]
+  nbs = nbs.strip().split(' ' * 50)[1:]
+
+  print('Parsing NBH forecast data...')
+
+  df_nbh = pd.DataFrame()
+  for forecast in tqdm(nbh):
+    location_forecast = parse_weather_data(forecast, 'nbh')
+    df_nbh = pd.concat([df_nbh, location_forecast])
+
+  df_nbh.to_csv('wx_nbh.csv', index=False)
+
+  print('Parsing NBS forecast data...')
+
+  df_nbs = pd.DataFrame()
+  for forecast in tqdm(nbs):
+    location_forecast = parse_weather_data(forecast, 'nbs')
+    df_nbs = pd.concat([df_nbs, location_forecast])
+
+  df_nbs.to_csv('wx_nbs.csv', index=False)
+
+  print('Uploading to BigQuery...')
+
+  # Upload to BigQuery
+  pandas_gbq.to_gbq(df_nbh, 'weather.nbh', project, if_exists='replace', credentials=credentials)
+  pandas_gbq.to_gbq(df_nbs, 'weather.nbs', project, if_exists='replace', credentials=credentials)

data/requirements.txt

@@ -0,0 +1,5 @@
+arinc424 @ git+https://github.com/andrewda/arinc424@996025203aebf0a3794d2e337fe536ff4f4845b0
+google-auth==2.16.3
+pandas==1.5.3
+pandas-gbq==0.19.1
+tqdm==4.64.1

next.config.js

@@ -1,6 +1,7 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   reactStrictMode: true,
+  distDir: 'build',
 }
 
 module.exports = nextConfig