parse-fares

#!/usr/bin/env python

from datetime import date
import json

from split.parse import unpack, cnv_full_date, loop, fare_file
from split.utils import ticket_types

lookups = {
    'F': ( ('from', 4), ('to', 4), ('route', 5), ('status', 3), ('usage', 1), ('direction', 1), ('todate', 8, cnv_full_date), ('fromdate', 8, cnv_full_date), ('toc', 3), ('london', 1), ('nsd', 1), ('pub', 1), ('id', 7) ),
    'T': ( ('id', 7), ('type', 3), ('price', 8), ('restriction', 2) ),
}

data = {}
data_by_id = {}
for row in loop(fare_file('FFL')):
    record_type = row[0]
    row = unpack(row[1:], *lookups[record_type])

    if record_type == 'T':
        if row['id'] not in data_by_id: continue
        if row['type'] not in ticket_types: continue
        data_by_id[row['id']]['prices'][row['type']] = ( row['price'], row['restriction'] )
    else:
        if row['usage'] == 'C': continue
        if row['nsd'] != '0': continue
        if row['todate'] < date.today(): continue
        if row['fromdate'] > date.today(): continue
        assert row['status'] == '000'
        del row['todate'], row['fromdate'], row['status'], row['nsd'], row['usage'], row['pub']
        d = {
            'prices': {},
            'toc': row['toc'],
            'route': row['route'],
            #'direction': row['direction'],
        }
        data.setdefault(row['from'], {}).setdefault(row['to'], []).append(d)
        if row['direction'] == 'R':
            data.setdefault(row['to'], {}).setdefault(row['from'], []).append(d)

        data_by_id[row['id']] = d
        #print row['id'], row['from'], row['to'], d

for fr, d1 in list(data.items()):
    for to, d2 in list(d1.items()):
        for i, d in enumerate(d2):
            if not d['prices']:
                del d2[i]
        if not d2:
            del d1[to]
    if not d1:
        del data[fr]

for fr, d1 in data.items():
    with open('data/fares/%s.json' % fr, 'w') as fp:
        json.dump(d1, fp, separators=(',', ': '), indent=2, sort_keys=True)