Task-2-profiling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import modin.pandas as dd\n",
    "import json\n",
    "import os\n",
    "import string\n",
    "import re\n",
    "from fuzzywuzzy import fuzz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "contents=['vwxi-2r5k.BOROUGH.txt.gz', 'gez6-674h.BORO.txt.gz', 'urc8-8bjy.BORO.txt.gz', 'crc3-tcnm.BOROUGH.txt.gz', 'm59i-mqex.QUEENS_CONDOMINIUM_PROPERTY_Building_Classification.txt.gz', 'en2c-j6tw.BRONX_CONDOMINIUM_PROPERTY_Building_Classification.txt.gz', 'f42p-xqaa.BROOKLYN___COOPERATIVES_COMPARABLE_PROPERTIES___Building_Classification.txt.gz', 'niy5-4j7q.MANHATTAN___COOPERATIVES_COMPARABLE_PROPERTIES___Building_Classification.txt.gz', '3btx-p4av.MANHATTAN___COOPERATIVES_COMPARABLE_PROPERTIES___Building_Classification.txt.gz', '5mw2-hzqx.BROOKLYN_CONDOMINIUM_PROPERTY_Building_Classification.txt.gz', '4s7y-vm5x.CORE_SUBJECT.txt.gz', 'yg5a-hytu.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', 'a2pm-dj2w.Borough.txt.gz', 'yahh-6yjc.School_Type.txt.gz', 'wcmg-48ep.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', 'ccgt-mp8e.Borough.txt.gz', 'vwxi-2r5k.CORE_SUBJECT.txt.gz', '3aka-ggej.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', 'fb26-34vu.CORE_SUBJECT__MS_CORE_and_9_12_ONLY_.txt.gz', '4qii-5cz9.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', 'rmv8-86p4.BROOKLYN_CONDOMINIUM_PROPERTY_Building_Classification.txt.gz', 'qusa-igsv.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', '4hiy-398i.CORE_SUBJECT___MS_CORE_and__09_12_ONLY_.txt.gz', 'crc3-tcnm.CORE_SUBJECT.txt.gz', 'dtmw-avzj.BOROUGH.txt.gz', 'i8ys-e4pm.CORE_SUBJECT_9_12_ONLY_.txt.gz', 'pdpg-nn8i.CORE_SUBJECT___MS_CORE_and__9_12_ONLY_.txt.gz', 'fzv4-jan3.SCHOOL_LEVEL_.txt.gz', 'weg5-33pj.SCHOOL_LEVEL_.txt.gz', 'yayv-apxh.SCHOOL_LEVEL_.txt.gz', 'dj4e-3xrn.SCHOOL_LEVEL_.txt.gz', 'vrn4-2abs.SCHOOL_LEVEL_.txt.gz', 'cgz5-877h.SCHOOL_LEVEL_.txt.gz', 'mrxb-9w9v.BOROUGH___COMMUNITY.txt.gz', '72ss-25qh.Borough.txt.gz', 'vhah-kvpj.Borough.txt.gz', '9z9b-6hvk.Borough.txt.gz', 'upwt-zvh3.SCHOOL_LEVEL_.txt.gz', '4n2j-ut8i.SCHOOL_LEVEL_.txt.gz', '8eux-rfe8.INPUT_1_Borough.txt.gz', 'ffnc-f3aa.SCHOOL_LEVEL_.txt.gz', 'mjux-q9d4.SCHOOL_LEVEL_.txt.gz', 'xne4-4v8f.SCHOOL_LEVEL_.txt.gz', '6je4-4x7e.SCHOOL_LEVEL_.txt.gz', '956m-xy24.COMPARABLE_RENTAL_2__Building_Classification.txt.gz', 'cvh6-nmyi.SCHOOL_LEVEL_.txt.gz', 'mdcw-n682.Middle_Initial.txt.gz', '3btx-p4av.COMPARABLE_RENTAL___1__Building_Classification.txt.gz', 'niy5-4j7q.COMPARABLE_RENTAL___2__Building_Classification.txt.gz', 'sv2w-rv3k.BORO.txt.gz', 'e9xc-u3ds.CANDMI.txt.gz', 'kwmq-dbub.CANDMI.txt.gz', 'cyfw-hfqk.STATEN_ISLAND_CONDOMINIUM_PROPERTY_Neighborhood.txt.gz', 'a5qt-5jpu.STATEN_ISLAND_CONDOMINIUM_PROPERTY_Neighborhood.txt.gz', 'k3cd-yu9d.CANDMI.txt.gz', '735p-zed8.CANDMI.txt.gz', 'qu8g-sxqf.MI.txt.gz', 'vx8i-nprf.MI.txt.gz', 'a9md-ynri.MI.txt.gz', 'uq7m-95z8.interest6.txt.gz', 'vw9i-7mzq.interest4.txt.gz', 's3k6-pzi2.interest4.txt.gz', 's3k6-pzi2.interest5.txt.gz', '7jkp-5w5g.Agency.txt.gz', 'brga-xeqy.Agency.txt.gz', 'pdk7-puui.Agency.txt.gz', 'n2s5-fumm.BRONX_CONDOMINIUM_PROPERTY_Neighborhood.txt.gz', 'i8ys-e4pm.CORE_COURSE_9_12_ONLY_.txt.gz', 'mqdy-gu73.Color.txt.gz', 'd3ge-anaz.CORE_COURSE__MS_CORE_and_9_12_ONLY_.txt.gz', 'jhjm-vsp8.Agency.txt.gz', 'q5za-zqz7.Agency.txt.gz', 'f3cg-u8bv.Agency.txt.gz', 'fxdy-q85h.Agency.txt.gz', '6wcu-cfa3.CORE_COURSE__MS_CORE_and_9_12_ONLY_.txt.gz', 'vw9i-7mzq.interest3.txt.gz', 'vw9i-7mzq.interest1.txt.gz', 's3k6-pzi2.interest1.txt.gz', 'ge8j-uqbf.interest.txt.gz', 'i9pf-sj7c.INTEREST.txt.gz', 'ruce-cnp6.Agency.txt.gz', 'uq7m-95z8.interest1.txt.gz', 'n84m-kx4j.VEHICLE_MAKE.txt.gz', 's3zn-tf7c.QUEENS_CONDOMINIUM_PROPERTY_Neighborhood.txt.gz', 'h9gi-nx95.VEHICLE_TYPE_CODE_5.txt.gz', 'ph7v-u5f3.WEBSITE.txt.gz', '3miu-myq2.COMPARABLE_RENTAL___2__Neighborhood.txt.gz', '4d7f-74pe.Agency.txt.gz', 'yrf7-4wry.COMPARABLE_RENTAL___2___Neighborhood.txt.gz', 'niy5-4j7q.MANHATTAN___COOPERATIVES_COMPARABLE_PROPERTIES___Neighborhood.txt.gz', '3btx-p4av.COMPARABLE_RENTAL___1__Neighborhood.txt.gz', 'w6yt-hctp.BROOKLYN_CONDOMINIUM_PROPERTY_Neighborhood.txt.gz', 'h9gi-nx95.VEHICLE_TYPE_CODE_4.txt.gz', 'qpm9-j523.org_neighborhood.txt.gz', 'ajxm-kzmj.NeighborhoodName.txt.gz', '8gqz-6v9v.Website.txt.gz', 'h9gi-nx95.VEHICLE_TYPE_CODE_3.txt.gz', '4kpn-sezh.Website.txt.gz', 'ph7v-u5f3.TOP_VEHICLE_MODELS___5.txt.gz', '5uac-w243.PREM_TYP_DESC.txt.gz', 'qgea-i56i.PREM_TYP_DESC.txt.gz', '7yay-m4ae.AGENCY_NAME.txt.gz', 'cvse-perd.Agency_Name.txt.gz', 'uq7m-95z8.neighborhood.txt.gz', 'mdcw-n682.Agency_Website.txt.gz', 'as69-ew8f.StartCity.txt.gz', '2v9c-2k7f.DBA.txt.gz', 'myei-c3fa.Neighborhood.txt.gz', 'as69-ew8f.TruckMake.txt.gz', 'myei-c3fa.Neighborhood_2.txt.gz', '5fn4-dr26.City.txt.gz', '2bmr-jdsv.DBA.txt.gz', 'h9gi-nx95.VEHICLE_TYPE_CODE_2.txt.gz', 'h9gi-nx95.VEHICLE_TYPE_CODE_1.txt.gz', 'va74-3m6c.company_phone.txt.gz', 'uqxv-h2se.independentwebsite.txt.gz', 'pvqr-7yc4.Vehicle_Body_Type.txt.gz', 'pvqr-7yc4.Vehicle_Body_Type.txt.gz', 'pvqr-7yc4.Vehicle_Color.txt.gz', 's3k6-pzi2.website.txt.gz', 'w9ak-ipjd.City.txt.gz', 'n3p6-zve2.website.txt.gz', 'sxx4-xhzg.Park_Site_Name.txt.gz', 'ipu4-2q9a.Site_Safety_Mgr_s_First_Name.txt.gz', 'aiww-p3af.Incident_Zip.txt.gz', '6rrm-vxj9.parkname.txt.gz', 'dm9a-ab7w.APPLICANT_FIRST_NAME.txt.gz', '3rfa-3xsf.Incident_Zip.txt.gz', 'uzcy-9puk.Incident_Zip.txt.gz', 'e9xc-u3ds.ZIP.txt.gz', '72ss-25qh.Agency_ID.txt.gz', 'urzf-q2g5.Phone_Number.txt.gz', 'pqg4-dm6b.Phone.txt.gz', 'kwmq-dbub.CITY.txt.gz', '7btz-mnc8.Provider_First_Name.txt.gz', 'w9ak-ipjd.Filing_Representative_First_Name.txt.gz', 's79c-jgrm.City.txt.gz', 'pvqr-7yc4.Vehicle_Make.txt.gz', 'pvqr-7yc4.Vehicle_Make.txt.gz', 'faiq-9dfq.Vehicle_Body_Type.txt.gz', 'faiq-9dfq.Vehicle_Body_Type.txt.gz', '8586-3zfm.School_Name.txt.gz', 'faiq-9dfq.Vehicle_Color.txt.gz', '2bnn-yakx.Vehicle_Body_Type.txt.gz', '2bnn-yakx.Vehicle_Body_Type.txt.gz', 'vr8p-8shw.DVT_MAKE.txt.gz', 'bty7-2jhb.Site_Safety_Mgr_s_Last_Name.txt.gz', '6anw-twe4.FirstName.txt.gz', 'dpm2-m9mq.owner_city.txt.gz', 'mdcw-n682.First_Name.txt.gz', '7btz-mnc8.Provider_Last_Name.txt.gz', 'kiv2-tbus.Vehicle_Body_Type.txt.gz', 'qcdj-rwhu.BUSINESS_NAME2.txt.gz', 'emnd-d8ba.PRINCIPAL_PHONE_NUMBER.txt.gz', 'ahjc-fdu3.PRINCIPAL_PHONE_NUMBER.txt.gz', 'a5td-mswe.Vehicle_Color.txt.gz', '9ck8-hj3u.PRINCIPAL_PHONE_NUMBER.txt.gz', '2bnn-yakx.Vehicle_Color.txt.gz', 'xck4-5xd5.website.txt.gz', 'kiv2-tbus.Vehicle_Color.txt.gz', 'pqg4-dm6b.Address1.txt.gz', 'ipu4-2q9a.Site_Safety_Mgr_s_Last_Name.txt.gz', 'r4c5-ndkx._Phone.txt.gz', 'cspg-yi7g.PHONE.txt.gz', 'jt7v-77mi.Vehicle_Color.txt.gz', 'c284-tqph.Vehicle_Color.txt.gz', 'iz2q-9x8d.Owner_s__Phone__.txt.gz', 'erm2-nwe9.Incident_Zip.txt.gz', 'ci93-uc8s.Vendor_DBA.txt.gz', 'sqcr-6mww.Park_Facility_Name.txt.gz', 'w9ak-ipjd.Applicant_Last_Name.txt.gz', 'ebkm-iyma.Street_Address.txt.gz', 'bbs3-q5us.ZIP.txt.gz', 't8hj-ruu2.License_Business_City.txt.gz', '6anw-twe4.LastName.txt.gz', '446w-773i.Address_1.txt.gz', 'sxmw-f24h.School_Name.txt.gz', 's9d3-x4fz.ZIP.txt.gz', '8gpu-s594.SCHOOL_NAME.txt.gz', 'cspg-yi7g.ADDRESS.txt.gz', 'szkz-syh6.Prequalified_Vendor_Address.txt.gz', 'ptev-4hud.City.txt.gz', 'mdcw-n682.Last_Name.txt.gz', 'wks3-66bn.School_Name.txt.gz', '8k4x-9mp5.Last_Name__only_2014_15_.txt.gz', 'mq9d-au8i.School_Name.txt.gz', 'uchs-jqh4.School_Name.txt.gz', 'rtws-c2ai.School_Name.txt.gz', 'hy4q-igkk.Park_Facility_Name.txt.gz', '5cd6-v74i.School_Name.txt.gz', 'mjux-q9d4.SCHOOL.txt.gz', 'cvh6-nmyi.SCHOOL.txt.gz', 's27g-2w3u.School_Name.txt.gz', '735p-zed8.CITY.txt.gz', '2bnn-yakx.Vehicle_Make.txt.gz', '2bnn-yakx.Vehicle_Make.txt.gz', 'dm9a-ab7w.AUTH_REP_FIRST_NAME.txt.gz', 'kiv2-tbus.Vehicle_Make.txt.gz', 'kiv2-tbus.Vehicle_Make.txt.gz', 'i6b5-j7bu.TOSTREETNAME.txt.gz', '922w-z7da.address_1.txt.gz', 'jt7v-77mi.Vehicle_Make.txt.gz', 'y4fw-iqfr.Address.txt.gz', 'w9ak-ipjd.Owner_s_Business_Name.txt.gz', 'bjuu-44hx.DVV_MAKE.txt.gz', 'c284-tqph.Vehicle_Make.txt.gz', 'c284-tqph.Vehicle_Make.txt.gz', 'pq5i-thsu.DVC_MAKE.txt.gz', 'aiww-p3af.School_Name.txt.gz', 'fbaw-uq4e.Location_1.txt.gz', '2sps-j9st.PERSON_FIRST_NAME.txt.gz', 'uzcy-9puk.School_Name.txt.gz', '3rfa-3xsf.Park_Facility_Name.txt.gz', 'ic3t-wcy2.Zip.txt.gz', '9b9u-8989.Establishment_Street.txt.gz', '4twk-9yq2.CrossStreet2.txt.gz', '4twk-9yq2.CrossStreet2.txt.gz', 'dm9a-ab7w.STREET_NAME.txt.gz', 'uwyv-629c.StreetName.txt.gz', 'w9if-3pyn.School_Name.txt.gz', 'ipu4-2q9a.Owner_s_House_Zip_Code.txt.gz', 'yjub-udmw.Location__Lat__Long_.txt.gz', 'ydkf-mpxb.CrossStreetName.txt.gz', 'dm9a-ab7w.AUTH_REP_LAST_NAME.txt.gz', '9b9u-8989.DBA.txt.gz', 'f4rp-2kvy.Street.txt.gz', '52dp-yji6.Owner_First_Name.txt.gz', 'kj4p-ruqc.StreetName.txt.gz', 'erm2-nwe9.Landmark.txt.gz', 'erm2-nwe9.Park_Facility_Name.txt.gz', '6bgk-3dad.RESPONDENT_CITY.txt.gz', 'bdjm-n7q4.CrossStreet2.txt.gz', 'ci93-uc8s.telephone.txt.gz', 'tg3t-nh4h.BusinessName.txt.gz', 'yvxd-uipr.Location_1.txt.gz', 'nhms-9u6g.Name__Last__First_.txt.gz', 't8hj-ruu2.First_Name.txt.gz', '4e2n-s75z.Address.txt.gz', 'ci93-uc8s.Website.txt.gz', 'sqcr-6mww.Cross_Street_2.txt.gz', 'xubg-57si.OWNER_BUS_STREET_NAME.txt.gz', 'jz4z-kudi.Violation_Location__City_.txt.gz', 'hy4q-igkk.Cross_Street_1.txt.gz', 'sxmw-f24h.Cross_Street_1.txt.gz', '3rfa-3xsf.Cross_Street_2.txt.gz', 'ic3t-wcy2.Applicant_s_First_Name.txt.gz', '8u86-bviy.Address_1__self_reported_.txt.gz', 'hy4q-igkk.Intersection_Street_2.txt.gz', 'tqtj-sjs8.FromStreetName.txt.gz', 'uzcy-9puk.Intersection_Street_2.txt.gz', 'p937-wjvj.HOUSE_NUMBER.txt.gz', 'aiww-p3af.Street_Name.txt.gz', 'jz4z-kudi.Violation_Location__Zip_Code_.txt.gz', '3rfa-3xsf.Street_Name.txt.gz', 'uzcy-9puk.Street_Name.txt.gz', 'jz4z-kudi.Respondent_Address__Zip_Code_.txt.gz', '52dp-yji6.Owner_Last_Name.txt.gz', 'jz4z-kudi.Respondent_Address__City_.txt.gz', 'sxmw-f24h.Street_Name.txt.gz', '2sps-j9st.PERSON_LAST_NAME.txt.gz', 'qu8g-sxqf.First_Name.txt.gz', 'n2mv-q2ia.Address_1__self_reported_.txt.gz', 'ji82-xba5.address.txt.gz', '43nn-pn8j.DBA.txt.gz', 't8hj-ruu2.Business_Phone_Number.txt.gz', 'bs8b-p36w.LOCATION.txt.gz', 'mu46-p9is.Location.txt.gz', 'p937-wjvj.LOCATION.txt.gz', 'sqcr-6mww.Location.txt.gz', 'aiww-p3af.Location.txt.gz', 'bdjm-n7q4.Location.txt.gz', '4pt5-3vv4.Location.txt.gz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different business types encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different business types.\n",
    "#We use a regex function to evaluate if the string passed in the function is present in the dictionary.\n",
    "def ValidateBusiness(val):\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0  \n",
    "    Dict_BusinessTypes = ['inc','inc.','llc','l.l.c.','p.c.','pc','lp','l.p.','corp','corp.','corporation','company','co.','co','ltd','ltd.','capital','holdings','services']\n",
    "    count_Business=0\n",
    "    for itr in Dict_BusinessTypes:\n",
    "        final=fuzz.ratio(itr, val.lower())\n",
    "        if final>=90:\n",
    "            count_Business=1\n",
    "    #Search using Regular Expressions\n",
    "    if re.search(r\"^(?!\\s)(?!.*\\s$)(?=.*[a-zA-Z0-9])[a-zA-Z0-9 '~?!]{2,}$\", val):\n",
    "        count_Business=1\n",
    "    return count_Business"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validatess the different names of people encountered in the NYC OPEN data.\n",
    "#We utilize regular expresssions for validation.\n",
    "def Validate_PersonName(val):\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    if re.search(r\"^[a-zA-Z]+(([',. -][a-zA-Z ])?[a-zA-Z]*)*$\",val):\n",
    "        return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different park and playground types encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different park and playground types.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_Park(val):\n",
    "    Dict_Parks=['Barretto Point Park', 'Bicentennial Veterans Memorial Park', 'Bridge Park', 'Bronx Park', 'New York Botanical Garden', 'Bronx Zoo', 'Castle Hill Park', 'Claremont Park', 'Concrete Plant Park', 'Crotona Park', 'Devoe Park', 'Estella Diggs Park', 'Ewen Park', 'Ferry Point Park', 'Givans Creek Woods', 'Grant Park', 'Haffen Park', 'Harding Park', 'Harris Park', 'Henry Hudson Park', 'Hunts Point Riverside Park', 'Jerome Park', 'Joseph Rodman Drake Park', 'Joyce Kilmer Park', 'Julius Richman Park', 'Macombs Dam Park', 'Mill Pond Park', 'Mullaly Park', 'Pelham Bay Park', 'Orchard Beach', \"Rodman's Neck\", 'Playground 52', 'Poe Park, including Edgar Allan Poe Cottage', \"Printer's Park\", 'Pugsley Creek Park', 'Rainey Park', 'Raoul Wallenberg Forest', 'Richman Park', 'Riverdale Park', 'Roberto Clemente State Park (NYSP)', 'Seton Falls Park', 'Seton Park', 'Soundview Park', 'Spuyten Duyvil Shorefront Park', 'St. James Park', \"St. Mary's Park\", 'Starlight Park', 'Tremont Park', 'University Woods', 'Van Cortlandt Park', 'Vidalia Park', 'Vinmont Veteran Park', \"Washington's Walk\", 'Wave Hill', 'Williamsbridge Oval', 'American Veterans Memorial Pier', 'Amersfort Park', 'Asser Levy Park', 'Bath Beach Park', 'Betsy Head Park', 'Bensonhurst Park', 'Breukelen Ballfields Park', 'Brooklyn Botanic Garden', 'Brooklyn Bridge Park', 'Brooklyn Heights Promenade', 'Brooklyn–Queens Greenway', 'Brooklyn Waterfront Greenway', 'Brower Park', 'Bushwick Inlet Park', 'Byrne Park', 'Cadman Plaza', 'Canarsie Park', 'Carroll Park', 'Calvert Vaux Park', 'City Line Park', 'Coffey Park', 'Columbus Park', 'Commodore Barry Park', 'Coney Island Beach & Boardwalk', 'Coney Island Creek Park', 'Continental Army Plaza', 'Cooper Park', 'Domino Park', 'Dyker Beach Park and Golf Course', 'East River State Park (NYSP)', 'Floyd Bennett Field (NPS - Gateway National Recreation Area)', 'Fort Greene Park', 'Four Sparrow Marsh', 'Frank M. Charles Memorial Park', 'Fresh Creek Nature Preserve', 'Friends Field', 'Fulton Park', 'Gravesend Park', 'Green Central Knoll', 'Herbert Von King Park', 'Highland Park', 'Irving Square Park', 'JJ Byrne Park', 'John J. Carty Park', 'John Paul Jones Park', 'Kaiser Park', 'Kelly Park', 'Leif Ericson Park', 'Leon S. Kaiser Playground', 'Lieutenant Joseph Petrosino Park', 'Lincoln Terrace Park', 'Linden Park', 'Lindower Park', 'Linton Park', 'Macri Triangle', 'Manhattan Beach Park', 'Maria Hernandez Park', 'Marine Park', 'McCarren Park', 'McGolrick Park', 'McKinley Park', 'Mount Prospect Park', \"Owl's Head Park\", 'Paerdegat Basin Park', 'Paerdegat Park', 'Prospect Park', 'Red Hook Park', 'Robert E. Venable Park', 'Saratoga Park', 'Scarangella Park', 'Seaside Park', 'Seth Low Park', 'Shore Park and Parkway', 'Spring Creek Park', 'Sternberg Park', 'St. Johns Park', 'Sunset Park', 'Tompkins Park', 'Trinity Park', 'Van Voorhees Park', 'Walt Whitman Park', 'Wingate Park', 'WNYC Transmitter Park', 'Zion Triangle', '79th Street Boat Basin', 'Abingdon Square Park', 'Albert Capsouto Park', 'Asphalt Green', 'The Battery', 'Bella Abzug (Hudson) Park', 'Bennett Park', 'Blackwell Island Light, Roosevelt Island', 'British Garden at Hanover Square', 'Bowling Green', 'Bryant Park', 'Captain Patrick J. Brown Walk', 'Carl Schurz Park', 'Central Park', 'Chelsea Park', 'City Hall Park', 'Collect Pond', 'Columbus Park', 'Corlears Hook', 'Dag Hammarskjold Plaza', 'Damrosch Park', 'Dante Park', 'DeSalvio Playground', 'DeWitt Clinton Park', 'Drumgoole Plaza', 'Duane Park', 'East River Greenway', 'East River Esplanade', 'East River Park', 'Stuyvesant Cove', 'Father Demo Square', 'Finn Square', 'Foley Square', 'Fort Tryon Park', 'Fort Washington Park', 'Franklin D. Roosevelt Four Freedoms Park (NYSP)', 'Gorman Park', 'Governors Island', 'Governors Island National Monument (NPS)', 'Gramercy Park', 'Greenacre Park', 'Hamilton Fish Park', 'Hanover Square', 'Harlem River Park', \"Hell's Kitchen Park\", 'Herald Square', 'High Line', 'Highbridge Park', 'Holcombe Rucker Park', 'Hudson River Park', 'Imagination Playground at Burling Slip', 'Inwood Hill Park', 'Isham Park', 'Jay Hood Wright Park', 'Jackie Robinson Park', 'Jackson Square Park', 'John Jay Park', 'Liberty Park', 'Louis Cuvillier Park', 'Madison Square', 'Manhattan Waterfront Greenway', 'Marcus Garvey Park', 'Mill Rock Park', 'Mitchell Square Park', 'Morningside Park', \"Murphy's Brother's Playground\", 'Muscota Marsh', 'Paley Park', 'Peretz Square', 'Peter Cooper Park', 'Plaza Lafayette', 'Queen Elizabeth II September 11th Garden', 'Ralph Bunche Park', 'Randalls and Wards Islands', 'Richard Tucker Square', 'Riverbank State Park (NYSP)', 'Riverside Park', 'Robert Moses Playground', 'Roosevelt Triangle', 'Rucker Park', 'Sakura Park', 'Samuel N. Bennerson 2nd Playground', 'Sara Delano Roosevelt Park', 'Septuagesimo Uno', 'Seward Park', 'Sheridan Square', 'Sherman Square', 'Southpoint Park, Roosevelt Island', 'St. Nicholas Park', 'St. Vartan Park', 'Straus Park', 'Stuyvesant Square', 'Swindler Cove Park', 'Teardrop Park', 'Theodore Roosevelt Park', 'Thomas Jefferson Park', 'Thomas Paine Park', 'Tompkins Square Park', 'Tribeca Park', 'Union Square', 'Verdi Square', 'Vesuvio Playground', 'Randalls and Wards Islands', 'Washington Market Park', 'Washington Square Park', 'West Side Community Garden', 'Winston Churchill Park', 'Zuccotti Park', 'Admiral Park', 'Alley Park', 'Alley Pond Park', 'Andrews Grove', 'Astoria Park', 'Baisley Pond Park', 'Bayside Fields', 'Bayswater Park', 'Bayswater Point State Park (NYSP)', 'Beach Channel Park', 'Big Bush Park', 'Bowne Park', 'Brant Point Wildlife Sanctuary', 'Breininger Park', 'Broad Channel American Park', 'Broad Channel Park', 'Broad Channel Wetlands', 'Brooklyn–Queens Greenway', 'Brookville Park', 'Bulova Park', 'Captain Tilly Park', 'Crocheron Park', 'Cunningham Park', 'Detective Keith L Williams Park', 'Detective William T. Gunn Park', 'Dr. Charles R. Drew Park', 'Dubos Point Wildlife Sanctuary', 'Doughboy Park', 'Douglaston Park', 'Elmhurst Park', 'Evergreen Park', 'Flushing Fields', 'Flushing Meadows–Corona Park', 'Forest Park', 'Fort Tilden (NPS - Gateway National Recreation Area)', 'Fort Totten', 'Francis Lewis Park', 'Frank Golden Park', 'Frank Principe Park', 'Gantry Plaza State Park (NYSP)', 'Haggerty Park', 'Harvey Park', 'Highland Park', 'Hinton Park', 'Hoffman Park', 'Hook Creek Park', \"Hunter's Point Park\", 'Idlewild Park', 'Jamaica Bay Park', 'Jamaica Bay Wildlife Refuge (NPS - Gateway National Recreation Area)', 'Jacob Riis Park (NPS - Gateway National Recreation Area)', 'John Golden Park', 'Juniper Valley Park', 'Kissena Park', 'Kohlreiter Square', 'Libra Triangle', 'Linnaeus Park', 'Little Bay Park', 'Louis Pasteur Park', 'Mafera Park', 'Macneil Park', 'Marconi Park', 'Marie Curie Park', 'Montbellier Park', \"O'Donohue Park\", 'Overlook Park', 'Park of the Americas', 'Phil \"Scooter\" Rizzuto', 'Police Officer Edward Byrne Park', \"Powell's Cove Park\", 'Queens Botanical Garden', 'Queens County Farm Museum', 'Queens Zoo', 'Queensbridge Park', 'Rachel Carson Playground', 'Railroad Park', 'Ralph Demarco Park', 'Rochdale Park', 'Rockaway Beach', 'Rockaway Community Park', 'Rufus King Park', 'Seagirt Avenue Wetlands', 'Socrates Sculpture Park', 'Spring Creek Park', 'Springfield Park', 'St. Albans Park', 'Travers Park', 'Triangle 54', 'Tudor Park', \"Udall's Park Preserve\", 'Wayanda Park', 'Aesop Park', 'Allison Pond Park', 'Amundsen Circle', 'Annadale Green', 'Arbutus Woods Park', 'Arden Woods', 'Alice Austen House', 'Barrett Park', 'Bayview Terrace Park', 'Blood Root Valley', 'Bloomingdale Park', 'Blue Heron Park Preserve', 'Blueberry Park', 'Bradys Park', 'Bunker Ponds Park', 'Buono Beach', 'Carlton Park', 'Clay Pit Ponds State Park Preserve (NYSP)', 'Clove Lakes Park', \"Clove's Tail\", 'Conference House Park', 'Corporal Thompson Park', 'Crescent Beach Park', 'Deere Park', 'Eibs Pond Park', 'Faber Pool and Park', 'Fairview Park', 'Father Macris Park', 'Forest Grove', 'Fort Hill Park', 'Fort Wadsworth (NPS - Gateway National Recreation Area)', 'South Beach–FDR Boardwalk', 'Freshkills Park', 'Gaeta Park', 'Great Kills Park (NPS - Gateway National Recreation Area)', 'Graniteville Quarry Park', 'Graniteville Swamp Park', 'Hero Park', 'High Rock Park Preserve', 'Huguenot Ponds Park', 'Hybrid Oak Woods Park', 'Ingram Woods', 'Jones Woods Park', 'Joseph Manna Park', 'King Fisher Park', 'Kingdom Pond Park', 'Last Chance Pond Park', 'LaTourette Park and Golf Course', 'Lemon Creek', 'Long Pond Park', 'General Douglas MacArthur Park', 'Maple Woods', 'Mariners Marsh Park', 'Meredith Woods', 'Midland Beach', 'Midland Field', 'Miller Field (NPS - Gateway National Recreation Area)', 'Mount Loretto Unique Area', 'New Dorp Beach', 'Northerleigh Park', 'Ocean Breeze Park', 'Old Place Creek Park', 'Olmsted-Beil House Park', \"Reed's Basket Willow Swamp Park\", 'Richmond Terrace Wetlands', \"Sailors' Snug Harbor\", 'Staten Island Botanical Garden', \"New York Chinese Scholar's Garden\", 'Saw Mill Creek Marsh', 'Schmul Park', 'Seaside Wildlife Nature Park', 'Siedenburg Park', 'Silver Lake', 'Sobel Court Park', 'South Beach Wetlands', 'St. George Park', 'Staten Island Greenbelt', 'Staten Island Industrial Park', 'Tappen Park', 'Tompkinsville Park', 'Tottenville Shore Park', 'Von Briesen Park', 'Walker Park', 'Wegener Park', 'Westerleigh Park', 'Westwood Park', 'Willowbrook Park', \"Wolfe's Pond Park\"]\n",
    "\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    count_park=0\n",
    "    for itr in Dict_Parks:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            count_park=1\n",
    "    return count_park"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the latitudes and logitudes encountered in the NYC OPEN data.\n",
    "#We use regular expressions to validate the inputted latitude and longitude.\n",
    "import re\n",
    "def Validate_Lat_Long(InputData):\n",
    "    try:\n",
    "        computeInput = InputData.split(',')[0][1:-1]\n",
    "        computeExpression = r'^[\\-\\+]?(0(\\.\\d{1,10})?|([1-9](\\d)?)(\\.\\d{1,10})?|1[0-7]\\d{1}(\\.\\d{1,10})?|180\\.0{1,10})$'  \n",
    "        if re.match(computeExpression, computeInput) != None:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different Zip-Codes of New York that are encountered in the NYC OPEN data.\n",
    "#We use regular expressions method for validation.\n",
    "import re\n",
    "def Validate_ZipCode(InputData):\n",
    "    if InputData==None:\n",
    "        return 0\n",
    "    if type(InputData)!=str:\n",
    "        InputData=str(InputData)\n",
    "    #Creating a regular expression\n",
    "    Compute_Expression = re.compile(r\"(^[0-9]{5}(?:-[0-9]{4})?$)\")\n",
    "    if re.fullmatch(Compute_Expression,InputData.lower()) != None:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different neighbourhood types of the different boroughs of New York-encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different neighbourhood types in the different boroughs.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_Neighbourhood(InputData):\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    Dict_Manhattan = ['Alphabet','Battery Park','Carnegie Hill','Chelsea','Chinatown','East Harlem','East Village',\\\n",
    "                'Financial District','Flatiron District','Gramercy Park','Greenwich Village','Harlem','Clinton','Inwood',\\\n",
    "                'Kips Bay','Lincoln Square','Lower East Side','Manhattan Valley','Midtown East','Midtown West',\\\n",
    "                'Morningside Heights','Murray Hill','NoLita','Little Italy','Roosevelt Island','SoHo','Tribeca','Upper East Side',\\\n",
    "                'Upper West Side','Washington Heights','West Village']\n",
    "    Dict_Bronx =['Baychester','Bedford Park','Belmont','Bronxdale','Castle Hill','City Island','Concourse Village',\\\n",
    "            'Grand Concourse','Morrisania','Country Club','Fieldston','Fordham','Hunts Point','Kingsbridge','Kingsbridge Heights',\\\n",
    "            'Melrose','Morris Heights','Morris Park','Mott Haven','Parkchester','Pelham Bay','Pelham Gardens','Pelham Parkway',\\\n",
    "            'Port Morris','Riverdale','Soundview','Throgs Neck','Tremont','University Heights','Wakefield','Williamsbridge','Woodlawn']\n",
    "    Dict_Brooklyn =['Bath Beach','Bay Ridge','Bedford-Stuyvesant','Bensonhurst','Bergen Beach','Boerum Hill','Borough Park',\\\n",
    "            'Brighton Beach','Brooklyn Heights','Brownsville','Bushwick','Canarsie','Carroll Gardens','Cobble Hill','Coney Island',\\\n",
    "            'Crown Heights','Cypress Hills','Downtown Brooklyn','Dumbo','Vinegar Hill','Dyker Heights','East New York','Flatbush',\\\n",
    "            'Flatlands','Fort Greene','Clinton Hill','Gerritsen Beach','Gowanus','Gravesend','Greenpoint','Greenwood Heights',\\\n",
    "            'Manhattan Beach','Marine Park','Midwood','Mill Basin','Park Slope','Prospect Heights','Prospect Park South','Kensington',\\\n",
    "            'Red Hook','Sea Gate','Sheepshead Bay','Sunset Park','Williamsburg','Windsor Terrace']\n",
    "    Dict_Queens = ['Arverne','Astoria','Bayside','Beechhurst','Belle Harbor','Neponsit','Bellerose','Briarwood','Broad Channel','Cambria Heights',\\\n",
    "            'College Point','Corona','Douglaston','East Elmhurst','Elmhurst','Far Rockaway','Floral Park','Flushing','Forest Hills',\\\n",
    "            'Fresh Meadows','Glen Oaks','Glendale','Hillcrest','Hollis','Hollis Hills','Howard Beach','Jackson Heights','Jamaica',\\\n",
    "            'Jamaica Estates','Jamaica Hills','Kew Gardens','Laurelton','Little Neck','Long Island City','Maspeth','Middle Village',\\\n",
    "            'Oakland Gardens','Ozone Park','Queens Village','Rego Park','Richmond Hill','Ridgewood','Rockaway Park','Rosedale',\\\n",
    "            'South Jamaica','South Ozone Park','Springfield Gardens','St. Albans','Sunnyside','Whitestone','Woodhaven','Woodside']\n",
    "    Dict_Staten = ['Annadale','Arden Heights','Arrochar','Bay Street','Bulls Head','Castleton Corners','Charleston','Clifton','Dongan Hills',\\\n",
    "            'Eltingville','Emerson Hill','Graniteville','Grant City','Grasmere','Concord','Great Kills','Grymes Hill','Huguenot',\\\n",
    "            'Livingston','Manor Heights','Mariners Harbor','Midland Beach','New Brighton','New Dorp','New Dorp Beach','New Springville',\\\n",
    "            'Oakwood','Pleasant Plains','Port Richmond','Prince Bay','Richmondtown','Rosebank','Rossville','Shore Acres','Silver Lake',\\\n",
    "            'South Beach','St. George','Stapleton','Sunnyside','Todt Hill','Tompkinsville','Tottenville','Travis','West New Brighton',\\\n",
    "            'Westerleigh','Willowbrook','Woodrow']\n",
    "    final = []\n",
    "    final.extend(Dict_Manhattan)\n",
    "    final.extend(Dict_Bronx)\n",
    "    final.extend(Dict_Brooklyn)\n",
    "    final.extend(Dict_Queens)\n",
    "    final.extend(Dict_Staten)\n",
    "    for itr in final:\n",
    "        if fuzz.ratio(itr.lower(),InputData.lower())>=85:\n",
    "            return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different colours encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different colours possible.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_Color(InputData):\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    Dict_Colors = ['white', 'black', 'pink', 'purple', 'blue', 'gray', 'green', 'beige', 'yellow', 'ivory', 'gold', \n",
    "                  'orange', 'red', 'brown', 'BK','WH','GR','RD','BL','GY','SL']  \n",
    "    for itr in Dict_Colors:\n",
    "        final=fuzz.ratio(itr.lower(), InputData.lower())\n",
    "        if final>=95:\n",
    "            return 1\n",
    "    return 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different school level types encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different school level types that are present in the US Education System.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_SchoolLevels(val):\n",
    "    Dict_SchoolLevels = ['K-8','K-2','K-3' 'Elementary', 'Elementary School', 'Middle', 'Middle School', 'Transfer', 'High',\n",
    "                    'High School','D75','YABC']\n",
    "    \n",
    "    if type(s)!=str or s==None:\n",
    "        return 0\n",
    "    ans=0\n",
    "    for itr in Dict_SchoolLevels:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            ans=1\n",
    "    \n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different colleges of New York that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different colleges in New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_College(val):\n",
    "    Dict_Colleges=['Borough of Manhattan Community College', 'Bronx Community College', 'Guttman Community College', 'Hostos Community College', 'Kingsborough Community College', 'LaGuardia Community College', 'Queensborough Community College', 'Baruch College', 'Brooklyn College', 'City College of New York', 'College of Staten Island', 'Hunter College', 'John Jay College of Criminal Justice', 'Lehman College', 'Medgar Evers College', 'New York City College of Technology', 'Queens College', 'York College', 'CUNY William E. Macaulay Honors College', 'CUNY Graduate Center', 'CUNY Graduate School of Journalism', 'CUNY School of Law', 'CUNY School of Medicine', 'CUNY School of Professional Studies', 'CUNY School of Public Health', 'Fashion Institute of Technology', 'SUNY College of Optometry', 'SUNY Downstate Medical Center', 'SUNY Empire State College (The Harry Van Arsdale Jr. Center for Labor Studies)', 'SUNY Maritime College', 'The Ailey School (Alvin Ailey American Dance Crew).', 'Albert Einstein College of Medicine', 'American Academy of Dramatic Arts', 'American Academy McAllister Institute', 'American Musical and Dramatic Academy', 'ASA College', 'Bank Street College of Education', 'Bard College', 'Bard Graduate Center', 'Globalization and International Affairs Program', 'Barnard College', 'Berkeley College', 'Bethel Seminary of the East', 'Boricua College', 'Bramson ORT College', 'Briarcliffe College - The Queens Center', 'Brooklyn Law School', \"Christie's Education\", 'College of Mount Saint Vincent', 'College of New Rochelle (School of New Resources)', 'Columbia University', 'Teachers College', 'Union Theological Seminary', 'Columbia University School of General Studies', 'Cooper Union', 'Cornell University', 'Cornell NYC Tech', 'Weill Cornell Graduate School of Medical Sciences', 'Weill Cornell Medical College', 'DeVry University', 'European School of Economics', 'Fordham University', 'Frank G. Zarb School of Business at Hofstra University[1]', 'General Theological Seminary', 'Gerstner Sloan Kettering Graduate School of Biomedical Science', 'Globe Institute of Technology', 'Hebrew Union College', 'Helene Fuld College of Nursing', 'Icahn School of Medicine at Mount Sinai', 'Jewish Theological Seminary of America', 'The Juilliard School', 'Keller Graduate School of Management', \"The King's College\", 'Laboratory Institute of Merchandising', 'Long Island Business Institute - Flushing', 'Long Island College Hospital School of Nursing', 'Long Island University', 'Mandl College of Allied Health', 'Manhattan College', 'Manhattan School of Music', 'Marymount Manhattan College', 'Mercy College', 'Metropolitan College of New York', 'Monroe College', 'The New School', 'Eugene Lang College The New School for Liberal Arts', 'Mannes College The New School for Music', 'Milano School of Management, Policy, and Environment', 'The New School for Drama', 'The New School for General Studies', 'The New School for Jazz and Contemporary Music', 'The New School for Social Research', 'Parsons The New School for Design', 'New York Academy of Art', 'New York Career Institute', 'New York Conservatory for Dramatic Arts', 'New York Film Academy', 'New York Graduate School of Psychoanalysis', 'New York Institute of Technology', 'New York Law School', 'New York School of Interior Design', 'New York School of Urban Ministry', 'New York Theological Seminary', 'New York University', 'Gallatin School of Individualized Study', 'New York University Polytechnic School of Engineering', 'New York University School of Law', 'New York University School of Medicine', 'Tisch School of Arts', 'New York University of Architecture', 'Nyack College (School of Music)', 'Pace University', 'Pacific College of Oriental Medicine', 'Phillips Beth Israel School of Nursing', 'Plaza College', 'Pratt Institute', 'Rabbi Isaac Elchanon Theological Seminary', 'Richard Gilder Graduate School - American Museum of Natural History', 'Rockefeller University', 'School of American Ballet', 'School of Visual Arts', 'St. Francis College', \"St. John's University\", 'New Brunswick Theological Seminary', \"St. Joseph's College\", \"Sotheby's Institute of Art\", 'Studio Maestro', 'Swedish Institute of Massage Therapy', 'Technical Career Institute College of Technology', 'Touro College', 'Tri-State College of Acupuncture', 'Vaughn College of Aeronautics & Technology', 'Wagner College', 'Wood Tobe-Coburn School', 'Yeshiva University', 'Benjamin N. Cardozo School of Law']\n",
    "\n",
    "    if type(s)!=str or s==None:\n",
    "        return 0\n",
    "    ans=0\n",
    "    for itr in Dict_Colleges:\n",
    "        ratio=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if ratio>=85:\n",
    "            ans=1\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different cities that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary of the different possible cities.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_City(InputData):\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    Dict_cities = ['Albany', 'Amsterdam', 'Auburn', 'Batavia', 'Beacon', 'Binghamton', 'Buffalo', 'Canandaigua', 'Cohoes', 'Corning', 'Cortland', 'Dunkirk', 'Elmira', 'Fulton', 'Geneva', 'Seneca[A]', '3609928640', 'Glen Cove', 'Glens Falls', 'Gloversville', 'Hornell', 'Hudson', 'Ithaca', 'Jamestown', 'Johnstown', 'Kingston', 'Lackawanna', 'Little Falls', 'Lockport', 'Long Beach', 'Mechanicville', 'Middletown', 'Mount Vernon', 'New Rochelle', 'New York', 'Newburgh', 'Niagara Falls', 'North Tonawanda', 'Norwich', 'Ogdensburg', 'Olean', 'Oneida', 'Oneonta', 'Oswego', 'Peekskill', 'Plattsburgh', 'Port Jervis', 'Poughkeepsie', 'Rensselaer', 'Rochester', 'Rome', 'Rye', 'Salamanca', 'Saratoga Springs', 'Schenectady', 'Sherrill', 'Syracuse', 'Tonawanda', 'Troy', 'Utica', 'Watertown', 'Watervliet', 'White Plains', 'Yonkers']\n",
    "    for itr in Dict_cities:\n",
    "        if fuzz.ratio(itr,InputData.lower())>=85:\n",
    "            return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different car makers that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different car-makers.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_CarMake(InputData):\n",
    "    Dict_CarMaker = ['BMW', 'TOYOT', 'AUDI', 'NISSA', 'DODGE', 'ACURA', 'HYUND', 'MINI', 'LEXUS', 'SUBAR', \n",
    "                'HONDA', 'KIA', 'CHEVR', 'FORD', 'BUICK', 'VOLKS', 'GMC', 'VOLVO', 'INFIN', 'JEEP',\n",
    "                'ACADE','AERO','AJAX','ALEXA','ALFAR','AMC','ANCIA','APRIL','AS/M','AUSTI','AUTOC','BENTL',\n",
    "                'BERTO','BL/B','BL/BI','BRIDG','CADIL','CARGO','CHECK','CHRYS','DORSE','DUCAT','FERRA',\n",
    "                'FIAT','FRUEH','HARLE','HIN','HINO','HUMME','INTER','ISUZU','JAGUA','KAWAS','KENWO',\n",
    "                'LINCO','MACK','MASSA','MAZDA','MERCU','MINI','MITSU','NS/OT']\n",
    "\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    for itr in Dict_CarMaker:\n",
    "        ratio=fuzz.partial_ratio(itr.lower(), InputData.lower())\n",
    "        if ratio>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different boroughs of New York that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different boroughs of New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_Borough(val):\n",
    "    Dict_Boroughs = ['brooklyn', 'bronx', 'queens', 'manhattan', 'staten island']\n",
    "    Dict_County=['K','M','Q','R','X']\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    if len(s)==1:\n",
    "        if s.upper() in Dict_County:\n",
    "            return 1\n",
    "    flag=0\n",
    "    for boro in Dict_Boroughs:\n",
    "        final=fuzz.ratio(val.lower(), boro.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different schools of New York that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different schools of New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_School(val):\n",
    "    Dict_Schools=['Academy for Language and Technology', \n",
    "             'Academy for Scholarship and Entrepreneurship: A College Board School', \n",
    "             'Academy of Mount Saint Ursula', 'Adlai E. Stevenson Educational Campus', \n",
    "             'Adlai E. Stevenson High School (closed 2009)', 'Bronx Guild', \n",
    "             'Gateway School for Environmental Research and Technology', \n",
    "             'Millennium Art Academy', 'Pablo Neruda Academy for Architecture and World Studies', \n",
    "             'School for Community Research and Learning', '(split)', \n",
    "             'Alfred E. Smith Career and Technical Education High School', 'Collocated schools:', \n",
    "             'Bronx Design and Construction Academy', 'Bronx Haven High School', 'X600', 'All Hallows High School', \n",
    "             'Aquinas High School', 'Astor Collegiate Academy', '(Christopher Columbus campus)', \n",
    "             'Banana Kelly High School', '(Collocated with Holcombe L. Rucker School of Community Research)', \n",
    "             'Belmont Preparatory High School', '(Theodore Roosevelt campus)', 'Bronx Academy of Health Careers', \n",
    "             '(Evander Childs campus)', 'Bronx Academy High School', 'Bronx Academy of Letters', \n",
    "             'Bronx Aerospace High School', '(Evander Childs campus)', 'Bronx Center for Science and Mathematics', 'Bronx Charter School for Excellence', 'Bronx Coalition Community High School', '(James Monroe Campus)', 'Bronx Collegiate Academy', '(formerly Bronx Expeditionary Learning High School)', '(William H. Taft campus)', 'Bronx Community High School', 'Bronx Design and Construction Academy', '(Alfred E. Smith Educational Campus)', 'Bronx Engineering and Technology Academy', '(John F. Kennedy campus)', 'Bronx Envision Academy', '(Collocated at Explorations Academy)', 'Bronx Guild', '(Adlai E. Stevenson Educational Campus)', 'Bronx Haven High School', '(Alfred E.Smith Campus)', 'Bronx High School for Law and Community Service', 'Theodore Roosevelt campus', 'Bronx High School for Medical Science', '(William H. Taft campus)', 'Bronx High School for the Visual Arts', 'Bronx High School for Writing and Communication Arts', '(Evander Childs campus)', 'Bronx High School of Business', '(William H. Taft campus)', 'Bronx High School of Science', 'Bronx International High School', '(Morris Campus)', 'Bronx Lab School', '(Evander Childs campus)', 'Bronx Latin', 'Bronx Leadership Academy High School', 'Bronx Leadership Academy II High School', '(Concourse Village campus)', 'Bronx Preparatory Charter School', 'Bronx Regional High School', 'Bronx School for Law, Government and Justice', 'Bronx School of Law and Finance', '(John F. Kennedy campus)', 'Bronx Studio School for Writers and Artists', 'Bronx Theatre High School', '(John F. Kennedy campus)', 'Bronxdale High School', '(Christopher Columbus campus)', 'The Bronxwood Preparatory Academy', 'Cardinal Hayes High School', 'Cardinal Spellman High School', 'The Celia Cruz Bronx High School of Music', '(Walton Educational Campus)', 'Christopher Columbus High School', 'Collocated specialty schools:', 'Astor Collegiate Academy', 'Bronxdale High School', 'Collegiate Institute for Math and Science', 'Global Enterprise High School', 'Pelham Preparatory Academy', 'X415', 'The Cinema School', '(James Monroe Campus)', 'Collegiate Institute for Math and Science', '(Christopher Columbus campus)', 'Community School for Social Justice', 'Concourse Village campus', 'See:', 'Bronx Leadership Academy II', 'New Explorers High School', 'Urban Assembly School for Careers in Sports', '(multiple)', 'Crotona Academy High School', 'DeWitt Clinton High School', 'Discovery High School', '(Walton Educational Campus)', 'DreamYard Preparatory School', '(William H. Taft campus)', 'Eagle Academy for Young Men', 'East Bronx Academy for the Future', 'Evander Childs Educational Campus', 'See:', 'Bronx Academy of Health Careers', 'Bronx Aerospace High School', 'Bronx High School for Writing and Communication Arts', 'Bronx Lab School', 'High School for Contemporary Arts', 'High School of Computers and Technology', '(split)', 'Eximius College Preparatory Academy: A College Board School', 'Explorations Academy', 'Collocated school: Bronx Envision Academy', 'Fannie Lou Hamer Freedom High School', 'The Felisa Rincon de Gautier Institute for Law and Public Policy', 'The Fieldston School', '(Ethical Culture Fieldston School)', 'Fordham High School for the Arts', '(Theodore Roosevelt campus)', 'Fordham Leadership Academy for Business and Technology', '(Theodore Roosevelt campus)', 'Fordham Preparatory School', 'Foreign Language Academy of Global Studies (FLAGS)', 'Frederick Douglass Academy III Secondary School', 'Gateway School for Environmental Research and Technology', '(Adlai E. Stevenson Educational Campus)', 'Global Enterprise High School', '(Christopher Columbus campus)', 'Grace Dodge Career and Technical Education High School', 'Harry S Truman High School', 'Health Opportunities High School', 'Herbert H. Lehman High School', 'Collocated school:', 'Renaissance High School for Musical Theater & Technology', 'X405', 'High School for Contemporary Arts', '(Evander Childs campus)', 'High School for Language and Innovation', '(Christopher Columbus Educational Campus)', 'High School for Teaching and the Professions', '(Walton Educational Campus)', 'High School for Violin and Dance', '(Morris Campus)', 'High School of American Studies at Lehman College', 'High School of Computers and Technology', '(Evander Childs campus)', 'High School of World Cultures', '(James Monroe Campus)', 'Holcombe L. Rucker School of Community Research', '(Collocated with Banana Kelly High School)', 'Horace Mann School', 'Hostos-Lincoln Academy of Science', 'In-Tech Academy', '(Information and Network Technology Academy)', 'International Community High School', 'International School for Liberal Arts', '(Walton Educational Campus)', 'James Monroe Educational Campus', 'See:', 'James Monroe High School (closed)', 'Bronx Coalition Community High School', 'The Cinema School', 'High School of World Cultures', 'The Metropolitan Soundview High School', 'Monroe Academy for Business/Law', 'Monroe Academy for Visual Arts & Design', 'Pan American International High School at Monroe', '(split)', 'Jane Addams High School for Academic Careers', 'Jill Chaifetz Transfer High School', 'John F. Kennedy High School', 'Collocated schools:', 'Bronx Engineering and Technology Academy', 'Bronx School of Law and Finance', 'Bronx Theatre High School', 'Marble Hill High School for International Studies', 'X475', 'Jonathan Levin High School for Media and Communications', '(William H. Taft campus)', 'Kingsbridge International High School', '(Walton Educational Campus)', 'Knowledge and Power Preparatory Academy International High School (KAPPA)', '(Theodore Roosevelt campus)', 'Leadership Institute', 'Marble Hill High School for International Studies', '(John F. Kennedy campus)', 'The Marie Curie School for Medicine, Nursing, and Health Professions', 'The Metropolitan High School', 'The Metropolitan Soundview High School', '(James Monroe Educational Campus)', 'Millennium Art Academy', '(Adlai E. Stevenson Educational Campus)', 'Monroe Academy for Business/Law', '(James Monroe Campus)', 'Monroe Academy for Visual Arts & Design', '(James Monroe Campus)', 'Monsignor Scanlan High School', 'Morris Academy for Collaborative Studies', '(Morris Campus)', 'Morris Educational Campus', 'See:', 'Morris High School (closed)', 'Bronx International High School', 'High School for Violin and Dance', 'Morris Academy for Collaborative Studies', 'School for Excellence', '(split)', 'Mott Hall Bronx High School', 'Mott Haven Village Preparatory High School', '(South Bronx campus)', 'Mount Saint Michael Academy', 'New Day Academy', 'New Explorers High School', '(Concourse Village campus)', 'New World High School', 'Pablo Neruda Academy for Architecture and World Studies', '(Adlai E. Stevenson Educational Campus)', 'Pan American International High School at Monroe', '(James Monroe Campus', 'Latino immigrants only', 'Peace and Diversity Academy', 'Pelham Preparatory Academy', '(Christopher Columbus campus)', 'Performance Conservatory High School', '(Bronx High School for Performance and Stagecraft)', 'Preston High School', 'Providing Urban Learners Success in Education (PULSE) High School', 'Renaissance High School for Musical Theater & Technology', '(Herbert H. Lehman Educational Campus)', 'Riverdale Country School', 'Riverdale Kingsbridge Academy', 'St. Barnabas High School', 'St. Catharine Academy', 'St. Pius V High School', 'St. Raymond Academy for Girls', 'St. Raymond High School for Boys', 'Salanter Akiba of Riverdale (SAR) High School', 'Samuel Gompers Career and Technical Education High School', 'School for Community Research and Learning', '(Adlai E. Stevenson Educational Campus)', 'School for Excellence', '(Morris Campus)', 'South Bronx Educational Campus', 'See:', 'South Bronx High School (closed)', 'Mott Haven Village Preparatory High School', 'University Heights Secondary School', '(split)', 'South Bronx Preparatory: A College Board School', 'Theatre Arts Production Company School', 'Theodore Roosevelt Educational Campus', 'See:', 'Theodore Roosevelt High School (closed)', 'Belmont Preparatory High School', 'Bronx High School for Law and Community Service', 'Fordham High School for the Arts', 'Fordham Leadership Academy for Business and Technology', 'Knowledge and Power Preparatory Academy International High School (KAPPA)', 'West Bronx Academy for the Future', '(split)', 'University Heights Secondary School', '(South Bronx campus)', 'The Urban Assembly Academy for History and Citizenship for Young Men (closed 2014)', '(William H. Taft campus)', 'The Urban Assembly School for Applied Math and Science', 'Careers in Sports High School', '(Concourse Village campus)', 'Validus Preparatory Academy: An Expeditionary Learning School', 'Walton Educational Campus', 'See:', 'Walton High School (closed)', 'Celia Cruz Bronx High School of Music', 'Discovery High School', 'High School for Teaching and the Professions', 'International School for Liberal Arts', 'Kingsbridge International High School', '(split)', 'West Bronx Academy for the Future', '(Theodore Roosevelt campus)', 'William Howard Taft Educational Campus', 'See:', 'William H. Taft High School (closed)', 'Bronx Collegiate Academy', 'Bronx High School for Medical Science', 'Bronx High School of Business', 'DreamYard Preparatory School', 'Jonathan Levin High School for Media and Communications', 'The Urban Assembly Academy for History and Citizenship for Young Men', '(split)', 'Wings Academy', \"Women's Academy of Excellence\", 'Yeshiva of Telshe Alumni', 'Brooklyn', 'This section: viewtalkedit', 'School', 'NYC DOE', 'Abraham Lincoln High School', 'Academy for College Preparation and Career Exploration:A College Board School', '(Erasmus Hall Educational Campus)', 'Academy for Conservation and the Environment', '(South Shore Educational Campus)', 'Academy for Environmental Leadership', '(Bushwick Educational Campus)', 'Academy for Young Writers', 'Academy of Hospitality and Tourism', '(Erasmus Hall Educational Campus)', 'Academy of Innovative Technology', \n",
    "             '(Franklin K. Lane Educational Campus)', 'Academy of Urban Planning', '(Bushwick Educational Campus)', 'ACORN Community High School', 'All City Leadership Secondary School', 'Al-Madinah School', 'Al-Noor School', 'Aspirations Diploma Plus High School', 'Automotive High School', 'Bedford Academy High School', 'Bedford Stuyvesant Preparatory High School', 'Benjamin Banneker Academy', 'Berkeley Carroll School', 'Beth Jacob High School', 'Beth Rivkah', 'Big Apple Academy', 'Bishop Ford Central Catholic High School', 'Bishop Kearney High School', 'Bishop Loughlin Memorial High School', 'B’nos Leah Prospect Park Yeshiva School', 'B’nos Yisroel High School for Girls', 'Boys and Girls High School', 'Brooklyn Academy of Science and the Environment (BASE)', '(Prospect Heights Educational Campus)', 'Brooklyn Academy High School', 'Brooklyn Bridge Academy', '(South Shore Educational Campus)', 'Brooklyn College Academy', 'Brooklyn Collegiate: A College Board School', 'Brooklyn Community High School of Communication, Arts and Media', 'Brooklyn Democracy Academy', 'Brooklyn Generation School', '(South Shore Educational Campus)', 'Brooklyn High School for Leadership and Community Service', 'Brooklyn High School of the Arts', 'Brooklyn High School for Law and Technology', '(Formerly ACORN High School for Social Justice)', 'Brooklyn International High School', '(Waters Edge Educational Complex)', 'Brooklyn Lab School', '(Franklin K. Lane Educational Campus)', 'The Brooklyn Latin School', 'Brooklyn Preparatory High School', '(Harry Van Arsdale Educational Campus)', 'Brooklyn School for Global Studies', 'The Brooklyn School for Math and Research', '(Bushwick Educational Campus)', 'Brooklyn School for Music & Theatre', '(Prospect Heights Educational Campus)', 'Brooklyn Secondary School for Collaborative Studies', 'Brooklyn Studio Secondary School', 'Brooklyn Technical High School', 'Brooklyn Theatre Arts High School', '(South Shore Educational Campus)', 'Brownsville Academy High School', 'Bushwick Community High School', 'Bushwick Educational Campus', 'See:', 'Bushwick High School (closed 2006)', 'Academy for Environmental Leadership', 'Academy of Urban Planning', 'The Brooklyn School for Math and Research', 'Bushwick School for Social Justice', '(split)', 'Bushwick Leaders High School for Academic Excellence', 'Bushwick School for Social Justice', '(Bushwick Educational Campus)', 'Canarsie High School', 'Catherine McAuley High School', 'City Polytechnic High School of Engineering, Architecture, and Technology', 'Clara Barton High School', 'Cobble Hill School of American Studies', 'Connie Lekas School', 'Cypress Hills Collegiate Preparatory School', '(Franklin K. Lane Educational Campus)', 'Dr. Susan S. McKinney Secondary School of the Arts', 'East New York Family Academy', 'EBC/ENY High School for Public Safety & Law', 'EBC High School for Public Service–Bushwick', 'Edward R. Murrow High School', 'El Puente Academy for Peace and Justice', 'Enterprise, Business and Technology High School', 'Expeditionary Learning School for Community Leaders', 'Erasmus Hall Educational Campus', 'See:', 'Erasmus Hall High School (closed 1994)', 'Academy for College Preparation and Career Exploration:A College Board School', 'Academy of Hospitality and Tourism', 'High School for Service & Learning at Erasmus', 'High School for Youth and Community Development at Erasmus', 'Science, Technology and Research Early College High School at Erasmus', '(split)', 'FDNY High School for Fire and Life Safety', '(Thomas Jefferson Educational Campus)', 'Fontbonne Hall Academy', 'Fort Hamilton High School', 'Foundations Academy', 'Franklin Delano Roosevelt High School', 'Franklin K. Lane Educational Campus', 'See:', 'Franklin K. Lane High School (closing 2012)', 'Academy of Innovative Technology', 'Brooklyn Lab School', 'Cypress Hills Collegiate Preparatory School', 'Multicultural High School', 'K420', 'Frederick Douglass Academy IV Secondary School', 'Frederick Douglass Academy VII High School', 'Freedom Academy High School', 'George Wingate Educational Campus', 'See:', 'George W. Wingate High School (closed 2000)', 'High School for Public Service: Heroes of Tomorrow', 'International Arts Business School', 'School for Democracy and Leadership', 'The School for Human Rights', '(split)', 'Gerer Mesivta Bais Yisroel School', 'George Westinghouse Career and Technical Education High School', 'Gotham Professional Arts Academy', 'Green School: An Academy for Environmental Careers', 'Harry Van Arsdale Educational Campus', 'See:', 'Harry Van Arsdale High School (closed 2007)', 'Brooklyn Preparatory High School', 'Williamsburg High School for Architecture and Design', 'Williamsburg Preparatory School', '(split)', 'High School for Civil Rights', '(Thomas Jefferson Educational Campus)', 'The High School for Global Citizenship', '(Prospect Heights Educational Campus)', 'High School for Public Service: Heroes of Tomorrow', '(George W. Wingate Educational Campus)', 'High School for Service & Learning at Erasmus', '(Erasmus Hall Educational Campus)', 'High School for Youth and Community Development at Erasmus', '(Erasmus Hall Educational Campus)', 'High School of Sports Management', 'High School of Telecommunication Arts and Technology', 'International Arts Business School', '(George W. Wingate Educational Campus)', 'International High School at Lafayette', 'International High School at Prospect Heights', 'Prospect Heights Educational Campus', 'It Takes a Village Academy', '(Samuel J. Tilden Educational Campus)', 'James Madison High School', 'John Dewey High School', 'John Jay Educational Campus', 'See:', 'John Jay High School (closed 2003)', 'Millennium Brooklyn High School', 'Park Slope Collegiate (formerly Secondary School for Research)', 'Secondary School for Journalism', 'Secondary School for Law', '(split)', 'Juan Morel Campos Secondary School', 'Kingsborough Early College Secondary School', 'Kurt Hahn Expeditionary Learning School', '(Samuel J. Tilden Educational Campus)', 'Lafayette High School', 'Leon M. Goldstein High School for the Sciences', 'Liberation Diploma Plus', 'Life Academy High School for Film and Music', 'Lyons Community School', 'Magen David Yeshivah High School', 'Medgar Evers College Preparatory School', 'Merkaz Bnos High School', \"Mesivta M'kor Chaim School\", 'Mesivta Nachlas Yakov School', 'Mesivta of Seagate School', 'Mesivta Rabbi Chaim Berlin High School', 'Mesivta/Yeshiva Gedola Manhattan Beach School', 'Metropolitan Corporate Academy', 'Metropolitan Diploma Plus High School', 'Midwood High School', 'Mikdash Melech Mechina School', 'Millennium Brooklyn High School', '(John Jay Educational Campus)', 'Mirrer Yeshiva High School', 'Multicultural High School', '(Franklin K. Lane Educational Campus)', 'Nazareth Regional High School', 'New Utrecht High School', 'New York Harbor School', 'Olympus Academy', 'Pacific High School', 'The Packer Collegiate Institute', 'Park Slope Collegiate', '(John Jay Educational Campus)', '(Formerly Secondary School for Research)', 'Pathways in Technology Early College High School', '(Collocated at Paul Robeson High School)', 'Paul Robeson High School for Business and Technology', 'Collocated school: Pathways in Technology Early College High School', 'K625', 'Performing Arts and Technology High School', '(Thomas Jefferson Educational Campus)', 'Poly Prep Country Day School', 'Progress High School for Professional Careers', 'Prospect Heights Educational Campus', 'See:', 'Prospect Heights High School (closed 2006)', 'Brooklyn Academy of Science and the Environment', 'Brooklyn School for Music & Theatre', 'The High School for Global Citizenship', 'International High School at Prospect Heights', '(split)', 'Rachel Carson High School for Coastal Studies', 'Roy Campanella Occupational Training Center', \"St. Ann's School\", 'St. Edmund Preparatory High School', 'St. Joseph High School', 'St. Saviour High School', 'Samuel J. Tilden Educational Campus', 'See:', 'Samuel J. Tilden High School (closed ~2010)', 'It Takes a Village Academy', 'Cultural Academy for the Arts and Sciences', 'The Kurt Hahn Expeditionary Learning School', '(split)', 'The School for Classics: An Academy of Thinkers, Writers and Performers', 'School for Democracy and Leadership', '(George W. Wingate Educational Campus)', 'The School for Human Rights', '(George W. Wingate Educational Campus)', 'School for International Studies', 'School for Legal Studies', 'Science Skills Center High School for Science, Technology and the Creative Arts', '(Waters Edge Educational Complex)', 'Science, Technology and Research Early College High School at Erasmus', '(Erasmus Hall Educational Campus)', 'Secondary School for Journalism', '(John Jay Educational Campus)', 'Secondary School for Law', '(John Jay Educational Campus)', 'Secondary School for Research (see Park Slope Collegiate)', '(renamed 2011)', 'Sheepshead Bay High School', 'Shulamith High School for Girls', 'Sinai Academy', 'Soille Bais Yaakov High School', 'South Brooklyn Community High School', 'South Shore Educational Campus', 'See:', 'South Shore High School (closed 2010)', 'Academy for Conservation and the Environment', 'Brooklyn Bridge Academy', 'Brooklyn Generation School', 'Brooklyn Theatre Arts High School', 'Victory Collegiate High School', 'K515', '(closing)', 'Sunset Park High School', 'Teachers Preparatory High School', 'Thomas Jefferson Educational Campus', 'See:', 'Thomas Jefferson High School (closed 2007)', 'FDNY High School for Fire and Life Safety', 'High School for Civil Rights', 'Performing Arts and Technology High School', 'World Academy for Total Community Health High School', '(split)', 'Tiferes Academy', 'Tiferes Bais Yaakov High School', 'Tiferes Miriam High School', 'Tomer Devora High School for Girls', 'Torah Academy High School', 'Transit Tech Career and Technical Education High School', 'United Lubavitcher Yeshiva', 'Urban Assembly Institute of Math and Science for Young Women', 'The Urban Assembly School for Law and Justice', \n",
    "             'The Urban Assembly School of Music and Art', '(Waters Edge Educational Complex)', 'Victory Collegiate High School', '(South Shore Educational Campus)', 'Waters Edge Educational Complex', 'See:', 'Brooklyn International High School', 'Science Skills Center High School for Science, Technology and the Creative Arts', 'Urban Assembly High School of Music and Art', '(collocated)', 'W.E.B. Dubois Academic High School', 'West Brooklyn Community High School', 'William E. Grady Career and Technical Education High School', 'William H. Maxwell Career and Technical Education High School', 'Williamsburg Charter High School', 'Williamsburg High School for Architecture and Design', '(Harry Van Arsdale Educational Campus)', 'Williamsburg Preparatory School', '(Harry Van Arsdale Educational Campus)', 'World Academy for Total Community Health High School', '(Thomas Jefferson Educational Campus)', 'Xaverian High School', 'Yeshiva Chanoch Lenaar', 'Yeshiva Derech HaTorah High School', 'Yeshiva Gedolah of Midwood', 'Yeshiva Gezdah Bais Yisrael', 'Yeshiva Karunas Halev', 'Yeshiva of Flatbush Joel Braverman High School', 'Yeshiva Ohr Moleh Koson', 'Yeshiva Sharro Yosher', 'Yeshiva Tiferes Shmiel', 'Yeshiva Toraf Hesed', 'Yeshiva Vyelipol School', 'Yeshivat Ateret Torah', 'Yeshivat Or Hatorah', 'Yeshivat Shaare Torah Boys High School', 'Yeshivat Shaare Torah Girls High School', \"Yeshiva/Mesivta V'yoel Moshe School\", 'Zvi Dov Roth Academy of Yeshiva Rambam', 'Manhattan', 'This section: viewtalkedit', 'School', 'NYC DOE', 'A. Philip Randolph Campus High School', 'Aaron School', 'Abraham Joshua Heschel School', 'Academy of Environmental Science Secondary High School', 'Academy for Social Action: A College Board School', 'Aichhorn School', 'The American Sign Language and English Secondary School', 'Avenues: The World School', 'Bard High School Early College', '(see also Bard High School Early College II in Queens)', 'Ballet Tech, NYC Public School for Dance', 'Baruch College Campus High School', 'Bayard Rustin Educational Complex', 'The Beacon School', '(Beacon High School)', 'The Beekman School', 'Bread & Roses Integrated Arts High School', 'Brearley School', 'The Browning School', 'The Calhoun School', 'Cascades High School', '(Cascades Center for Teaching & Learning)', 'Cathedral High School', 'Central Park East High School', 'Chapin School (Manhattan)', 'Chelsea Career and Technical Education High School)', 'Choir Academy of Harlem', 'City-As-School', '(High School M560 - City As School)', 'Coalition School for Social Change', 'Collegiate School', 'Columbia Grammar & Preparatory School', 'Columbia Secondary School', '(Columbia Secondary School for Math, Science & Engineering)', 'Cristo Rey New York High School', 'Convent of the Sacred Heart', 'The Dalton School', 'Daytop Preparatory School', 'Dominican Academy', 'The Dwight School', 'East Side Community High School', 'Edward A. Reynolds West Side High School', 'Eleanor Roosevelt High School', 'Elisabeth Irwin High School', '(Little Red Schoolhouse & Elisabeth Irwin High School) (LREI)', 'Essex Street Academy', '(Seward Park campus)', 'The Facing History School', '(Park West campus)', 'Fiorello H. LaGuardia High School of Music & Art and Performing Arts', 'Food and Finance High School', '(Park West campus)', 'Frederick Douglass Academy', 'See also:', 'Frederick Douglass Academy II Secondary School (Manhattan)', 'Frederick Douglass Academy III Secondary School (Bronx)', 'Frederick Douglass Academy IV Secondary School (Brooklyn)', 'Frederick Douglass Academy VI High School (Queens)', 'Frederick Douglass Academy VII High School (Brooklyn)', 'M499', 'Frederick Douglass Academy II Secondary School', 'Friends Seminary', 'Fusion Academy & Learning Center', 'George Washington High School', 'See:', 'High School for Health Careers and Sciences', 'High School for International Business and Finance', 'High School for Law and Public Service', 'High School for Media and Communications', '(split)', 'Gramercy Arts High School', 'Greenwich Village High School', 'Gregorio Luperon High School for Math & Science', '(Gregorio Luperon High School for Science and Mathematics)', 'Harlem Renaissance High School', 'Harvey Milk High School', 'Henry Street School for International Studies', 'The Heritage School', 'The Hewitt School', 'High School for Arts, Imagination and Inquiry', '(Martin Luther King, Jr. campus)', 'High School for Dual Language and Asian Studies', '(Seward Park campus)', 'High School For Environmental Studies', 'High School for Health Careers and Sciences', '(George Washington campus)', 'High School for Health Professions and Human Services', 'High School for Humanities Educational Campus', 'See:', 'Bayard Rustin High School for the Humanities', 'Humanities Preparatory Academy', 'The James Baldwin School', ' ', 'High School for International Business and Finance', '(George Washington campus)', 'High School for Law, Advocacy, and Community Justice', '(Martin Luther King, Jr. campus)', 'High School for Law and Public Service', '(George Washington campus)', 'High School for Leadership and Public Service', 'High School for Math, Science and Engineering at City College', 'High School for Media and Communications', '(George Washington campus)', 'High School of Art and Design', 'High School of Arts and Technology', '(Martin Luther King, Jr. campus)', 'High School of Economics and Finance', 'High School of Fashion Industries', 'High School of Graphic Communication Arts', 'Collocated school: Urban Assembly Gateway School for Technology', 'High School of Hospitality Management', '(Park West campus)', 'Humanities Preparatory Academy', '(High School for Humanities Educational Campus)', 'Hunter College High School', 'Independence High School', 'Institute for Collaborative Education', 'Jacqueline Kennedy Onassis High School', 'The James Baldwin School: A School for Expeditionary Learning', '(High School for Humanities Educational Campus)', 'John V. Lindsay Wildcat Academy Charter School', 'Julia Richman Education Complex', 'See:', 'Julia Richman High School (closed 1995)', 'Manhattan International High School', 'Talent Unlimited High School', 'Vanguard High School', '(split)', 'La Salle Academy', 'Landmark High School', 'Legacy School for Integrated Studies', 'Léman Manhattan Preparatory School', 'Liberty High School Academy for Newcomers', 'Life Sciences Secondary School', 'Louis D. Brandeis High School', 'Lower East Side Preparatory High School', 'Lower Manhattan Arts Academy', '(Seward Park campus)', 'Loyola School', 'Lycée français de New York (LFNY)', 'Manhattan Bridges High School', '(Park West campus)', 'Manhattan Center for Science and Mathematics', 'Manhattan Comprehensive Night and Day High School', 'Manhattan High School for Girls', 'Manhattan/Hunter College High School for Sciences', '(Manhattan/Hunter Science High School)', '(Hunter College High School for the Sciences)', '(Martin Luther King, Jr. campus)', 'Manhattan International High School', '(Julia Richman Education Complex - MIHS)', 'Manhattan Theatre Lab High School', '(Martin Luther King, Jr. campus)', 'Manhattan Village Academy', 'Marta Valle High School', '(Marta Valle Secondary School)', 'Martin Luther King High School', 'See:', 'High School for Arts, Imagination and Inquiry', 'High School for Law, Advocacy and Community Justice', 'High School of Arts and Technology', 'Hunter College High School (Manhattan/Hunter Science High School)', 'Manhattan Theatre Lab High School', 'The Urban Assembly School for Media Studies', '(split)', 'Marymount School', 'Millennium High School', 'Mother Cabrini High School', 'Mott Hall High School', 'Murry Bergtraum High School for Business Careers', 'New Design High School', '(Seward Park campus)', 'New Explorations into Science, Technology and Math High School (NEST+m High School)', 'NYC iSchool', 'New York City Lab School for Collaborative Studies', 'New York City Museum School', 'Nightingale-Bamford School', 'Norman Thomas High School', 'Northeastern Academy', 'Notre Dame School', 'Pace High School', 'Park East High School', 'Park West High School', 'See:', 'The Facing History School', 'Food and Finance High School', 'High School of Hospitality Management', 'Manhattan Bridges High School', 'The Urban Assembly School of Design and Construction', '(split)', \"Professional Children's School\", 'Professional Performing Arts School', 'Quest to Learn', 'Ramaz School', 'Rebecca School', 'Regis High School', 'Repertory Company High School for Theatre Arts', 'Rice High School', 'Richard R. Green High School of Teaching', 'The Robert Louis Stevenson School', 'St. Agnes Boys High School', 'St. Jean Baptiste High School', 'St. Michael Academy', 'St. Vincent Ferrer High School', 'Sacred Heart', '(See Convent of the Sacred Heart)', 'Satellite Academy High School', 'School for the Physical City High School', 'School of the Future', 'Seward Park High School', 'See:', 'Essex Street Academy', 'High School for Dual Language and Asian Studies', 'Lower Manhattan Arts Academy', 'New Design High School', 'The Urban Assembly Academy of Government and Law', '(split)', 'The Smith School', 'Solomon Schechter High School of New York', 'Spence School', 'Stuyvesant High School', 'Talent Unlimited High School', '(Julia Richman Education Complex - TUHS)', 'Thurgood Marshall Academy for Learning and Social Change', 'Trevor Day School', 'Trinity School', 'United Nations International School', 'Unity Center for Urban Technologies', 'University Neighborhood High School', 'Urban Academy Laboratory High School', '(Julia Richman Education Complex - UALHS)', 'The Urban Assembly Academy of Government and Law', '(Seward Park campus)', 'Urban Assembly Gateway School for Technology', 'Collocated at High School of Graphic Communication Arts', 'The Urban Assembly New York Harbor School', '(Governors Island)', 'The Urban Assembly School for Media Studies', '(Martin Luther King, Jr. campus)', 'Urban Assembly School for the Performing Arts'\n",
    "             'The Urban Assembly School of Business for Young Women', 'The Urban Assembly School of Design and Construction', '(Park West campus)', 'Urban Peace Academy', 'Vanguard High School', '(Julia Richman Education Complex - VHS)', 'Wadleigh Secondary School for the Performing & Visual Arts', 'Washington Irving High School', 'Winston Preparatory School', 'Xavier High School', 'Yeshiva University High School for Boys', 'York Preparatory School', \"Young Women's Leadership School\", 'Washington Heights Expeditionary Learning School', 'Queens', 'This section: viewtalkedit', 'School', 'NYC DOE', 'Academy for Careers in Television and Film', 'Academy of American Studies', 'Academy of Finance and Enterprise', 'Academy of Medical Technology', '(Far Rockaway Educational Campus)', 'Al-Iman School', 'Archbishop Molloy High School', 'August Martin High School', 'Aviation High School', 'Q610', 'Baccalaureate School for Global Education', 'Bais Yaakov Machon Academy', 'Bard High School Early College II', '(see also Bard High School Early College in Manhattan)', 'Bayside High School', 'Beach Channel High School (closing 2014)', 'Collocated schools:', 'Channel View School for Research – Q262', 'Rockaway Collegiate High School – Q351', 'Rockaway Park High School for Environmental Sustainability – Q324', 'Q410', 'Cambria Heights Academy for New Literacies', 'Campus Magnet Educational Campus', 'Collocated schools:', 'Andrew Jackson High School (Queens) (closed 1994)', 'Humanities and the Arts Magnet High School – Q498', 'Mathematics, Science Research and Technology Magnet High School – Q492', 'Institute for Health Professions at Cambria Heights – Q243', 'Benjamin Franklin High School for Finance & Information Technology - Q313', 'Business, Computer Applications & Entrepreneurship High School (closed 2016)', 'Law, Government and Community Service High School (closed 2016)', 'Q490', 'Benjamin N. Cardozo High School', 'Cathedral Preparatory Seminary', 'Channel View School for Research', '(Beach Channel Educational Campus)', 'The Child School Legacy High School, Roosevelt Island', 'Christ The King Regional High School', 'East-West School of International Studies', 'Evangel Christian School', 'Excelsior Preparatory High School', '(Springfield Gardens Educational Campus)', 'Ezra Academy', 'Far Rockaway Educational Campus', 'See:', 'Far Rockaway High School – Q465 (closed 2011)', 'Academy of Medical Technology: A College Board School – Q309', 'Frederick Douglass Academy VI High School – Q260', 'Queens High School for Information, Research, and Technology – Q302', '(split)', 'Flushing High School', 'Flushing International High School', 'Forest Hills High School', 'Francis Lewis High School', 'Frank Sinatra School of the Arts High School', 'Frederick Douglass Academy VI High School', '(Far Rockaway Educational Campus)', 'Garden School', 'Gateway to Health Science High School', 'George Washington Carver High School for the Sciences', '(Springfield Gardens Educational Campus)', 'Greater New York Academy of Seventh-day Adventists', 'Grover Cleveland High School', 'High School for Arts and Business', 'High School for Community Leadership', '(Jamaica High School campus)', 'High School for Construction Trades, Engineering and Architecture', 'High School for Law Enforcement and Public Safety', 'High School of Applied Communication', 'Hillcrest High School', 'Hillside Arts & Letters Academy', '(Jamaica High School campus)', 'Holy Cross High School', 'Humanities & Arts Magnet High School', 'Information Technology High School', 'International High School at LaGuardia Community College', 'Jamaica Gateway to the Sciences', '(Jamaica High School campus)', 'Jamaica High School', 'Collocated schools:', 'High School for Community Leadership – Q328', 'Hillside Arts & Letters Academy – Q325', 'Jamaica Gateway to the Sciences – Q350', 'Queens Collegiate: A College Board School – Q310', 'Q470', 'John Adams High School', 'John Bowne High School', 'John F. Kennedy Jr. School', '(formerly Queens Occupational Training Center)', 'Long Island City High School', 'The Lowell School', 'Martin Luther High School', 'Martin Van Buren High School', 'The Mary Louis Academy', 'Maspeth High School', 'Mathematics, Science Research and Technology Magnet High School', 'Mesivta Ohr Torah School', 'Mesivta Yesodei Yeshurun', 'Metropolitan Expeditionary Learning School', '(Queens Metropolitan High School campus)', 'Middle College High School at LaGuardia Community College', 'Monsignor McClancy Memorial High School', 'Newcomers High School', 'Newtown High School', 'North Queens Community High School', 'Pan American International High School', 'Pathways College Preparatory School: A College Board School', 'Preparatory Academy for Writers: A College Board School', '(Springfield Gardens Educational Campus)', 'Project Blend School, Jamaica', 'Queens Academy High School', 'Queens Collegiate: A College Board School', '(Jamaica High School campus)', 'Queens Gateway to Health Sciences Secondary School', 'Queens High School for Information, Research, and Technology', '(Far Rockaway Educational Campus)', 'Queens High School for the Sciences at York College', 'Queens High School of Teaching, Liberal Arts and the Sciences', 'Queens Metropolitan High School', 'Collocated schools:', 'Metropolitan Expeditionary Learning High School – Q167', 'Q686', 'Queens Preparatory Academy', '(Springfield Gardens Educational Campus)', 'Queens Vocational and Technical High School', 'Razi School', 'The Renaissance Charter School', 'Richmond Hill High School', 'Robert F. Kennedy Community High School', 'Robert F. Wagner, Jr. Secondary School for Arts and Technology', '(Robert F. Wagner, Jr. Institute for Arts & Technology)', 'Robert H. Goddard High School of Communication Arts and Technology', 'Rockaway Collegiate High School', '(Beach Channel Educational Campus)', 'Rockaway Park High School for Environmental Sustainability', '(Beach Channel Educational Campus)', 'St. Agnes High School', 'St. Demetrios School', 'St. Francis Preparatory School', 'St. George Academy', \"St. John's Preparatory School\", \"Scholars' Academy\", 'Shevach High School', 'Springfield Gardens Educational Campus, includes:', 'Excelsior Preparatory High School', 'George Washington Carver High School for the Sciences', 'Preparatory Academy for Writers: A College Board School', 'Queens Preparatory Academy', ' ', 'Stella Maris High School (closing June 2010)', 'Summit School', 'Thomas A. Edison Vocational and Technical High School', 'Torah Academy High School for Girls', 'Townsend Harris High School', 'VOYAGES Preparatory', 'Whitestone Academy', 'William Cullen Bryant High School', 'Windsor School', 'World Journalism Preparatory: A College Board School', 'Yeshiva Berachel David-Torah School', 'Yeshiva Binat Chaim - Boys School', 'Yeshiva of Far Rockaway', '(Yeshiva Derech Ayson)', 'Yeshiva Shaar Hatoreh Research', 'Yeshiva University High School for Girls', '(Samuel H. Wang Yeshiva University High School for Girls)', \"Young Women's Leadership School, Astoria\", \"Young Women's Leadership School, Queens\", 'Staten Island', 'This section: viewtalkedit', 'School', 'NYC DOE', 'College of Staten Island High School for International Studies', '(CSI High School for International Studies)', 'Concord High School', 'Curtis High School', 'Gaynor McCown Expeditionary Learning School', 'Mesivta Kesser Yisroel', 'Michael J. Petrides School', 'Miraj Islamic School', 'Monsignor Farrell High School', 'Moore Catholic High School', 'New Dorp High School', 'Notre Dame Academy High School', 'Port Richmond High School', 'Ralph R. McKee Career and Technical Education High School', 'Richard H. Hungerford School', 'St. Joseph by the Sea High School', 'St. Joseph Hill Academy', \"St. Peter's Boys High School\", 'Staten Island Academy', 'Staten Island Technical High School', 'Susan E. Wagner High School', 'Tottenville High School', 'Yeshiva of Staten Island', 'Yeshiva Tifereth Torah']\n",
    "\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    for itr in Dict_Schools:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different vehicle types of New York that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different vehicle types of New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_VehicleType(val):\n",
    "    Dict_VehicleTypes=['AGC AGRICULTURE', 'AGR AGRICULTURAL', 'AMB AMBULANCE', 'ARG AIR NATIONAL GUARD', 'ATD ATV DEALER', \n",
    "          'ATV ALL-TERRAIN VEHICLE', 'AYG ARMY NATIONAL GUARD', 'BOB BIRTHPLACE OF BASEBALL', 'BOT MOTORBOATS', \n",
    "          'CBS COUNTY BOARD OF LEGISLATORS', 'CCK COUNTY CLERK', 'CHC COMM. HS/HOLD CARRIER', 'CLG COUNTY LEGISLATORS', \n",
    "          'CME CORONER/MEDICAL EXAMINER', 'CMH CONGRESSIONAL MEDAL-OF-HONOR', 'COM COMMERCIAL', 'CSP SPORTS COMMERCIAL', \n",
    "          'DLR DEALER', 'EDU EDUCATOR', 'FAR FARM', 'FPW FORMER PRISONER OF WAR', 'GAC GOVERNOR ADDITIONAL CAR', \n",
    "          'GSC GOVERNOR SECOND CAR', 'GSM GOLD STAR MOTHERS', 'HAC HAM OPERATOR COMMERCIAL', 'HAM HAM OPERATOR', \n",
    "          'HIF HEARSE INV. SPEC. FEE', 'HIR HEARSE/HEARSE INVALID', 'HIS HISTORICAL', 'HOU HOUSE/COACH TRAILER', \n",
    "          'HSM HISTORICAL MOTORCYCLE', 'IRP INTERNATIONAL REGISTRATION PLAN', 'ITP INTRANSIT PERMIT', \n",
    "          'JCA JUSTICE CRT. OF APPEALS', 'JCL JUSTICE CRT. OF CLAIMS', 'JSC JSC APPELLATE DIV.', \n",
    "          'JWV JEWISH WAR VETERANS', 'LMA LIMITED USE MCY C1 A', 'LMB LIMITED USE MCY C1 B', \n",
    "          'LMC LIMITED USE MCY C1 C', 'LOC LOCOMOTIVE', 'LTR LIGHT TRAILER', 'LUA LIMITED USE AUTOMOBILE', \n",
    "          'MCD MOTORCYCLE DEALER', 'MCL MARINE CORP LEAGUE', 'MED MEDICAL DOCTOR', 'MOT MOTORCYCLE', \n",
    "          'NLM NAVAL MILITIA', 'NYA NEW YORK ASSEMBLY', 'NYC NEW YORK COUNCIL', 'NYS NEW YORK SENATE', \n",
    "          'OMF OMNIBUS FRANCHISE BUS', 'OML OMNIBUS LIVERY', 'OMO OMNIBUS OUT OF STATE', 'OMR OMNIBUS REGULAR', \n",
    "          'OMS OMNIBUS PRIVATE RENTAL', 'OMT OMNIBUS TAXI', 'OMV VANITY OMNIBUS', 'ORC COMMERCIAL ORGANIZATIONS', \n",
    "          'ORG ORGANIZATIONS', 'ORM ORGANIZATION MOTORCYCLE', 'PAS PASSENGER', 'PHS PEARL HARBOR SURVIVORS', \n",
    "          'PPH PURPLE HEART', 'PSD POLITICAL SUBDIVISION', 'RGC REGIONAL COMMERCIAL', 'RGL REGIONAL', \n",
    "          'SCL SCHOOL CAR', 'SEM SEMI-TRAILER', 'SNO SNOWMOBILE', 'SOS SURVIVORS OF THE SHIELD', \n",
    "          'SPC SPECIAL PURPOSE COMM.', 'SPO SPORTS', 'SRF SPECIAL REGISTRATION FEE', 'SRN', 'SPECIAL REGISTRATION', \n",
    "          '(JUDGES/OFFICIALS)', 'STA STATE OWNED VEHICLE', 'STG STATE NATIONAL GUARD', 'SUP JUSTICE SUPREME COURT', \n",
    "          'THC TRACTOR HOUSE/HOLD CARRIER', 'TOW TOW TRUCK', 'TRA TRANSPORTER', 'TRC TRACTOR (REGULAR)', \n",
    "          'TRL TRAILER (REGULAR)', 'USC US CONGRESS', 'USS US SENATE', 'VAS VOLUNTEER AMBULANCE SERVICES', \n",
    "          'VPL VAN POOL', 'WUG WORLD UNIVERSITY GAMES', 'FIRE FIRE TRUCK', 'CONV CONVERTIBLE', 'SEDN SEDAN', \n",
    "          'SUBN SUBURBAN', '4DSD FOUR-DOOR SEDAN', '2DSD TWO-DOOR SEDAN', 'H/WH HOUSE ON WHEELS', \n",
    "          'ATV ALL TERRAIN VEHICLE', 'MCY MOTORCYCLE', 'H/IN HEARSE-INVALID', 'LOCO LOCOMOTIVE', 'CUST CUSTOM', \n",
    "          'RPLC REPLICA', 'AMBU AMBULANCE', 'P/SH POWER SHOVEL', 'RBM ROAD BUILDING MACHINE', 'R/RD ROAD ROLLER', \n",
    "          'RD/S ROAD SWEEPER', 'S/SP', 'SAND OR AGRICULTRAL', 'SPREADER/SPRAYER', 'SN/P SNOW PLOW', 'TRAV SNOW TRAVELER', \n",
    "          'MOBL SNOWMOBILE', 'TR/E TRACTION ENGINE', 'T/CR TRACTOR CRANE', 'TR/C TRUCK CRANE', 'SWT TRUCK W/SMALL WHEELS', \n",
    "          'W/DR WELL DRILLER', 'W/SR WELL SERVICING RIG', 'FPM FEED PROCESSING MACHINE', 'MCC MOBILE CAR CRUSHER', \n",
    "          'EMVR EARTH MOVER', 'TRAC TRACTOR', 'N/A NOT APPLICABLE', 'DELV DELIVERY TRUCK', 'DUMP DUMP TRUCK', \n",
    "          'FLAT FLAT BED TRUCK', 'PICK PICK-UP TRUCK', 'STAK STAKE TRUCK', 'TANK TANK TRUCK', 'REFG REFRIGERATOR TRAILER',\n",
    "          'TOW TOW TRUCK', 'VAN VAN TRUCK', 'UTIL UTILITY', 'POLE POLE TRAILER', 'BOAT BOAT', 'H/TR HOUSE TRAILER', \n",
    "          'SEMI SEMI-TRAILER', 'TRLR TRAILER', 'LTRL LIGHT TRAILER', 'LSVT LOW SPEED VEHICLE - TRUCK', 'BUS BUS(OMNIBUS)',\n",
    "          'LIM LIMOUSINE(OMNIBUS)', 'HRSE HEARSE(AMBULANCE)', 'TAXI TAXI', 'DCOM DISABLED COMMERICAL', 'CMIX CEMENT MIXER',\n",
    "          'MOPD MOPED', 'MFH MANUFACTURED HOME', 'SNOW SNOWMOBILE', 'LSV LOW SPEED VEHICLE']\n",
    "    \n",
    "    \n",
    "\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    for itr in Dict_VehicleTypes:\n",
    "        ratio=fuzz.partial_ratio(itr[0].lower(), val.lower())\n",
    "        if ratio>=85:\n",
    "            flag=1\n",
    "    return flag\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different building classifiations that are encountered in the NYC OPEN data.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_BuildingClassification(val):\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    \n",
    "    Dict_Buildings=['A0-CAPE COD', 'A1-TWO STORIES - DETACHED SM OR MID', 'A2-ONE STORY - PERMANENT LIVING QUARTER', 'A3-LARGE SUBURBAN RESIDENCE', 'A4-CITY RESIDENCE ONE FAMILY', 'A5-ONE FAMILY ATTACHED OR SEMI-DETACHED', 'A6-SUMMER COTTAGE', 'A7-MANSION TYPE OR TOWN HOUSE', 'A8-BUNGALOW COLONY - COOPERATIVELY OWNED LAND', 'A9-MISCELLANEOUS ONE FAMILY', 'B1-TWO FAMILY BRICK', 'B2-TWO FAMILY FRAME', 'B3-TWO FAMILY CONVERTED FROM ONE FAMILY', 'B9-MISCELLANEOUS TWO FAMILY', 'C0-THREE FAMILIES', 'C1-OVER SIX FAMILIES WITHOUT STORES', 'C2-FIVE TO SIX FAMILIES', 'C3-FOUR FAMILIES', 'C4-OLD LAW TENEMENT', 'C5-CONVERTED DWELLINGS OR ROOMING HOUSE', 'C6-WALK-UP COOPERATIVE', 'C7-WALK-UP APT. OVER SIX FAMILIES WITH STORES', 'C8-WALK-UP CO-OP; CONVERSION FROM LOFT/WAREHOUSE', 'C9-GARDEN APARTMENTS', 'CM-MOBILE HOMES/TRAILER PARKS', 'D0-ELEVATOR CO-OP; CONVERSION FROM LOFT/WAREHOUSE', 'D1-ELEVATOR APT; SEMI-FIREPROOF WITHOUT STORES', 'D2-ELEVATOR APT; ARTISTS IN RESIDENCE', 'D3-ELEVATOR APT; FIREPROOF WITHOUT STORES', 'D4-ELEVATOR COOPERATIVE', 'D5-ELEVATOR APT; CONVERTED', 'D6-ELEVATOR APT; FIREPROOF WITH STORES', 'D7-ELEVATOR APT; SEMI-FIREPROOF WITH STORES', 'D8-ELEVATOR APT; LUXURY TYPE', 'D9-ELEVATOR APT; MISCELLANEOUS', 'E1-FIREPROOF WAREHOUSE', 'E2-CONTRACTORS WAREHOUSE', 'E3-SEMI-FIREPROOF WAREHOUSE', 'E4-METAL FRAME WAREHOUSE', 'E7-SELF-STORAGE WAREHOUSES', 'E9-MISCELLANEOUS WAREHOUSE', 'F1-FACTORY; HEAVY MANUFACTURING - FIREPROOF', 'F2-FACTORY; SPECIAL CONSTRUCTION - FIREPROOF', 'F4-FACTORY; INDUSTRIAL SEMI-FIREPROOF', 'F5-FACTORY; LIGHT MANUFACTURING', 'F8-FACTORY; TANK FARM', 'F9-FACTORY; INDUSTRIAL-MISCELLANEOUS', 'G0-GARAGE; RESIDENTIAL TAX CLASS 1', 'G1-ALL PARKING GARAGES', 'G2-AUTO BODY/COLLISION OR AUTO REPAIR', 'G3-GAS STATION WITH RETAIL STORE', 'G4-GAS STATION WITH SERVICE/AUTO REPAIR', 'G5-GAS STATION ONLY WITH/WITHOUT SMALL KIOSK', 'G6-LICENSED PARKING LOT', 'G7-UNLICENSED PARKING LOT', 'G8-CAR SALES/RENTAL WITH SHOWROOM', 'G9-MISCELLANEOUS GARAGE OR GAS STATION', 'GU-CAR SALES OR RENTAL LOTS WITHOUT SHOWROOM', 'GW-CAR WASH OR LUBRITORIUM FACILITY', 'G9-MISCELLANEOUS GARAGE', 'HB-BOUTIQUE: 10-100 ROOMS, W/LUXURY FACILITIES, THEMED, STYLISH, W/FULL SVC ACCOMMODATIONS', 'HH-HOSTELS- BED RENTALS IN DORMITORY-LIKE SETTINGS W/SHARED ROOMS & BATHROOMS', 'HR-SRO- 1 OR 2 PEOPLE HOUSED IN INDIVIDUAL ROOMS IN MULTIPLE DWELLING AFFORDABLE HOUSING', 'HS-EXTENDED STAY/SUITE: AMENITIES SIMILAR TO APT; TYPICALLY CHARGE WEEKLY RATES & LESS EXPENSIVE THAN FULL-SERVICE HOTEL', 'H1-LUXURY HOTEL', 'H2-FULL SERVICE HOTEL', 'H3-LIMITED SERVICE; MANY AFFILIATED WITH NATIONAL CHAIN', 'H4-MOTEL', 'H5-HOTEL; PRIVATE CLUB, LUXURY TYPE', 'H6-APARTMENT HOTEL', 'H7-APARTMENT HOTEL - COOPERATIVELY OWNED', 'H8-DORMITORY', 'H9-MISCELLANEOUS HOTEL', 'I1-HOSPITAL, SANITARIUM, MENTAL INSTITUTION', 'I2-INFIRMARY', 'I3-DISPENSARY', 'I4-HOSPITAL; STAFF FACILITY', 'I5-HEALTH CENTER, CHILD CENTER, CLINIC', 'I6-NURSING HOME', 'I7-ADULT CARE FACILITY', 'I9-MISCELLANEOUS HOSPITAL, HEALTH CARE FACILITY', 'J1-THEATRE; ART TYPE LESS THAN 400 SEATS', 'J2-THEATRE; ART TYPE MORE THAN 400 SEATS', 'J3-MOTION PICTURE THEATRE WITH BALCONY', 'J4-LEGITIMATE THEATRE, SOLE USE', 'J5-THEATRE IN MIXED-USE BUILDING', 'J6-TELEVISION STUDIO', 'J7-OFF BROADWAY TYPE THEATRE', 'J8-MULTIPLEX PICTURE THEATRE', 'J9-MISCELLANEOUS THEATRE', 'K1-ONE STORY RETAIL BUILDING', 'K2-MULTI-STORY RETAIL BUILDING (2 OR MORE)', 'K3-MULTI-STORY DEPARTMENT STORE', 'K4-PREDOMINANT RETAIL WITH OTHER USES', 'K5-STAND-ALONE FOOD ESTABLISHMENT', 'K6-SHOPPING CENTER WITH OR WITHOUT PARKING', 'K7-BANKING FACILITIES WITH OR WITHOUT PARKING', \"K8-BIG BOX RETAIL: NOT AFFIXED & STANDING ON OWN LOT W/PARKING, E.G. COSTCO & BJ'S\", 'K9-MISCELLANEOUS STORE BUILDING', 'L1-LOFT; OVER 8 STORIES (MID MANH. TYPE)', 'L2-LOFT; FIREPROOF AND STORAGE TYPE WITHOUT STORES', 'L3-LOFT; SEMI-FIREPROOF', 'L8-LOFT; WITH RETAIL STORES OTHER THAN TYPE ONE', 'L9-MISCELLANEOUS LOFT', 'M1-CHURCH, SYNAGOGUE, CHAPEL', 'M2-MISSION HOUSE (NON-RESIDENTIAL)', 'M3-PARSONAGE, RECTORY', 'M4-CONVENT', 'M9-MISCELLANEOUS RELIGIOUS FACILITY', 'N1-ASYLUM', 'N2-HOME FOR INDIGENT CHILDREN, AGED, HOMELESS', 'N3-ORPHANAGE', 'N4-DETENTION HOUSE FOR WAYWARD GIRLS', 'N9-MISCELLANEOUS ASYLUM, HOME', 'O1-OFFICE ONLY - 1 STORY', 'O2-OFFICE ONLY 2 - 6 STORIES', 'O3-OFFICE ONLY 7 - 19 STORIES', 'O4-OFFICE ONLY WITH OR WITHOUT COMM - 20 STORIES OR MORE', 'O5-OFFICE WITH COMM - 1 TO 6 STORIES', 'O6-OFFICE WITH COMM 7 - 19 STORIES', 'O7-PROFESSIONAL BUILDINGS/STAND ALONE FUNERAL HOMES', 'O8-OFFICE WITH APARTMENTS ONLY (NO COMM)', 'O9-MISCELLANEOUS AND OLD STYLE BANK BLDGS.', 'P1-CONCERT HALL', 'P2-LODGE ROOM', 'P3-YWCA, YMCA, YWHA, YMHA, PAL', 'P4-BEACH CLUB', 'P5-COMMUNITY CENTER', 'P6-AMUSEMENT PLACE, BATH HOUSE, BOAT HOUSE', 'P7-MUSEUM', 'P8-LIBRARY', 'P9-MISCELLANEOUS INDOOR PUBLIC ASSEMBLY', 'Q1-PARKS/RECREATION FACILTY', 'Q2-PLAYGROUND', 'Q3-OUTDOOR POOL', 'Q4-BEACH', 'Q5-GOLF COURSE', 'Q6-STADIUM, RACE TRACK, BASEBALL FIELD', 'Q7-TENNIS COURT', 'Q8-MARINA, YACHT CLUB', 'Q9-MISCELLANEOUS OUTDOOR RECREATIONAL FACILITY', 'RA-CULTURAL, MEDICAL, EDUCATIONAL, ETC.', 'RB-OFFICE SPACE', 'RG-INDOOR PARKING', 'RH-HOTEL/BOATEL', 'RK-RETAIL SPACE', 'RP-OUTDOOR PARKING', 'RR-CONDOMINIUM RENTALS', 'RS-NON-BUSINESS STORAGE SPACE', 'RT-TERRACES/GARDENS/CABANAS', 'RW-WAREHOUSE/FACTORY/INDUSTRIAL', 'R0-SPECIAL CONDOMINIUM BILLING LOT', 'R1-CONDO; RESIDENTIAL UNIT IN 2-10 UNIT BLDG.', 'R2-CONDO; RESIDENTIAL UNIT IN WALK-UP BLDG.', 'R3-CONDO; RESIDENTIAL UNIT IN 1-3 STORY BLDG.', 'R4-CONDO; RESIDENTIAL UNIT IN ELEVATOR BLDG.', 'R5-MISCELLANEOUS COMMERCIAL', 'R6-CONDO; RESID.UNIT OF 1-3 UNIT BLDG-ORIG CLASS 1', 'R7-CONDO; COMML.UNIT OF 1-3 UNIT BLDG-ORIG CLASS 1', 'R8-CONDO; COMML.UNIT OF 2-10 UNIT BLDG.', 'R9-CO-OP WITHIN A CONDOMINIUM', 'RR-CONDO RENTALS', 'S0-PRIMARILY 1 FAMILY WITH 2 STORES OR OFFICES', 'S1-PRIMARILY 1 FAMILY WITH 1 STORE OR OFFICE', 'S2-PRIMARILY 2 FAMILY WITH 1 STORE OR OFFICE', 'S3-PRIMARILY 3 FAMILY WITH 1 STORE OR OFFICE', 'S4-PRIMARILY 4 FAMILY WITH 1 STORE OROFFICE', 'S5-PRIMARILY 5-6 FAMILY WITH 1 STORE OR OFFICE', 'S9-SINGLE OR MULTIPLE DWELLING WITH STORES OR OFFICES', 'T1-AIRPORT, AIRFIELD, TERMINAL', 'T2-PIER, DOCK, BULKHEAD', 'T9-MISCELLANEOUS TRANSPORTATION FACILITY', 'U0-UTILITY COMPANY LAND AND BUILDING', 'U1-BRIDGE, TUNNEL, HIGHWAY', 'U2-GAS OR ELECTRIC UTILITY', 'U3-CEILING RAILROAD', 'U4-TELEPHONE UTILITY', 'U5-COMMUNICATION FACILITY OTHER THAN TELEPHONE', 'U6-RAILROAD - PRIVATE OWNERSHIP', 'U7-TRANSPORTATION - PUBLIC OWNERSHIP', 'U8-REVOCABLE CONSENT', 'U9-MISCELLANEOUS UTILITY PROPERTY', 'V0-ZONED RESIDENTIAL; NOT MANHATTAN', 'V1-ZONED COMMERCIAL OR MANHATTAN RESIDENTIAL', 'V2-ZONED COMMERCIAL ADJACENT TO CLASS 1 DWELLING: NOT MANHATTAN', 'V3-ZONED PRIMARILY RESIDENTIAL; NOT MANHATTAN', 'V4-POLICE OR FIRE DEPARTMENT', 'V5-SCHOOL SITE OR YARD', 'V6-LIBRARY, HOSPITAL OR MUSEUM', 'V7-PORT AUTHORITY OF NEW YORK AND NEW JERSEY', 'V8-NEW YORK STATE OR US GOVERNMENT', 'V9-MISCELLANEOUS VACANT LAND', 'W1-PUBLIC ELEMENTARY, JUNIOR OR SENIOR HIGH', 'W2-PAROCHIAL SCHOOL, YESHIVA', 'W3-SCHOOL OR ACADEMY', 'W4-TRAINING SCHOOL', 'W5-CITY UNIVERSITY', 'W6-OTHER COLLEGE AND UNIVERSITY', 'W7-THEOLOGICAL SEMINARY', 'W8-OTHER PRIVATE SCHOOL', 'W9-MISCELLANEOUS EDUCATIONAL FACILITY', 'Y1-FIRE DEPARTMENT', 'Y2-POLICE DEPARTMENT', 'Y3-PRISON, JAIL, HOUSE OF DETENTION', 'Y4-MILITARY AND NAVAL INSTALLATION', 'Y5-DEPARTMENT OF REAL ESTATE', 'Y6-DEPARTMENT OF SANITATION', 'Y7-DEPARTMENT OF PORTS AND TERMINALS', 'Y8-DEPARTMENT OF PUBLIC WORKS', 'Y9-DEPARTMENT OF ENVIRONMENTAL PROTECTION', 'Z0-TENNIS COURT, POOL, SHED, ETC.', 'Z1-COURT HOUSE', 'Z2-PUBLIC PARKING AREA', 'Z3-POST OFFICE', 'Z4-FOREIGN GOVERNMENT', 'Z5-UNITED NATIONS', 'Z7-EASEMENT', 'Z8-CEMETERY', 'Z9-OTHER MISCELLANEOUS']\n",
    "    \n",
    "    \n",
    "    flag=0\n",
    "    for itr2 in Dict_Buildings:\n",
    "        final=fuzz.partial_ratio(itr2.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different types of location that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different types of location possible in New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_LocationTypes(val):\n",
    "    \n",
    "    Dict_LocationTypes=['accounting', 'airport', 'amusement_park', 'aquarium', 'art_gallery', 'atm', 'bakery', 'bank', \n",
    "                'bar', 'beauty_salon', 'bicycle_store', 'book_store', 'bowling_alley', 'bus_station', 'cafe', \n",
    "                'campground', 'car_dealer', 'car_rental', 'car_repair', 'car_wash', 'casino', 'cemetery', 'church',\n",
    "                'city_hall', 'clothing_store', 'convenience_store', 'courthouse', 'dentist', 'department_store',\n",
    "                'doctor', 'drugstore', 'electrician', 'electronics_store', 'embassy', 'fire_station', 'florist',\n",
    "                'funeral_home', 'furniture_store', 'gas_station', 'grocery_or_supermarket', 'gym', 'hair_care',\n",
    "                'hardware_store', 'hindu_temple', 'home_goods_store', 'hospital', 'insurance_agency', \n",
    "                'jewelry_store', 'laundry', 'lawyer', 'library', 'light_rail_station', 'liquor_store',\n",
    "                'local_government_office', 'locksmith', 'lodging', 'meal_delivery', 'meal_takeaway', 'mosque',\n",
    "                'movie_rental', 'movie_theater', 'moving_company', 'museum', 'night_club', 'painter', 'park',\n",
    "                'parking', 'pet_store', 'pharmacy', 'physiotherapist', 'plumber', 'police', 'post_office',\n",
    "                'primary_school', 'real_estate_agency', 'restaurant', 'roofing_contractor', 'rv_park', 'school',\n",
    "                'secondary_school', 'shoe_store', 'shopping_mall', 'spa', 'stadium', 'storage', 'store',\n",
    "                'subway_station', 'supermarket', 'synagogue', 'taxi_stand', 'tourist_attraction', 'train_station',\n",
    "                'transit_station', 'travel_agency', 'university', 'veterinary_care', 'zoo', \n",
    "                'administrative_area_level_1', 'administrative_area_level_2', 'administrative_area_level_3',\n",
    "                'administrative_area_level_4', 'administrative_area_level_5', 'archipelago', 'colloquial_area',\n",
    "                'continent', 'country', 'establishment', 'finance', 'floor', 'food', 'general_contractor',\n",
    "                'geocode', 'health', 'intersection', 'locality', 'natural_feature', 'neighborhood', \n",
    "                'place_of_worship', 'point_of_interest', 'political', 'post_box', 'postal_code',\n",
    "                'postal_code_prefix', 'postal_code_suffix', 'postal_town', 'premise', 'room', 'route',\n",
    "                'street_address', 'street_number', 'sublocality', 'sublocality_level_1', 'sublocality_level_2', \n",
    "                'sublocality_level_3', 'sublocality_level_4', 'sublocality_level_5', 'subpremise', 'town_square',\n",
    "                'ABANDONED BUILDING','AIRPORT TERMINAL', 'ATM', 'BANK', 'BAR/NIGHT CLUB', 'BEAUTY & NAIL SALON',\n",
    "                'BOOK/CARD', 'BRIDGE', 'BUS (NYC TRANSIT)', 'BUS (OTHER)', 'BUS STOP', 'BUS TERMINAL',\n",
    "                'CANDY STORE', 'CEMETERY', 'CHAIN STORE', 'CHECK CASHING BUSINESS', 'CHURCH', \n",
    "                'CLOTHING/BOUTIQUE', 'COMMERCIAL BUILDING', 'CONSTRUCTION SITE', 'DEPARTMENT STORE',\n",
    "                'DOCTOR/DENTIST OFFICE', 'DRUG STORE', 'DRY CLEANER/LAUNDRY', 'FACTORY/WAREHOUSE', 'FAST FOOD',\n",
    "                'FERRY/FERRY TERMINAL', 'FOOD SUPERMARKET', 'GAS STATION', 'GROCERY/BODEGA', 'GYM/FITNESS FACILITY',\n",
    "                'HIGHWAY/PARKWAY', 'HOSPITAL', 'HOTEL/MOTEL', 'JEWELRY', 'LIQUOR STORE', 'LOAN COMPANY',\n",
    "                'MAILBOX INSIDE', 'MAILBOX OUTSIDE', 'MARINA/PIER', 'MOSQUE', 'OPEN AREAS (OPEN LOTS)', 'OTHER',\n",
    "                'OTHER HOUSE OF WORSHIP', 'PARK/PLAYGROUND', 'PARKING LOT/GARAGE (PRIVATE)',\n",
    "                'PARKING LOT/GARAGE (PUBLIC)', 'PHOTO/COPY', 'PRIVATE/PAROCHIAL SCHOOL', 'PUBLIC BUILDING',\n",
    "                'PUBLIC SCHOOL', 'RESIDENCE - APT. HOUSE', 'RESIDENCE - PUBLIC HOUSING', 'RESIDENCE-HOUSE',\n",
    "                'RESTAURANT/DINER', 'SHOE', 'SMALL MERCHANT', 'SOCIAL CLUB/POLICY', 'STORAGE FACILITY', \n",
    "                'STORE UNCLASSIFIED', 'STREET', 'SYNAGOGUE', 'TAXI (LIVERY LICENSED)', 'TAXI (YELLOW LICENSED)', \n",
    "                'TAXI/LIVERY (UNLICENSED)', 'TELECOMM. STORE', 'TRAMWAY', 'TRANSIT - NYC SUBWAY', \n",
    "                'TRANSIT FACILITY (OTHER)', 'TUNNEL', 'VARIETY STORE', 'VIDEO STORE']\n",
    "    \n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    for itr in Dict_LocationTypes:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different website names that are encountered in the NYC OPEN data.\n",
    "import re\n",
    "def Validate_Website(InputData):\n",
    "    if InputData==None or type(InputData)!=str:\n",
    "        return 0\n",
    "    flag=0\n",
    "    itr = re.split('\\.|\\/',InputData.lower())\n",
    "    Dict_WebsiteExtensions = ['gov', 'com', 'org', 'info', 'http', 'https', 'net','edu','www','me','info',\n",
    "          'ai', 'nyc']\n",
    "    if any(item in itr for item in Dict_WebsiteExtensions):\n",
    "        flag=1\n",
    "    else:\n",
    "        flag=0\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different address and street name types that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different address and street names of New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_StreetAddress(InputData):\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    Dict_StreetNames = ['street','st.','avenue','boulevard', 'pl', 'rd', 'alley', 'ave', 'bridge', 'blvd', 'ln', 'dr',\n",
    "                  'slip', 'broaway', 'terrace', 'plaza', 'square', 'expy']\n",
    "    for itr in Dict_StreetNames:\n",
    "        if fuzz.partial_ratio(itr,InputData.lower())>=85 and InputData[0].isdigit() == True:\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "def Validate_StreetName(InputData):\n",
    "    if type(InputData)!=str or InputData==None:\n",
    "        return 0\n",
    "    Dict_StreetNames = ['street','st.','avenue','boulevard', 'pl', 'rd', 'alley', 'ave', 'bridge', 'blvd', 'ln', 'dr',\n",
    "                  'slip', 'broaway', 'terrace', 'plaza', 'square', 'expy']\n",
    "    for itr in Dict_StreetNames:\n",
    "        if fuzz.partial_ratio(itr,InputData.lower())>=85 and InputData[0].isdigit() == False:\n",
    "            return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different city agency types that are encountered in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different city agencies of New York.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_CityAgency(val):\n",
    "    Dict_Agencies=['Actuary, NYC Office of the (NYCOA)', 'Administrative Justice Coordinator, NYC Office of (AJC)', 'Administrative Tax Appeals, Office of', 'Administrative Trials and Hearings, Office of (OATH)', 'Aging, Department for the (DFTA)', \"Appointments, Mayor's Office of (MOA)\", 'Brooklyn Public Library (BPL)', 'Buildings, Department of (DOB)', 'Business Integrity Commission (BIC)', 'Campaign Finance Board (CFB)', 'Center for Innovation through Data Intelligence (CIDI)', 'Charter Revision Commission', 'Chief Medical Examiner, NYC Office of (OCME)', \"Children's Services, Administration for (ACS)\", 'City Clerk, Office of the (CLERK)', 'City Council, New York', 'City Planning, Department of (DCP)', 'City University of New York (CUNY)', 'Citywide Administrative Services, Department of (DCAS)', 'Citywide Event Coordination and Management, Office of (CECM)', 'Civic Engagement Commission (CEC)', 'Civil Service Commission (CSC)', 'Civilian Complaint Review Board (CCRB)', 'Climate Policy & Programs', 'Commission on Gender Equity (CGE)', 'Commission to Combat Police Corruption (CCPC)', 'Community Affairs Unit (CAU)', 'Community Boards (CB)', 'Comptroller (COMP)', 'Conflicts of Interest Board (COIB)', 'Consumer Affairs, Department of (DCA)', 'Consumer and Worker Protection, Department of (DCWP)', \"Contract Services, Mayor's Office of (MOCS)\", 'Correction, Board of (BOC)', 'Correction, Department of (DOC)', \"Criminal Justice, Mayor's Office of\", 'Cultural Affairs, Department of (DCLA)', \"Data Analytics, Mayor's Office of (MODA)\", 'Design and Construction, Department of (DDC)', 'District Attorney - Bronx County', 'District Attorney - Kings County (Brooklyn)', 'District Attorney - New York County (Manhattan)', 'District Attorney - Queens County', 'District Attorney - Richmond County (Staten Island)', 'Education, Department of (DOE)', 'Elections, Board of (BOE)', 'Emergency Management, NYC', 'Environmental Coordination, Mayor’s Office of (MOEC)', 'Environmental Protection, Department of (DEP)', 'Equal Employment Practices Commission (EEPC)', 'Finance, Department of (DOF)', 'Fire Department, New York City (FDNY)', 'Fiscal Year 2005 Securitization Corporation', 'Food Policy Director, Office of the', 'GreeNYC (GNYC)', 'Health and Mental Hygiene, Department of (DOHMH)', 'Homeless Services, Department of (DHS)', 'Housing Authority, New York City (NYCHA)', 'Housing Preservation and Development, Department of (HPD)', 'Housing Recovery Operations (HRO)', 'Hudson Yards Infrastructure Corporation', 'Human Resources Administration (HRA)', 'Human Rights, City Commission on (CCHR)', \"Immigrant Affairs, Mayor's Office of (MOIA)\", 'Independent Budget Office, NYC (IBO)', \"Information Privacy, Mayor's Office of (MOIP)\", 'Information Technology and Telecommunications, Department of (DOITT)', 'Inspector General NYPD, Office of the', \"Intergovernmental Affairs, Mayor's Office of (MOIGA)\", 'Investigation, Department of (DOI)', \"Judiciary, Mayor's Advisory Committee on the (MACJ)\", 'Labor Relations, NYC Office of (OLR)', 'Landmarks Preservation Commission (LPC)', 'Law Department (LAW)', 'Library, Brooklyn Public (BPL)', 'Library, New York Public (NYPL)', 'Library, Queens Public (QL)', 'Loft Board (LOFT)', 'Management and Budget, Office of (OMB)', \"Mayor's Committee on City Marshals (MCCM)\", \"Mayor's Fund to Advance NYC (Mayor's Fund)\", \"Mayor's Office (OM)\", \"Mayor's Office for Economic Opportunity\", \"Mayor's Office for International Affairs (IA)\", \"Mayor's Office for People with Disabilities (MOPD)\", \"Mayor's Office of Environmental Remediation (OER)\", \"Mayor's Office of Special Projects & Community Events (MOSPCE)\", \"Mayor's Office of the Chief Technology Officer\", 'Mayor’s Office of Minority and Women-Owned Business Enterprises (OMWBE)', 'Mayor’s Office of Strategic Partnerships (OSP)', 'Mayor’s Office to End Domestic and Gender-Based Violence (ENDGBV)', \"Media and Entertainment, Mayor's Office of (MOME)\", 'Media, NYC', 'NYC & Company (NYCGO)', \"NYC Children's Cabinet\", 'NYC Cyber Command', 'NYC Economic Development Corporation (NYCEDC)', \"NYC Employees' Retirement System (NYCERS)\", 'NYC Health + Hospitals', 'NYC Service (SERVICE)', 'NYC Young Men’s Initiative', 'New York City Transitional Finance Authority (TFA)', 'New York Public Library (NYPL)', 'Office of Recovery & Resiliency', 'Office of ThriveNYC', 'Office of the Census for NYC', \"Operations, Mayor's Office of (OPS)\", 'Parks and Recreation, Department of (DPR)', 'Payroll Administration, Office of (OPA)', 'Police Department (NYPD)', 'Police Pension Fund (PPF)', 'Probation, Department of (DOP)', 'Procurement Policy Board (PPB)', 'Property Tax Reform, Advisory Commission on', 'Public Administrator - Bronx County (BCPA)', 'Public Administrator - Kings County (KCPA)', 'Public Administrator - New York County (NYCountyPA)', 'Public Administrator - Queens County (QPA)', 'Public Administrator - Richmond County (RCPA)', 'Public Advocate (PUB ADV)', 'Public Design Commission', 'Queens Public Library (QPL)', 'Records and Information Services, Department of (DORIS)', 'Rent Guidelines Board (RGB)', 'Sales Tax Asset Receivable Corporation (STAR)', 'Sanitation, Department of (DSNY)', 'School Construction Authority (SCA)', 'Small Business Services (SBS)', 'Social Services, Department of (DSS)', 'Special Commissioner of Investigation for the New York City School District', 'Special Enforcement, Mayor’s Office of (OSE)', 'Special Narcotics Prosecutor, NYC Office of the (SNP)', 'Standards and Appeals, Board of (BSA)', \"Sustainability, Mayor's Office Of\", 'TSASC, Inc.', 'Tax Appeals Tribunal, New York City (TAT)', 'Tax Commission, New York City (TC)', 'Taxi and Limousine Commission (TLC)', \"Teachers' Retirement System of the City of New York\", 'Transportation, Department of (DOT)', \"Veterans' Services, Department of (DVS)\", 'Water Board (NYWB)', 'Water Finance Authority, NYC Municipal (NYW)', \"Workforce Development, Mayor's Office of\", 'Youth and Community Development, Department of (DYCD)']\n",
    "\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    for itr in Dict_Agencies:\n",
    "        final=fuzz.partial_ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different academic majors pursued by people in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different majors opted by the people.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_AcademicMajor(val):\n",
    "    Dict_Majors=['Accounting', 'Actuarial Science', 'Advertising', 'Aerospace Engineering', 'African Languages, Literatures, and Linguistics', 'African Studies', 'African-American Studies', 'Agricultural Business and Management', 'Agricultural Economics', 'Agricultural Education', 'Agricultural Journalism', 'Agricultural Mechanization', 'Agricultural Technology Management', 'Agricultural/Biological Engineering and Bioengineering', 'Agriculture', 'Agronomy and Crop Science', 'Air Traffic Control', 'American History', 'American Literature', 'American Sign Language', 'American Studies', 'Anatomy', 'Ancient Studies', 'Animal Behavior and Ethology', 'Animal Science', 'Animation and Special Effects', 'Anthropology', 'Applied Mathematics', 'Aquaculture', 'Aquatic Biology', 'Arabic', 'Archeology', 'Architectural Engineering', 'Architectural History', 'Architecture', 'Art', 'Art Education', 'Art History', 'Art Therapy', 'Artificial Intelligence and Robotics', 'Asian-American Studies', 'Astronomy', 'Astrophysics', 'Athletic Training', 'Atmospheric Science', 'Automotive Engineering', 'Aviation', 'Bakery Science', 'Biblical Studies', 'Biochemistry', 'Bioethics', 'Biology', 'Biomedical Engineering', 'Biomedical Science', 'Biopsychology', 'Biotechnology', 'Botany/Plant Biology', 'Business Administration/Management', 'Business Communications', 'Business Education', 'Canadian Studies', 'Caribbean Studies', 'Cell Biology', 'Ceramic Engineering', 'Ceramics', 'Chemical Engineering', 'Chemical Physics', 'Chemistry', 'Child Care', 'Child Development', 'Chinese', 'Chiropractic', 'Church Music', 'Cinematography and Film/Video Production', 'Circulation Technology', 'Civil Engineering', 'Classics', 'Clinical Psychology', 'Cognitive Psychology', 'Communication Disorders', 'Communications Studies/Speech Communication and Rhetoric', 'Comparative Literature', 'Computer and Information Science', 'Computer Engineering', 'Computer Graphics', 'Computer Systems Analysis', 'Construction Management', 'Counseling', 'Crafts', 'Creative Writing', 'Criminal Science', 'Criminology', 'Culinary Arts', 'Dance', 'Data Processing', 'Dental Hygiene', 'Developmental Psychology', 'Diagnostic Medical Sonography', 'Dietetics', 'Digital Communications and Media/Multimedia', 'Drawing', 'Early Childhood Education', 'East Asian Studies', 'East European Studies', 'Ecology', 'Economics', 'Education', 'Education Administration', 'Education of the Deaf', 'Educational Psychology', 'Electrical Engineering', 'Elementary Education', 'Engineering Mechanics', 'Engineering Physics', 'English', 'English Composition', 'English Literature', 'Entomology', 'Entrepreneurship', 'Environmental Design/Architecture', 'Environmental Science', 'Environmental/Environmental Health Engineering', 'Epidemiology', 'Equine Studies', 'Ethnic Studies', 'European History', 'Experimental Pathology', 'Experimental Psychology', 'Fashion Design', 'Fashion Merchandising', 'Feed Science', 'Fiber, Textiles, and Weaving Arts', 'Film', 'Finance', 'Floriculture', 'Food Science', 'Forensic Science', 'Forestry', 'French', 'Furniture Design', 'Game Design', 'Gay and Lesbian Studies', 'Genetics', 'Geography', 'Geological Engineering', 'Geology', 'Geophysics', 'German', 'Gerontology', 'Government', 'Graphic Design', 'Health Administration', 'Hebrew', 'Hispanic-American, Puerto Rican, and Chicano Studies', 'Historic Preservation', 'History', 'Home Economics', 'Horticulture', 'Hospitality', 'Human Development', 'Human Resources Management', 'Illustration', 'Industrial Design', 'Industrial Engineering', 'Industrial Management', 'Industrial Psychology', 'Information Technology', 'Interior Architecture', 'Interior Design', 'International Agriculture', 'International Business', 'International Relations', 'International Studies', 'Islamic Studies', 'Italian', 'Japanese', 'Jazz Studies', 'Jewelry and Metalsmithing', 'Jewish Studies', 'Journalism', 'Kinesiology', 'Korean', 'Land Use Planning and Management', 'Landscape Architecture', 'Landscape Horticulture', 'Latin American Studies', 'Library Science', 'Linguistics', 'Logistics Management', 'Management Information Systems', 'Managerial Economics', 'Marine Biology', 'Marine Science', 'Marketing', 'Mass Communication', 'Massage Therapy', 'Materials Science', 'Mathematics', 'Mechanical Engineering', 'Medical Technology', 'Medieval and Renaissance Studies', 'Mental Health Services', 'Merchandising and Buying Operations', 'Metallurgical Engineering', 'Microbiology', 'Middle Eastern Studies', 'Military Science', 'Mineral Engineering', 'Missions', 'Modern Greek', 'Molecular Biology', 'Molecular Genetics', 'Mortuary Science', 'Museum Studies', 'Music', 'Music Education', 'Music History', 'Music Management', 'Music Therapy', 'Musical Theater', 'Native American Studies', 'Natural Resources Conservation', 'Naval Architecture', 'Neurobiology', 'Neuroscience', 'Nuclear Engineering', 'Nursing', 'Nutrition', 'Occupational Therapy', 'Ocean Engineering', 'Oceanography', 'Operations Management', 'Organizational Behavior Studies', 'Painting', 'Paleontology', 'Pastoral Studies', 'Peace Studies', 'Petroleum Engineering', 'Pharmacology', 'Pharmacy', 'Philosophy', 'Photography', 'Photojournalism', 'Physical Education', 'Physical Therapy', 'Physician Assistant', 'Physics', 'Physiological Psychology', 'Piano', 'Planetary Science', 'Plant Pathology', 'Playwriting and Screenwriting', 'Political Communication', 'Political Science', 'Portuguese', 'Pre-Dentistry', 'Pre-Law', 'Pre-Medicine', 'Pre-Optometry', 'Pre-Seminary', 'Pre-Veterinary Medicine', 'Printmaking', 'Psychology', 'Public Administration', 'Public Health', 'Public Policy Analysis', 'Public Relations', 'Radio and Television', 'Radiologic Technology', 'Range Science and Management', 'Real Estate', 'Recording Arts Technology', 'Recreation Management', 'Rehabilitation Services', 'Religious Studies', 'Respiratory Therapy', 'Risk Management', 'Rural Sociology', 'Russian', 'Scandinavian Studies', 'Sculpture', 'Slavic Languages and Literatures', 'Social Psychology', 'Social Work', 'Sociology', 'Soil Science', 'Sound Engineering', 'South Asian Studies', 'Southeast Asia Studies', 'Spanish', 'Special Education', 'Speech Pathology', 'Sport and Leisure Studies', 'Sports Management', 'Statistics', 'Surveying', 'Sustainable Resource Management', 'Teacher Education', 'Teaching English as a Second Language', 'Technical Writing', 'Technology Education', 'Textile Engineering', 'Theatre', 'Theology', 'Tourism', 'Toxicology', 'Turfgrass Science', 'Urban Planning', 'Urban Studies', 'Visual Communication', 'Voice', 'Web Design', 'Webmaster and Web Management', 'Welding Engineering', 'Wildlife Management', \"Women's Studies\", 'Youth Ministries', 'Zoology']\n",
    "\n",
    "    if type(val)!=str or val==None:\n",
    "        return 0\n",
    "    flag=0\n",
    "    \n",
    "    for itr in Dict_Majors:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different school subjects opted by school students in the NYC OPEN data.\n",
    "#We define a dictionary to represent the different possible school subjects opted by students.\n",
    "#We use fuzzy ratio method for validation.\n",
    "def Validate_SchoolSubject(val):\n",
    "    Dict_SchoolSubjects=['Math 7 (Grade Level Class for 7th Graders)', 'Pre-Algebra (Above Grade Level, but most students take this)', 'Algebra 1 (High School Course; most 8th graders take this, some really advanced 7th graders take this)', 'Math 8 (Grade Level Class for 8th Graders)', 'Geometry (High School Course for very advanced 8th graders)', 'Math Workshop (Extra Class to assist below grade level math students)', 'English:', 'English 7 (Grade level English class for 7th Graders)', 'Advanced English 7 (Advanced English class for 7th Graders)', 'English 8 (Grade Level English class for 8th Graders)', 'Advanced English 8 (Advance English class for 8th Graders)', 'Reading Workshop (Extra Class to assist below grade level English students)', 'Science:', 'Science 7 (Grade level Science class for 7th graders)', 'Advanced Science 7 (Advanced Science class for 7th graders)', 'Science 8 (Grade Level Science class for 8th graders)', 'Advanced Science 8 (Advance Science class for 8th graders)', 'Social Studies:', 'Social Studies 7 (Social Studies Class for 7th graders)', 'Social Studies 8 (Grade Level Social Studies Class for 8th Graders)', 'Advanced Social Studies 8 (Advanced Social Studies class for 8th Graders)', 'Electives (* means required to leave middle school):', 'Spanish', 'Spanish 1', 'Spanish 2', 'Spanish for Native Speakers', 'French', 'French 1', 'French 2', 'Introduction to World Languages', 'Art', 'Art Exploration', 'Art 2D', 'Art 3D', 'Humanities', 'STEM', 'GTT: Design and Modeling', 'GTT: Robotics and Automation', 'Projects in Industrial Technology (Woodshop)', 'Business', 'Career Explorations and Money Management', 'Computer Dimensions 1', 'Computer Dimensions 2', 'Family Science', 'Introduction to Family and Consumer Sciences', 'Exploring Sewing Technology', 'Exploring Foods and Nutrition', 'Performing Arts', 'Theater', 'Performance 1', 'Performance 2', 'Band', 'Beginning Band', 'Band 1', 'Band 2', 'Orchestra/Strings', 'Beginning Orchestra', 'Orchestra 1', 'Orchestra 2', 'Choir', 'Vocal Music', 'Chorus 1', 'Chorus 2', 'Creative Writing', 'SEEK (Gifted Class)', 'Physical Education*']\n",
    "\n",
    "    if type(val)!=str or val==None :\n",
    "        return 0\n",
    "    flag=0\n",
    "    \n",
    "    for itr in Dict_SchoolSubjects:\n",
    "        final=fuzz.ratio(itr.lower(), val.lower())\n",
    "        if final>=85:\n",
    "            flag=1\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This function validates the different phone numbers encountered in the NYC OPEN data.\n",
    "#We use regular expressions method for validation.\n",
    "import re\n",
    "def Validate_PhoneNumber(InputData):\n",
    "    if InputData==None or type(InputData)!=int or type(InputData)!=str:\n",
    "        return 0\n",
    "    if type(InputData)==int:\n",
    "        InputData=str(InputData)\n",
    "    InputData = InputData.replace('-','')\n",
    "    if re.search(r\"((\\(\\d{3}\\)?)|(\\d{3}))([\\s./]?)(\\d{3})([\\s./]?)(\\d{4})\", InputData):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Compute_Semantic_Profiling(arg):\n",
    "\n",
    "    if Validate_Lat_Long(arg)==1:\n",
    "        return 'lat_lon_cord'\n",
    "    elif ValidateBusiness(arg)==1:\n",
    "        return 'business_name'\n",
    "    elif Validate_Park(arg)==1:\n",
    "        return 'park_playground'\n",
    "    elif Validate_Color(arg)==1:\n",
    "        return 'color'\n",
    "    elif Validate_PhoneNumber(arg)==1:\n",
    "        return 'phone_number'\n",
    "    elif Validate_SchoolLevels(arg)==1:\n",
    "        return 'school_level'\n",
    "    elif Validate_City(arg)==1:\n",
    "        return 'city'\n",
    "    elif Validate_SchoolSubject(arg)==1:\n",
    "        return 'subject_in_school'\n",
    "    elif Validate_AcademicMajor(arg)==1:\n",
    "        return 'area_of_study'\n",
    "    elif Validate_Borough(arg)==1:\n",
    "        return 'borough'\n",
    "    elif Validate_LocationTypes(arg)==1:\n",
    "        return 'location_type'\n",
    "    elif Validate_VehicleType(arg)==1:\n",
    "        return 'vehicle_type'\n",
    "    elif Validate_CityAgency(arg)==1:\n",
    "        return 'person_name'\n",
    "    elif Validate_ZipCode(arg)==1:\n",
    "        return 'zip_code'\n",
    "    elif Validate_School(arg)==1:\n",
    "        return 'school_name'\n",
    "    elif Validate_Neighbourhood(arg)==1:\n",
    "        return 'neighborhood'\n",
    "    elif Validate_StreetName(arg)==1:\n",
    "        return 'street_name'\n",
    "    elif Validate_CarMake(arg)==1:\n",
    "        return 'car_make'\n",
    "    elif Validate_StreetAddress(arg)==1:\n",
    "        return 'address'\n",
    "    elif Validate_Website(arg)==1:\n",
    "        return 'website'\n",
    "    elif Validate_College(arg)==1:\n",
    "        return 'college_name'\n",
    "    elif Validate_CityAgency(arg)==1:\n",
    "        return 'city_agency'\n",
    "    elif Validate_BuildingClassification(arg)==1:\n",
    "        return 'building_classification'\n",
    "    else:\n",
    "        return 'other'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('Task-2_Results_JSON'):\n",
    "    os.makedirs('Task-2_Results_JSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "Dict_AllLabels=[\"person_name\", \"business_name\", \"phone_number\", \"address\", \"street_name\",\n",
    "\"city\", \"neighborhood\", \"lat_lon_cord\", \"zip_code\", \"borough\", \"school_name\",\n",
    "\"color\", \"car_make\", \"city_agency\", \"area_of_study\", \"subject_in_school\",\n",
    "\"school_level\", \"college_name\", \"website\", \"building_classification\",\n",
    "\"vehicle_type\", \"location_type\", \"park_playground\", \"other\"]\n",
    "\n",
    "Dict_CountCols=\"\"\n",
    "for x in contents:\n",
    "    def_cols=x.split('.')[1]\n",
    "    x=x.replace(\" \",\"\")\n",
    "    fc='NYCColumns/'+x\n",
    "    df=dd.read_csv(fc, sep=\"\\t\", names=[x.split('.')[1], 'dummy-column'])\n",
    "    df['Predicted_Labels']=df[def_cols].apply(lambda x: Compute_Semantic_Profiling(x))\n",
    "    Dict_CountCols=Dict_CountCols+str(df['Predicted_Labels'].count())+','\n",
    "    Dict_Predicts= list(set(list(df['Predicted_Labels'])))\n",
    "    \n",
    "    json_dict={\"column_name\": def_cols, \"semantic_types\":[]}\n",
    "    \n",
    "    for itr in Dict_Predicts:\n",
    "        if itr in Dict_AllLabels and itr!='other':\n",
    "            Dict_SemanticAnalysis={\"Compute_Semantic_Profiling\":itr, \"count\": int((df['Identified_Labels']==itr).sum())}\n",
    "            json_dict['semantic_types'].append(Dict_SemanticAnalysis)\n",
    "        else:\n",
    "            Dict_SemanticAnalysis={\"Compute_Semantic_Profiling\":itr,\"label\":\"\" ,\"count\": int((df['Identified_Labels']==itr).sum())}\n",
    "            json_dict['semantic_types'].append(Dict_SemanticAnalysis)\n",
    "    json_file_path='output-task2/Task-2_Results_JSON/'+x.split('.')[0]+'.'+x.split('.')[1]+'.json'\n",
    "    with open(json_file_path, 'w', newline='\\n') as json_file:\n",
    "        json.dump(json_dict, json_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}