notebooks/wprdc_exploration/wprdc_exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from ckanapi import RemoteCKAN\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "\n",
    "ckan = RemoteCKAN('https://data.wprdc.org')"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Download dataset and resource metadata"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "333"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "packages = ckan.action.current_package_list_with_resources(limit=500)\n",
    "len(packages)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create single table of all resources"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "data": {
      "text/plain": "                                              mimetype cache_url hash  \\\nid                                                                      \n36fb4629-8003-4acc-a1ca-3302778a530d          text/csv      None        \n4ab1e23f-3262-4bd3-adbf-f72f0119108b          text/csv      None        \n01297a0a-4160-41b3-8caf-90f7857ab7bd         text/html      None        \nb646336a-deb4-4075-aee4-c5d28d88c426          text/csv      None        \n7f7072ce-7c19-4813-a45c-6135cf4505bb  application/json      None        \n...                                                ...       ...  ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed   application/zip      None        \n2bd36135-1104-4988-92c6-64a43ffbdf0d   application/zip      None        \n1ac3a889-61b5-452a-950d-c7533d9a5e23  application/json      None        \n6c45d829-9b68-4093-a054-8a79784fa060   application/zip      None        \n2eb6f0db-8c61-4854-be28-7661c43f2b9f          text/csv      None        \n\n                                                                            description  \\\nid                                                                                        \n36fb4629-8003-4acc-a1ca-3302778a530d  Since this is a large table, if you wish to do...   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b                                                      \n01297a0a-4160-41b3-8caf-90f7857ab7bd                                                      \nb646336a-deb4-4075-aee4-c5d28d88c426                                                      \n7f7072ce-7c19-4813-a45c-6135cf4505bb                                                      \n...                                                                                 ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed                                                      \n2bd36135-1104-4988-92c6-64a43ffbdf0d                                                      \n1ac3a889-61b5-452a-950d-c7533d9a5e23                                                      \n6c45d829-9b68-4093-a054-8a79784fa060                                  zippped shapefile   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f                                                      \n\n                                                                                   name  \\\nid                                                                                        \n36fb4629-8003-4acc-a1ca-3302778a530d                            Hourly Air Quality Data   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b                                     Daily AQI Data   \n01297a0a-4160-41b3-8caf-90f7857ab7bd  Air Quality List of Acronyms and Flag Definitions   \nb646336a-deb4-4075-aee4-c5d28d88c426                                   Sensor Locations   \n7f7072ce-7c19-4813-a45c-6135cf4505bb                         Sensor Locations (GeoJSON)   \n...                                                                                 ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed                2017 Parking Purchases (compressed)   \n2bd36135-1104-4988-92c6-64a43ffbdf0d                2018 Parking Purchases (compressed)   \n1ac3a889-61b5-452a-950d-c7533d9a5e23                      Green Infrastructure Projects   \n6c45d829-9b68-4093-a054-8a79784fa060                      Green Infrastructure Projects   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f                      Green Infrastructure Projects   \n\n                                       format  \\\nid                                              \n36fb4629-8003-4acc-a1ca-3302778a530d      CSV   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b      CSV   \n01297a0a-4160-41b3-8caf-90f7857ab7bd     HTML   \nb646336a-deb4-4075-aee4-c5d28d88c426      CSV   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  GeoJSON   \n...                                       ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed      ZIP   \n2bd36135-1104-4988-92c6-64a43ffbdf0d      ZIP   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  GeoJSON   \n6c45d829-9b68-4093-a054-8a79784fa060      SHP   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f      CSV   \n\n                                                                                    url  \\\nid                                                                                        \n36fb4629-8003-4acc-a1ca-3302778a530d  https://tools.wprdc.org/downstream/36fb4629-80...   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  https://data.wprdc.org/datastore/dump/4ab1e23f...   \n01297a0a-4160-41b3-8caf-90f7857ab7bd  https://tools.wprdc.org/pages/air-quality-docs...   \nb646336a-deb4-4075-aee4-c5d28d88c426  https://data.wprdc.org/datastore/dump/b646336a...   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  https://data.wprdc.org/dataset/c7b3266c-adc6-4...   \n...                                                                                 ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed  https://data.wprdc.org/dataset/e87ccf87-90f6-4...   \n2bd36135-1104-4988-92c6-64a43ffbdf0d  https://data.wprdc.org/dataset/e87ccf87-90f6-4...   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  https://data.wprdc.org/dataset/30bbb37c-ca2d-4...   \n6c45d829-9b68-4093-a054-8a79784fa060  https://data.wprdc.org/dataset/30bbb37c-ca2d-4...   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  https://data.wprdc.org/dataset/30bbb37c-ca2d-4...   \n\n                                      datastore_active cache_last_updated  \\\nid                                                                          \n36fb4629-8003-4acc-a1ca-3302778a530d              True               None   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b              True               None   \n01297a0a-4160-41b3-8caf-90f7857ab7bd             False               None   \nb646336a-deb4-4075-aee4-c5d28d88c426              True               None   \n7f7072ce-7c19-4813-a45c-6135cf4505bb              True               None   \n...                                                ...                ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed             False               None   \n2bd36135-1104-4988-92c6-64a43ffbdf0d             False               None   \n1ac3a889-61b5-452a-950d-c7533d9a5e23             False               None   \n6c45d829-9b68-4093-a054-8a79784fa060             False               None   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f              True               None   \n\n                                                                package_id  \\\nid                                                                           \n36fb4629-8003-4acc-a1ca-3302778a530d  c7b3266c-adc6-41c0-b19a-8d4353bfcdaf   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  c7b3266c-adc6-41c0-b19a-8d4353bfcdaf   \n01297a0a-4160-41b3-8caf-90f7857ab7bd  c7b3266c-adc6-41c0-b19a-8d4353bfcdaf   \nb646336a-deb4-4075-aee4-c5d28d88c426  c7b3266c-adc6-41c0-b19a-8d4353bfcdaf   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  c7b3266c-adc6-41c0-b19a-8d4353bfcdaf   \n...                                                                    ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed  e87ccf87-90f6-4023-ba4e-5a2f58793a72   \n2bd36135-1104-4988-92c6-64a43ffbdf0d  e87ccf87-90f6-4023-ba4e-5a2f58793a72   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  30bbb37c-ca2d-47e6-bae4-72c57272383c   \n6c45d829-9b68-4093-a054-8a79784fa060  30bbb37c-ca2d-47e6-bae4-72c57272383c   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  30bbb37c-ca2d-47e6-bae4-72c57272383c   \n\n                                      ... mimetype_inner  \\\nid                                    ...                  \n36fb4629-8003-4acc-a1ca-3302778a530d  ...           None   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  ...           None   \n01297a0a-4160-41b3-8caf-90f7857ab7bd  ...           None   \nb646336a-deb4-4075-aee4-c5d28d88c426  ...           None   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  ...           None   \n...                                   ...            ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed  ...           None   \n2bd36135-1104-4988-92c6-64a43ffbdf0d  ...           None   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  ...           None   \n6c45d829-9b68-4093-a054-8a79784fa060  ...           None   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  ...           None   \n\n                                                   last_modified position  \\\nid                                                                          \n36fb4629-8003-4acc-a1ca-3302778a530d  2023-04-09T21:22:51.471930        0   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  2023-04-09T21:20:08.806816        1   \n01297a0a-4160-41b3-8caf-90f7857ab7bd                        None        2   \nb646336a-deb4-4075-aee4-c5d28d88c426  2023-04-09T21:22:59.025202        3   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  2023-04-09T21:23:04.872534        4   \n...                                                          ...      ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed  2019-04-08T17:57:22.921679        3   \n2bd36135-1104-4988-92c6-64a43ffbdf0d  2019-04-08T18:14:11.674512        4   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  2019-07-10T17:59:13.219310        0   \n6c45d829-9b68-4093-a054-8a79784fa060  2019-07-10T17:59:45.639595        1   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  2019-07-10T17:59:54.494163        2   \n\n                                                               revision_id  \\\nid                                                                           \n36fb4629-8003-4acc-a1ca-3302778a530d  d3095d71-057e-4c7f-bf59-f20585f556d1   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  e5b8010d-4296-4b9d-9767-0f228932a710   \n01297a0a-4160-41b3-8caf-90f7857ab7bd  b63179f0-6a69-461e-889c-ac4ab5b0a279   \nb646336a-deb4-4075-aee4-c5d28d88c426  0ba2eddf-cfc7-4e22-bfbe-1a6a800230d7   \n7f7072ce-7c19-4813-a45c-6135cf4505bb  ed31b212-e4c9-479d-99fa-9650223558c7   \n...                                                                    ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed  191cc266-59ca-4596-b827-42d471741d3f   \n2bd36135-1104-4988-92c6-64a43ffbdf0d  5c59d4aa-4a48-49d0-9f13-58820c759536   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  4a98e52a-9a86-4808-8a4b-b7c78bac33e4   \n6c45d829-9b68-4093-a054-8a79784fa060  5429216b-974c-49b6-b2f6-65de877e6817   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  03ec842b-d93d-4e65-a4fe-39995a84d0f8   \n\n                                        url_type resource_type         size  \\\nid                                                                            \n36fb4629-8003-4acc-a1ca-3302778a530d  datapusher          None          NaN   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b  datapusher          None          NaN   \n01297a0a-4160-41b3-8caf-90f7857ab7bd                      None          NaN   \nb646336a-deb4-4075-aee4-c5d28d88c426  datapusher          None       1421.0   \n7f7072ce-7c19-4813-a45c-6135cf4505bb      upload          None       6309.0   \n...                                          ...           ...          ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed      upload          None  124634321.0   \n2bd36135-1104-4988-92c6-64a43ffbdf0d      upload          None  116408173.0   \n1ac3a889-61b5-452a-950d-c7533d9a5e23      upload          None       5171.0   \n6c45d829-9b68-4093-a054-8a79784fa060      upload          None       8855.0   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f      upload          None       2593.0   \n\n                                                       package_name  \\\nid                                                                    \n36fb4629-8003-4acc-a1ca-3302778a530d   allegheny-county-air-quality   \n4ab1e23f-3262-4bd3-adbf-f72f0119108b   allegheny-county-air-quality   \n01297a0a-4160-41b3-8caf-90f7857ab7bd   allegheny-county-air-quality   \nb646336a-deb4-4075-aee4-c5d28d88c426   allegheny-county-air-quality   \n7f7072ce-7c19-4813-a45c-6135cf4505bb   allegheny-county-air-quality   \n...                                                             ...   \n99217cce-24e0-4136-a619-f28da1b0b0ed              parking-purchases   \n2bd36135-1104-4988-92c6-64a43ffbdf0d              parking-purchases   \n1ac3a889-61b5-452a-950d-c7533d9a5e23  green-infrastructure-projects   \n6c45d829-9b68-4093-a054-8a79784fa060  green-infrastructure-projects   \n2eb6f0db-8c61-4854-be28-7661c43f2b9f  green-infrastructure-projects   \n\n                                      pg_size_pretty pg_relation_size  \nid                                                                     \n36fb4629-8003-4acc-a1ca-3302778a530d         2372 MB     2.487026e+09  \n4ab1e23f-3262-4bd3-adbf-f72f0119108b           14 MB     1.449165e+07  \n01297a0a-4160-41b3-8caf-90f7857ab7bd             NaN     0.000000e+00  \nb646336a-deb4-4075-aee4-c5d28d88c426           32 kB     3.276800e+04  \n7f7072ce-7c19-4813-a45c-6135cf4505bb           24 kB     2.457600e+04  \n...                                              ...              ...  \n99217cce-24e0-4136-a619-f28da1b0b0ed             NaN     0.000000e+00  \n2bd36135-1104-4988-92c6-64a43ffbdf0d             NaN     0.000000e+00  \n1ac3a889-61b5-452a-950d-c7533d9a5e23             NaN     0.000000e+00  \n6c45d829-9b68-4093-a054-8a79784fa060             NaN     0.000000e+00  \n2eb6f0db-8c61-4854-be28-7661c43f2b9f           16 kB     1.638400e+04  \n\n[2217 rows x 22 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mimetype</th>\n      <th>cache_url</th>\n      <th>hash</th>\n      <th>description</th>\n      <th>name</th>\n      <th>format</th>\n      <th>url</th>\n      <th>datastore_active</th>\n      <th>cache_last_updated</th>\n      <th>package_id</th>\n      <th>...</th>\n      <th>mimetype_inner</th>\n      <th>last_modified</th>\n      <th>position</th>\n      <th>revision_id</th>\n      <th>url_type</th>\n      <th>resource_type</th>\n      <th>size</th>\n      <th>package_name</th>\n      <th>pg_size_pretty</th>\n      <th>pg_relation_size</th>\n    </tr>\n    <tr>\n      <th>id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>36fb4629-8003-4acc-a1ca-3302778a530d</th>\n      <td>text/csv</td>\n      <td>None</td>\n      <td></td>\n      <td>Since this is a large table, if you wish to do...</td>\n      <td>Hourly Air Quality Data</td>\n      <td>CSV</td>\n      <td>https://tools.wprdc.org/downstream/36fb4629-80...</td>\n      <td>True</td>\n      <td>None</td>\n      <td>c7b3266c-adc6-41c0-b19a-8d4353bfcdaf</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2023-04-09T21:22:51.471930</td>\n      <td>0</td>\n      <td>d3095d71-057e-4c7f-bf59-f20585f556d1</td>\n      <td>datapusher</td>\n      <td>None</td>\n      <td>NaN</td>\n      <td>allegheny-county-air-quality</td>\n      <td>2372 MB</td>\n      <td>2.487026e+09</td>\n    </tr>\n    <tr>\n      <th>4ab1e23f-3262-4bd3-adbf-f72f0119108b</th>\n      <td>text/csv</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Daily AQI Data</td>\n      <td>CSV</td>\n      <td>https://data.wprdc.org/datastore/dump/4ab1e23f...</td>\n      <td>True</td>\n      <td>None</td>\n      <td>c7b3266c-adc6-41c0-b19a-8d4353bfcdaf</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2023-04-09T21:20:08.806816</td>\n      <td>1</td>\n      <td>e5b8010d-4296-4b9d-9767-0f228932a710</td>\n      <td>datapusher</td>\n      <td>None</td>\n      <td>NaN</td>\n      <td>allegheny-county-air-quality</td>\n      <td>14 MB</td>\n      <td>1.449165e+07</td>\n    </tr>\n    <tr>\n      <th>01297a0a-4160-41b3-8caf-90f7857ab7bd</th>\n      <td>text/html</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Air Quality List of Acronyms and Flag Definitions</td>\n      <td>HTML</td>\n      <td>https://tools.wprdc.org/pages/air-quality-docs...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>c7b3266c-adc6-41c0-b19a-8d4353bfcdaf</td>\n      <td>...</td>\n      <td>None</td>\n      <td>None</td>\n      <td>2</td>\n      <td>b63179f0-6a69-461e-889c-ac4ab5b0a279</td>\n      <td></td>\n      <td>None</td>\n      <td>NaN</td>\n      <td>allegheny-county-air-quality</td>\n      <td>NaN</td>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>b646336a-deb4-4075-aee4-c5d28d88c426</th>\n      <td>text/csv</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Sensor Locations</td>\n      <td>CSV</td>\n      <td>https://data.wprdc.org/datastore/dump/b646336a...</td>\n      <td>True</td>\n      <td>None</td>\n      <td>c7b3266c-adc6-41c0-b19a-8d4353bfcdaf</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2023-04-09T21:22:59.025202</td>\n      <td>3</td>\n      <td>0ba2eddf-cfc7-4e22-bfbe-1a6a800230d7</td>\n      <td>datapusher</td>\n      <td>None</td>\n      <td>1421.0</td>\n      <td>allegheny-county-air-quality</td>\n      <td>32 kB</td>\n      <td>3.276800e+04</td>\n    </tr>\n    <tr>\n      <th>7f7072ce-7c19-4813-a45c-6135cf4505bb</th>\n      <td>application/json</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Sensor Locations (GeoJSON)</td>\n      <td>GeoJSON</td>\n      <td>https://data.wprdc.org/dataset/c7b3266c-adc6-4...</td>\n      <td>True</td>\n      <td>None</td>\n      <td>c7b3266c-adc6-41c0-b19a-8d4353bfcdaf</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2023-04-09T21:23:04.872534</td>\n      <td>4</td>\n      <td>ed31b212-e4c9-479d-99fa-9650223558c7</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>6309.0</td>\n      <td>allegheny-county-air-quality</td>\n      <td>24 kB</td>\n      <td>2.457600e+04</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>99217cce-24e0-4136-a619-f28da1b0b0ed</th>\n      <td>application/zip</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>2017 Parking Purchases (compressed)</td>\n      <td>ZIP</td>\n      <td>https://data.wprdc.org/dataset/e87ccf87-90f6-4...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>e87ccf87-90f6-4023-ba4e-5a2f58793a72</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2019-04-08T17:57:22.921679</td>\n      <td>3</td>\n      <td>191cc266-59ca-4596-b827-42d471741d3f</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>124634321.0</td>\n      <td>parking-purchases</td>\n      <td>NaN</td>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>2bd36135-1104-4988-92c6-64a43ffbdf0d</th>\n      <td>application/zip</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>2018 Parking Purchases (compressed)</td>\n      <td>ZIP</td>\n      <td>https://data.wprdc.org/dataset/e87ccf87-90f6-4...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>e87ccf87-90f6-4023-ba4e-5a2f58793a72</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2019-04-08T18:14:11.674512</td>\n      <td>4</td>\n      <td>5c59d4aa-4a48-49d0-9f13-58820c759536</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>116408173.0</td>\n      <td>parking-purchases</td>\n      <td>NaN</td>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>1ac3a889-61b5-452a-950d-c7533d9a5e23</th>\n      <td>application/json</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Green Infrastructure Projects</td>\n      <td>GeoJSON</td>\n      <td>https://data.wprdc.org/dataset/30bbb37c-ca2d-4...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>30bbb37c-ca2d-47e6-bae4-72c57272383c</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2019-07-10T17:59:13.219310</td>\n      <td>0</td>\n      <td>4a98e52a-9a86-4808-8a4b-b7c78bac33e4</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>5171.0</td>\n      <td>green-infrastructure-projects</td>\n      <td>NaN</td>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>6c45d829-9b68-4093-a054-8a79784fa060</th>\n      <td>application/zip</td>\n      <td>None</td>\n      <td></td>\n      <td>zippped shapefile</td>\n      <td>Green Infrastructure Projects</td>\n      <td>SHP</td>\n      <td>https://data.wprdc.org/dataset/30bbb37c-ca2d-4...</td>\n      <td>False</td>\n      <td>None</td>\n      <td>30bbb37c-ca2d-47e6-bae4-72c57272383c</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2019-07-10T17:59:45.639595</td>\n      <td>1</td>\n      <td>5429216b-974c-49b6-b2f6-65de877e6817</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>8855.0</td>\n      <td>green-infrastructure-projects</td>\n      <td>NaN</td>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>2eb6f0db-8c61-4854-be28-7661c43f2b9f</th>\n      <td>text/csv</td>\n      <td>None</td>\n      <td></td>\n      <td></td>\n      <td>Green Infrastructure Projects</td>\n      <td>CSV</td>\n      <td>https://data.wprdc.org/dataset/30bbb37c-ca2d-4...</td>\n      <td>True</td>\n      <td>None</td>\n      <td>30bbb37c-ca2d-47e6-bae4-72c57272383c</td>\n      <td>...</td>\n      <td>None</td>\n      <td>2019-07-10T17:59:54.494163</td>\n      <td>2</td>\n      <td>03ec842b-d93d-4e65-a4fe-39995a84d0f8</td>\n      <td>upload</td>\n      <td>None</td>\n      <td>2593.0</td>\n      <td>green-infrastructure-projects</td>\n      <td>16 kB</td>\n      <td>1.638400e+04</td>\n    </tr>\n  </tbody>\n</table>\n<p>2217 rows × 22 columns</p>\n</div>"
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# flatten packages to list of just their resources.\n",
    "all_resources = [resource for sublist in [package['resources'] for package in packages] for resource in sublist]\n",
    "\n",
    "resources_df = pd.DataFrame(all_resources)\n",
    "resources_df = resources_df.set_index('id')\n",
    "# enrich with some package metadata\n",
    "package_lookup = {p['id']: p for p in packages}\n",
    "\n",
    "resources_df['package_name']  = resources_df['package_id'].map(lambda pid: package_lookup[pid]['name'])\n",
    "# get sizes from database export\n",
    "size_df = pd.read_csv('resource_sizes.csv')\n",
    "resources_df = resources_df.join(size_df.set_index('id'))\n",
    "\n",
    "resources_df.to_csv('wprdc_resources.csv')\n",
    "resources_df['pg_relation_size'] = resources_df['pg_relation_size'].fillna(0)\n",
    "resources_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "data": {
      "text/plain": "<AxesSubplot: ylabel='Frequency'>"
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGvCAYAAABB3D9ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAosUlEQVR4nO3dfXBUVYL+8aeTkBAgaUyAhCwJRAQBERBQyACuQDQiS4FkXF+wCEjpOBMYIMNYpEZgXNHwsryIvDlTGKBmIppZYIZxgcWoYV0DQhBFZyeAIgkm3eBoukmm0mSS/v1h2b9teTFpOtw+yfdTdavsc2+ffqAp89TJud02r9frFQAAgIHCrA4AAAAQKIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYEVYHaGmNjY2qrKxUTEyMbDab1XEAAEATeL1eXbx4UUlJSQoLu/q6S6svMpWVlUpOTrY6BgAACEBFRYV69Ohx1fOtvsjExMRI+vYvIjY21uI0AACgKdxut5KTk30/x6+m1ReZ736dFBsbS5EBAMAwP7QthM2+AADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaKsDqAyXotfPOysS+WTbQgCQAAbRMrMgAAwFgUGQAAYCyKDAAAMBZFBgAAGMvSItPQ0KBFixYpNTVV0dHR6t27t55//nl5vV7fNV6vV4sXL1b37t0VHR2t9PR0nTp1ysLUAAAgVFhaZJYvX65NmzZp/fr1+t///V8tX75cK1as0Msvv+y7ZsWKFVq3bp02b96sw4cPq2PHjsrIyFBdXZ2FyQEAQCiw9Pbr999/X5MnT9bEid/estyrVy+99tpr+uCDDyR9uxqzdu1aPfvss5o8ebIkafv27UpISNDu3bv1yCOPWJYdAABYz9IVmR/96EcqKirSyZMnJUkfffSR3nvvPU2YMEGSdObMGTkcDqWnp/ueY7fbNWLECJWUlFxxTo/HI7fb7XcAAIDWydIVmYULF8rtdqtfv34KDw9XQ0ODXnjhBU2bNk2S5HA4JEkJCQl+z0tISPCd+768vDw999xzLRscAACEBEtXZN544w39/ve/V0FBgY4dO6Zt27bp3//937Vt27aA58zNzZXL5fIdFRUVQUwMAABCiaUrMr/85S+1cOFC316X22+/XWfPnlVeXp6ysrKUmJgoSXI6nerevbvveU6nU0OGDLninFFRUYqKimrx7AAAwHqWrsj8/e9/V1iYf4Tw8HA1NjZKklJTU5WYmKiioiLfebfbrcOHDystLe2GZgUAAKHH0hWZSZMm6YUXXlBKSopuu+02ffjhh1q9erWeeOIJSZLNZtO8efO0dOlS9enTR6mpqVq0aJGSkpI0ZcoUK6MDAIAQYGmRefnll7Vo0SL97Gc/0/nz55WUlKSf/OQnWrx4se+aZ555RrW1tXrqqadUXV2t0aNHa9++fWrfvr2FyQEAQCiwef/vx+i2Qm63W3a7XS6XS7GxsUGdu9fCNy8b+2LZxKC+BgAAbVFTf37zXUsAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEsLTK9evWSzWa77MjOzpYk1dXVKTs7W/Hx8erUqZMyMzPldDqtjAwAAEKIpUXmyJEjqqqq8h0HDhyQJD300EOSpPnz52vPnj0qLCxUcXGxKisrNXXqVCsjAwCAEBJh5Yt37drV7/GyZcvUu3dv/fM//7NcLpe2bNmigoICjRs3TpKUn5+v/v3769ChQxo5cqQVkQEAQAgJmT0yly5d0u9+9zs98cQTstlsKi0tVX19vdLT033X9OvXTykpKSopKbnqPB6PR2632+8AAACtU8gUmd27d6u6ulozZsyQJDkcDkVGRqpz585+1yUkJMjhcFx1nry8PNntdt+RnJzcgqkBAICVQqbIbNmyRRMmTFBSUtJ1zZObmyuXy+U7KioqgpQQAACEGkv3yHzn7Nmzeuutt7Rz507fWGJioi5duqTq6mq/VRmn06nExMSrzhUVFaWoqKiWjAsAAEJESKzI5Ofnq1u3bpo4caJvbNiwYWrXrp2Kiop8Y2VlZSovL1daWpoVMQEAQIixfEWmsbFR+fn5ysrKUkTE/49jt9s1a9Ys5eTkKC4uTrGxsZozZ47S0tK4YwkAAEgKgSLz1ltvqby8XE888cRl59asWaOwsDBlZmbK4/EoIyNDGzdutCAlAAAIRTav1+u1OkRLcrvdstvtcrlcio2NDercvRa+ednYF8smXuFKAADQHE39+R0Se2QAAAACQZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABjL8iLz5Zdf6vHHH1d8fLyio6N1++236+jRo77zXq9XixcvVvfu3RUdHa309HSdOnXKwsQAACBUWFpkvvnmG40aNUrt2rXT3r179Ze//EWrVq3STTfd5LtmxYoVWrdunTZv3qzDhw+rY8eOysjIUF1dnYXJAQBAKIiw8sWXL1+u5ORk5efn+8ZSU1N9/+31erV27Vo9++yzmjx5siRp+/btSkhI0O7du/XII4/c8MwAACB0WLoi86c//UnDhw/XQw89pG7duumOO+7Qb3/7W9/5M2fOyOFwKD093Tdmt9s1YsQIlZSUXHFOj8cjt9vtdwAAgNbJ0iLz+eefa9OmTerTp4/279+vn/70p/r5z3+ubdu2SZIcDockKSEhwe95CQkJvnPfl5eXJ7vd7juSk5Nb9g8BAAAsY2mRaWxs1NChQ/Xiiy/qjjvu0FNPPaUnn3xSmzdvDnjO3NxcuVwu31FRURHExAAAIJRYWmS6d++uAQMG+I31799f5eXlkqTExERJktPp9LvG6XT6zn1fVFSUYmNj/Q4AANA6WVpkRo0apbKyMr+xkydPqmfPnpK+3fibmJiooqIi33m3263Dhw8rLS3thmYFAAChx9K7lubPn68f/ehHevHFF/Wv//qv+uCDD/Sb3/xGv/nNbyRJNptN8+bN09KlS9WnTx+lpqZq0aJFSkpK0pQpU6yMDgAAQoClRebOO+/Url27lJubq3/7t39Tamqq1q5dq2nTpvmueeaZZ1RbW6unnnpK1dXVGj16tPbt26f27dtbmBwAAIQCm9fr9VodoiW53W7Z7Xa5XK6g75fptfDNy8a+WDYxqK8BAEBb1NSf35Z/RQEAAECgKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaytMj8+te/ls1m8zv69evnO19XV6fs7GzFx8erU6dOyszMlNPptDAxAAAIJZavyNx2222qqqryHe+9957v3Pz587Vnzx4VFhaquLhYlZWVmjp1qoVpAQBAKIkI5Emff/65br755uAEiIhQYmLiZeMul0tbtmxRQUGBxo0bJ0nKz89X//79dejQIY0cOTIorw8AAMwV0IrMLbfcorFjx+p3v/ud6urqrivAqVOnlJSUpJtvvlnTpk1TeXm5JKm0tFT19fVKT0/3XduvXz+lpKSopKTkqvN5PB653W6/AwAAtE4BFZljx45p0KBBysnJUWJion7yk5/ogw8+aPY8I0aM0NatW7Vv3z5t2rRJZ86c0ZgxY3Tx4kU5HA5FRkaqc+fOfs9JSEiQw+G46px5eXmy2+2+Izk5udm5AACAGQIqMkOGDNFLL72kyspKvfrqq6qqqtLo0aM1cOBArV69WhcuXGjSPBMmTNBDDz2kQYMGKSMjQ//5n/+p6upqvfHGG4HEkiTl5ubK5XL5joqKioDnAgAAoe26NvtGRERo6tSpKiws1PLly3X69GktWLBAycnJmj59uqqqqpo1X+fOndW3b1+dPn1aiYmJunTpkqqrq/2ucTqdV9xT852oqCjFxsb6HQAAoHW6riJz9OhR/exnP1P37t21evVqLViwQJ999pkOHDigyspKTZ48uVnz1dTU6LPPPlP37t01bNgwtWvXTkVFRb7zZWVlKi8vV1pa2vXEBgAArURAdy2tXr1a+fn5Kisr0wMPPKDt27frgQceUFjYt70oNTVVW7duVa9eva45z4IFCzRp0iT17NlTlZWVWrJkicLDw/Xoo4/Kbrdr1qxZysnJUVxcnGJjYzVnzhylpaVxxxIAAJAUYJHZtGmTnnjiCc2YMUPdu3e/4jXdunXTli1brjnPuXPn9Oijj+pvf/ubunbtqtGjR+vQoUPq2rWrJGnNmjUKCwtTZmamPB6PMjIytHHjxkAiAwCAVsjm9Xq9VodoSW63W3a7XS6XK+j7ZXotfPOysS+WTQzqawAA0BY19ed3QHtk8vPzVVhYeNl4YWGhtm3bFsiUAAAAzRZQkcnLy1OXLl0uG+/WrZtefPHF6w4FAADQFAEVmfLycqWmpl423rNnT98n8wIAALS0gIpMt27d9PHHH182/tFHHyk+Pv66QwEAADRFQEXm0Ucf1c9//nO98847amhoUENDg95++23NnTtXjzzySLAzAgAAXFFAt18///zz+uKLLzR+/HhFRHw7RWNjo6ZPn84eGQAAcMMEVGQiIyP1+uuv6/nnn9dHH32k6Oho3X777erZs2ew8wEAAFxVQEXmO3379lXfvn2DlQUAAKBZAioyDQ0N2rp1q4qKinT+/Hk1Njb6nX/77beDEg4AAOBaAioyc+fO1datWzVx4kQNHDhQNpst2LkAAAB+UEBFZseOHXrjjTf0wAMPBDsPAABAkwV0+3VkZKRuueWWYGcBAABoloCKzC9+8Qu99NJLauXfNwkAAEJcQL9aeu+99/TOO+9o7969uu2229SuXTu/8zt37gxKOAAAgGsJqMh07txZDz74YLCzAAAANEtARSY/Pz/YOQAAAJotoD0ykvSPf/xDb731ll555RVdvHhRklRZWamampqghQMAALiWgFZkzp49q/vvv1/l5eXyeDy69957FRMTo+XLl8vj8Wjz5s3BzgkAAHCZgFZk5s6dq+HDh+ubb75RdHS0b/zBBx9UUVFR0MIBAABcS0ArMv/93/+t999/X5GRkX7jvXr10pdffhmUYAAAAD8koBWZxsZGNTQ0XDZ+7tw5xcTEXHcoAACApgioyNx3331au3at77HNZlNNTY2WLFnC1xYAAIAbJqBfLa1atUoZGRkaMGCA6urq9Nhjj+nUqVPq0qWLXnvttWBnBAAAuKKAikyPHj300UcfaceOHfr4449VU1OjWbNmadq0aX6bfwEAAFpSQEVGkiIiIvT4448HMwsAAECzBFRktm/ffs3z06dPDygMAABAcwRUZObOnev3uL6+Xn//+98VGRmpDh06UGQAAMANEdBdS998843fUVNTo7KyMo0ePZrNvgAA4IYJ+LuWvq9Pnz5atmzZZas1AAAALSVoRUb6dgNwZWVlMKcEAAC4qoD2yPzpT3/ye+z1elVVVaX169dr1KhRQQkGAADwQwIqMlOmTPF7bLPZ1LVrV40bN06rVq0KRi4AAIAfFPB3Lf3fo6GhQQ6HQwUFBerevXtAQZYtWyabzaZ58+b5xurq6pSdna34+Hh16tRJmZmZcjqdAc0PAABan6DukQnUkSNH9Morr2jQoEF+4/Pnz9eePXtUWFio4uJiVVZWaurUqRalBAAAoSagXy3l5OQ0+drVq1df83xNTY2mTZum3/72t1q6dKlv3OVyacuWLSooKNC4ceMkSfn5+erfv78OHTqkkSNHBhIdAAC0IgEVmQ8//FAffvih6uvrdeutt0qSTp48qfDwcA0dOtR3nc1m+8G5srOzNXHiRKWnp/sVmdLSUtXX1ys9Pd031q9fP6WkpKikpOSqRcbj8cjj8fgeu93uZv/5AACAGQIqMpMmTVJMTIy2bdumm266SdK3H5I3c+ZMjRkzRr/4xS+aNM+OHTt07NgxHTly5LJzDodDkZGR6ty5s994QkKCHA7HVefMy8vTc8891/Q/DAAAMFZAe2RWrVqlvLw8X4mRpJtuuklLly5t8l1LFRUVmjt3rn7/+9+rffv2gcS4otzcXLlcLt9RUVERtLkBAEBoCajIuN1uXbhw4bLxCxcu6OLFi02ao7S0VOfPn9fQoUMVERGhiIgIFRcXa926dYqIiFBCQoIuXbqk6upqv+c5nU4lJiZedd6oqCjFxsb6HQAAoHUKqMg8+OCDmjlzpnbu3Klz587p3Llz+o//+A/NmjWryXcVjR8/XidOnNDx48d9x/DhwzVt2jTff7dr105FRUW+55SVlam8vFxpaWmBxAYAAK1MQHtkNm/erAULFuixxx5TfX39txNFRGjWrFlauXJlk+aIiYnRwIED/cY6duyo+Ph43/isWbOUk5OjuLg4xcbGas6cOUpLS+OOJQAAICnAItOhQwdt3LhRK1eu1GeffSZJ6t27tzp27BjUcGvWrFFYWJgyMzPl8XiUkZGhjRs3BvU1AACAuQIqMt+pqqpSVVWV7r77bkVHR8vr9Tbpluureffdd/0et2/fXhs2bNCGDRuuJyYAAGilAtoj87e//U3jx49X37599cADD6iqqkrSt78Kauqt1wAAANcroCIzf/58tWvXTuXl5erQoYNv/OGHH9a+ffuCFg4AAOBaAvrV0n/9139p//796tGjh994nz59dPbs2aAEAwAA+CEBrcjU1tb6rcR85+uvv1ZUVNR1hwIAAGiKgIrMmDFjtH37dt9jm82mxsZGrVixQmPHjg1aOAAAgGsJ6FdLK1as0Pjx43X06FFdunRJzzzzjD799FN9/fXX+p//+Z9gZwQAALiigFZkBg4cqJMnT2r06NGaPHmyamtrNXXqVH344Yfq3bt3sDMCAABcUbNXZOrr63X//fdr8+bN+tWvftUSmQAAAJqk2Ssy7dq108cff9wSWQAAAJoloF8tPf7449qyZUuwswAAADRLQJt9//GPf+jVV1/VW2+9pWHDhl32HUurV68OSjgAAIBraVaR+fzzz9WrVy998sknGjp0qCTp5MmTftdcz3ctAQAANEezikyfPn1UVVWld955R9K3X0mwbt06JSQktEg4AACAa2nWHhmv1+v3eO/evaqtrQ1qIAAAgKYKaLPvd75fbAAAAG6kZhUZm8122R4Y9sQAAACrNGuPjNfr1YwZM3xfDFlXV6enn376sruWdu7cGbyEAAAAV9GsIpOVleX3+PHHHw9qGAAAgOZoVpHJz89vqRwAAADNdl2bfQEAAKxEkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWJYWmU2bNmnQoEGKjY1VbGys0tLStHfvXt/5uro6ZWdnKz4+Xp06dVJmZqacTqeFiQEAQCixtMj06NFDy5YtU2lpqY4ePapx48Zp8uTJ+vTTTyVJ8+fP1549e1RYWKji4mJVVlZq6tSpVkYGAAAhxOb1er1Wh/i/4uLitHLlSv34xz9W165dVVBQoB//+MeSpL/+9a/q37+/SkpKNHLkyCbN53a7Zbfb5XK5FBsbG9SsvRa+ednYF8smBvU1AABoi5r68ztk9sg0NDRox44dqq2tVVpamkpLS1VfX6/09HTfNf369VNKSopKSkquOo/H45Hb7fY7AABA62R5kTlx4oQ6deqkqKgoPf3009q1a5cGDBggh8OhyMhIde7c2e/6hIQEORyOq86Xl5cnu93uO5KTk1v4TwAAAKxieZG59dZbdfz4cR0+fFg//elPlZWVpb/85S8Bz5ebmyuXy+U7KioqgpgWAACEkgirA0RGRuqWW26RJA0bNkxHjhzRSy+9pIcffliXLl1SdXW136qM0+lUYmLiVeeLiopSVFRUS8cGAAAhwPIVme9rbGyUx+PRsGHD1K5dOxUVFfnOlZWVqby8XGlpaRYmBAAAocLSFZnc3FxNmDBBKSkpunjxogoKCvTuu+9q//79stvtmjVrlnJychQXF6fY2FjNmTNHaWlpTb5jCQAAtG6WFpnz589r+vTpqqqqkt1u16BBg7R//37de++9kqQ1a9YoLCxMmZmZ8ng8ysjI0MaNG62MDAAAQkjIfY5MsPE5MgAAmMe4z5EBAABoLooMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYlhaZvLw83XnnnYqJiVG3bt00ZcoUlZWV+V1TV1en7OxsxcfHq1OnTsrMzJTT6bQoMQAACCWWFpni4mJlZ2fr0KFDOnDggOrr63XfffeptrbWd838+fO1Z88eFRYWqri4WJWVlZo6daqFqQEAQKiIsPLF9+3b5/d469at6tatm0pLS3X33XfL5XJpy5YtKigo0Lhx4yRJ+fn56t+/vw4dOqSRI0daERsAAISIkNoj43K5JElxcXGSpNLSUtXX1ys9Pd13Tb9+/ZSSkqKSkpIrzuHxeOR2u/0OAADQOoVMkWlsbNS8efM0atQoDRw4UJLkcDgUGRmpzp07+12bkJAgh8NxxXny8vJkt9t9R3JycktHBwAAFgmZIpOdna1PPvlEO3bsuK55cnNz5XK5fEdFRUWQEgIAgFBj6R6Z78yePVt//vOfdfDgQfXo0cM3npiYqEuXLqm6utpvVcbpdCoxMfGKc0VFRSkqKqqlIwMAgBBg6YqM1+vV7NmztWvXLr399ttKTU31Oz9s2DC1a9dORUVFvrGysjKVl5crLS3tRscFAAAhxtIVmezsbBUUFOiPf/yjYmJifPte7Ha7oqOjZbfbNWvWLOXk5CguLk6xsbGaM2eO0tLSuGMJAABYW2Q2bdokSbrnnnv8xvPz8zVjxgxJ0po1axQWFqbMzEx5PB5lZGRo48aNNzgpAAAIRZYWGa/X+4PXtG/fXhs2bNCGDRtuQCIAAGCSkLlrCQAAoLkoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjBUSX1HQmvRa+Kbf4y+WTbQoCQAArR8rMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFiWFpmDBw9q0qRJSkpKks1m0+7du/3Oe71eLV68WN27d1d0dLTS09N16tQpa8ICAICQY2mRqa2t1eDBg7Vhw4Yrnl+xYoXWrVunzZs36/Dhw+rYsaMyMjJUV1d3g5MCAIBQFGHli0+YMEETJky44jmv16u1a9fq2Wef1eTJkyVJ27dvV0JCgnbv3q1HHnnkRkYFAAAhKGT3yJw5c0YOh0Pp6em+MbvdrhEjRqikpOSqz/N4PHK73X4HAABonSxdkbkWh8MhSUpISPAbT0hI8J27kry8PD333HMtmq05ei1887KxL5ZNtCAJAACtT8iuyAQqNzdXLpfLd1RUVFgdCQAAtJCQLTKJiYmSJKfT6TfudDp9564kKipKsbGxfgcAAGidQrbIpKamKjExUUVFRb4xt9utw4cPKy0tzcJkAAAgVFi6R6ampkanT5/2PT5z5oyOHz+uuLg4paSkaN68eVq6dKn69Omj1NRULVq0SElJSZoyZYp1oQEAQMiwtMgcPXpUY8eO9T3OycmRJGVlZWnr1q165plnVFtbq6eeekrV1dUaPXq09u3bp/bt21sVGQAAhBCb1+v1Wh2iJbndbtntdrlcrqDvl7nSHUlNwV1LAABcW1N/fofsHhkAAIAfQpEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAADAWRQYAABiLIgMAAIxFkQEAAMaiyAAAAGNRZAAAgLEoMgAAwFgUGQAAYCyKDAAAMBZFBgAAGIsiAwAAjEWRAQAAxqLIAAAAY0VYHaAt6rXwTb/HXyybaFESAADMxooMAAAwFkUGAAAYiyIDAACMxR6ZEPD9PTNX05S9NE2Ziz05NwZ7oQCg5bEiAwAAjEWRAQAAxqLIAAAAY1FkAACAsdjsiysKZKPqlTYaB/I8NsVeW6B/zy01DwBYiRUZAABgLIoMAAAwFkUGAAAYy4g9Mhs2bNDKlSvlcDg0ePBgvfzyy7rrrrusjnXDNfWD827Ua1u5/yVYfxd8yGDztdY9TTfy32pr+TtD2xRq/w8I+RWZ119/XTk5OVqyZImOHTumwYMHKyMjQ+fPn7c6GgAAsFjIF5nVq1frySef1MyZMzVgwABt3rxZHTp00Kuvvmp1NAAAYLGQ/tXSpUuXVFpaqtzcXN9YWFiY0tPTVVJScsXneDweeTwe32OXyyVJcrvdQc/X6Pl70Oe8EZryd9GUP9v35wnkOVd6XrDyNUVL/V1c6XnB+jd4pTyBzB3oPC3157JaqL9fQKi4Uf8P+G5er9d77Qu9IezLL7/0SvK+//77fuO//OUvvXfdddcVn7NkyRKvJA4ODg4ODo5WcFRUVFyzK4T0ikwgcnNzlZOT43vc2Nior7/+WvHx8bLZbEF5DbfbreTkZFVUVCg2NjYocyJwvB+hg/cidPBehA7ei8B4vV5dvHhRSUlJ17wupItMly5dFB4eLqfT6TfudDqVmJh4xedERUUpKirKb6xz584tki82NpZ/lCGE9yN08F6EDt6L0MF70Xx2u/0Hrwnpzb6RkZEaNmyYioqKfGONjY0qKipSWlqahckAAEAoCOkVGUnKyclRVlaWhg8frrvuuktr165VbW2tZs6caXU0AABgsZAvMg8//LAuXLigxYsXy+FwaMiQIdq3b58SEhIsyxQVFaUlS5Zc9issWIP3I3TwXoQO3ovQwXvRsmxe7w/d1wQAABCaQnqPDAAAwLVQZAAAgLEoMgAAwFgUGQAAYCyKTAA2bNigXr16qX379hoxYoQ++OADqyO1SQcPHtSkSZOUlJQkm82m3bt3Wx2pTcrLy9Odd96pmJgYdevWTVOmTFFZWZnVsdqsTZs2adCgQb4PX0tLS9PevXutjtXmLVu2TDabTfPmzbM6SqtDkWmm119/XTk5OVqyZImOHTumwYMHKyMjQ+fPn7c6WptTW1urwYMHa8OGDVZHadOKi4uVnZ2tQ4cO6cCBA6qvr9d9992n2tpaq6O1ST169NCyZctUWlqqo0ePaty4cZo8ebI+/fRTq6O1WUeOHNErr7yiQYMGWR2lVeL262YaMWKE7rzzTq1fv17St580nJycrDlz5mjhwoUWp2u7bDabdu3apSlTplgdpc27cOGCunXrpuLiYt19991Wx4GkuLg4rVy5UrNmzbI6SptTU1OjoUOHauPGjVq6dKmGDBmitWvXWh2rVWFFphkuXbqk0tJSpaen+8bCwsKUnp6ukpISC5MBocPlckn69ocnrNXQ0KAdO3aotraWr3WxSHZ2tiZOnOj3cwPBFfKf7BtKvvrqKzU0NFz2qcIJCQn661//alEqIHQ0NjZq3rx5GjVqlAYOHGh1nDbrxIkTSktLU11dnTp16qRdu3ZpwIABVsdqc3bs2KFjx47pyJEjVkdp1SgyAIImOztbn3zyid577z2ro7Rpt956q44fPy6Xy6U//OEPysrKUnFxMWXmBqqoqNDcuXN14MABtW/f3uo4rRpFphm6dOmi8PBwOZ1Ov3Gn06nExESLUgGhYfbs2frzn/+sgwcPqkePHlbHadMiIyN1yy23SJKGDRumI0eO6KWXXtIrr7xicbK2o7S0VOfPn9fQoUN9Yw0NDTp48KDWr18vj8ej8PBwCxO2HuyRaYbIyEgNGzZMRUVFvrHGxkYVFRXx+2e0WV6vV7Nnz9auXbv09ttvKzU11epI+J7GxkZ5PB6rY7Qp48eP14kTJ3T8+HHfMXz4cE2bNk3Hjx+nxAQRKzLNlJOTo6ysLA0fPlx33XWX1q5dq9raWs2cOdPqaG1OTU2NTp8+7Xt85swZHT9+XHFxcUpJSbEwWduSnZ2tgoIC/fGPf1RMTIwcDockyW63Kzo62uJ0bU9ubq4mTJiglJQUXbx4UQUFBXr33Xe1f/9+q6O1KTExMZftE+vYsaPi4+PZPxZkFJlmevjhh3XhwgUtXrxYDodDQ4YM0b59+y7bAIyWd/ToUY0dO9b3OCcnR5KUlZWlrVu3WpSq7dm0aZMk6Z577vEbz8/P14wZM258oDbu/Pnzmj59uqqqqmS32zVo0CDt379f9957r9XRgBbB58gAAABjsUcGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBYFBkAAGAsigwAAGi2gwcPatKkSUpKSpLNZtPu3bubPccbb7yhIUOGqEOHDurZs6dWrlzZ7DkoMgAAoNlqa2s1ePBgbdiwIaDn7927V9OmTdPTTz+tTz75RBs3btSaNWu0fv36Zs3DJ/sCAIDrYrPZtGvXLk2ZMsU35vF49Ktf/UqvvfaaqqurNXDgQC1fvtz3dSaPPfaY6uvrVVhY6HvOyy+/rBUrVqi8vFw2m61Jr82KDAAACLrZs2erpKREO3bs0Mcff6yHHnpI999/v06dOiXp26LTvn17v+dER0fr3LlzOnv2bJNfhyIDAACCqry8XPn5+SosLNSYMWPUu3dvLViwQKNHj1Z+fr4kKSMjQzt37lRRUZEaGxt18uRJrVq1SpJUVVXV5Nfi268BAEBQnThxQg0NDerbt6/fuMfjUXx8vCTpySef1GeffaZ/+Zd/UX19vWJjYzV37lz9+te/VlhY09dZKDIAACCoampqFB4ertLSUoWHh/ud69Spk6Rv99UsX75cL774ohwOh7p27aqioiJJ0s0339zk16LIAACAoLrjjjvU0NCg8+fPa8yYMde8Njw8XP/0T/8kSXrttdeUlpamrl27Nvm1KDIAAKDZampqdPr0ad/jM2fO6Pjx44qLi1Pfvn01bdo0TZ8+XatWrdIdd9yhCxcuqKioSIMGDdLEiRP11Vdf6Q9/+IPuuece1dXV+fbUFBcXNysHt18DAIBme/fddzV27NjLxrOysrR161bV19dr6dKl2r59u7788kt16dJFI0eO1HPPPafbb79dX331lSZNmqQTJ07I6/UqLS1NL7zwgkaMGNGsHBQZAABgLG6/BgAAxqLIAAAAY1FkAACAsSgyAADAWBQZAABgLIoMAAAwFkUGAAAYiyIDAACMRZEBAADGosgAAABjUWQAAICxKDIAAMBY/w/fNYZl/f55ZwAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "resources_df[resources_df['pg_relation_size'] > 10485760]['pg_relation_size'].plot.hist(bins=100)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "data": {
      "text/plain": "<AxesSubplot: ylabel='Frequency'>"
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGvCAYAAACn9fQdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAo1UlEQVR4nO3dfXTU1Z3H8c8EyARqEkHIAxCByoMiD+FBIKglCBIhyyF6jktRm0iRrS6cDUXlQNsjpbQNyAZkKxI8FlLrwSDKwx4UMAaBRWIVSCTQLgpiwkMScMWEpGVIM7/9wzJ1yAMzk0lm5vp+nfP7Y+7c32++dy5z8uHOnRmbZVmWAAAADBEW6AIAAAD8iXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADBK+0AX0NacTqfOnz+vyMhI2Wy2QJcDAAA8YFmWLl++rO7duyssrPm1me9cuDl//rwSEhICXQYAAPDBmTNn1LNnz2b7fOfCTWRkpKRvnpyoqKgAVwMAADxRXV2thIQE19/x5nznws21t6KioqIINwAAhBhPtpSwoRgAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjBLQcLN27VoNGTLE9VMISUlJ2rlzZ7PnbN68WbfffrsiIiI0ePBgvfPOO21ULQAACAUBDTc9e/bUsmXLdPjwYR06dEj33Xefpk2bpuPHjzfa/+DBg5oxY4ZmzZqloqIipaWlKS0tTceOHWvjygEAQLCyWZZlBbqIb+vSpYtWrFihWbNmNbhv+vTpqq2t1Y4dO1xtY8aMUWJionJycjy6fnV1taKjo1VVVcUPZwIAECK8+fsdNHtu6uvrlZeXp9raWiUlJTXap7CwUBMnTnRrS0lJUWFhYZPXdTgcqq6udjsAAIC52ge6gJKSEiUlJenKlSu66aabtHXrVg0cOLDRvhUVFYqNjXVri42NVUVFRZPXz8rK0pIlS/xaMwKv98K3G7R9sSw1AJUAAIJNwFduBgwYoOLiYv3pT3/SU089pYyMDP35z3/22/UXLVqkqqoq13HmzBm/XRsAAASfgK/chIeHq2/fvpKkESNG6OOPP9bq1au1bt26Bn3j4uJUWVnp1lZZWam4uLgmr2+322W32/1bNAAACFoBX7m5ntPplMPhaPS+pKQkFRQUuLXl5+c3uUcHAAB89wR05WbRokWaPHmybr31Vl2+fFkbN27U3r17tXv3bklSenq6evTooaysLElSZmamxo0bp+zsbKWmpiovL0+HDh3Syy+/HMhhAACAIBLQcHPhwgWlp6ervLxc0dHRGjJkiHbv3q37779fklRWVqawsH8uLo0dO1YbN27UL37xC/3sZz9Tv379tG3bNg0aNChQQwAAAEEm6L7nprXxPTdm4NNSAPDdEpLfcwMAAOAPhBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRAhpusrKydNdddykyMlIxMTFKS0vTiRMnmj0nNzdXNpvN7YiIiGijigEAQLALaLjZt2+f5syZow8//FD5+fmqq6vTpEmTVFtb2+x5UVFRKi8vdx2lpaVtVDEAAAh27QP54Lt27XK7nZubq5iYGB0+fFg/+MEPmjzPZrMpLi6utcsDAAAhKKj23FRVVUmSunTp0my/mpoa9erVSwkJCZo2bZqOHz/eZF+Hw6Hq6mq3AwAAmCtowo3T6dS8efN09913a9CgQU32GzBggNavX6/t27frtddek9Pp1NixY3X27NlG+2dlZSk6Otp1JCQktNYQAABAELBZlmUFughJeuqpp7Rz504dOHBAPXv29Pi8uro63XHHHZoxY4aWLl3a4H6HwyGHw+G6XV1drYSEBFVVVSkqKsovtaPt9V74doO2L5alBqASAEBbqK6uVnR0tEd/vwO65+aauXPnaseOHdq/f79XwUaSOnTooGHDhunkyZON3m+322W32/1RJgAACAEBfVvKsizNnTtXW7du1Z49e9SnTx+vr1FfX6+SkhLFx8e3QoUAACDUBHTlZs6cOdq4caO2b9+uyMhIVVRUSJKio6PVsWNHSVJ6erp69OihrKwsSdKvfvUrjRkzRn379tXXX3+tFStWqLS0VE888UTAxgEAAIJHQMPN2rVrJUnJyclu7Rs2bNDjjz8uSSorK1NY2D8XmC5duqTZs2eroqJCnTt31ogRI3Tw4EENHDiwrcoGAABBLGg2FLcVbzYkIXixoRgAvlu8+fsdNB8FBwAA8AfCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFECGm6ysrJ01113KTIyUjExMUpLS9OJEydueN7mzZt1++23KyIiQoMHD9Y777zTBtUCAIBQENBws2/fPs2ZM0cffvih8vPzVVdXp0mTJqm2trbJcw4ePKgZM2Zo1qxZKioqUlpamtLS0nTs2LE2rBwAAAQrm2VZVqCLuObixYuKiYnRvn379IMf/KDRPtOnT1dtba127NjhahszZowSExOVk5Nzw8eorq5WdHS0qqqqFBUV5bfa0bZ6L3y7QdsXy1IDUAkAoC148/c7qPbcVFVVSZK6dOnSZJ/CwkJNnDjRrS0lJUWFhYWN9nc4HKqurnY7AACAudoHuoBrnE6n5s2bp7vvvluDBg1qsl9FRYViY2Pd2mJjY1VRUdFo/6ysLC1ZssSvtTbn+hUFT1YTQmEVwpdx4cZ4XgHA/4Jm5WbOnDk6duyY8vLy/HrdRYsWqaqqynWcOXPGr9cHAADBJShWbubOnasdO3Zo//796tmzZ7N94+LiVFlZ6dZWWVmpuLi4Rvvb7XbZ7Xa/1QoAAIJbQFduLMvS3LlztXXrVu3Zs0d9+vS54TlJSUkqKChwa8vPz1dSUlJrlQkAAEJIQFdu5syZo40bN2r79u2KjIx07ZuJjo5Wx44dJUnp6enq0aOHsrKyJEmZmZkaN26csrOzlZqaqry8PB06dEgvv/xywMYBAACCR0BXbtauXauqqiolJycrPj7edWzatMnVp6ysTOXl5a7bY8eO1caNG/Xyyy9r6NChevPNN7Vt27ZmNyEDAIDvjoCu3HjyFTt79+5t0Pbwww/r4YcfboWKAABAqAuaT0sBAAD4A+EGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIziU7j5/PPP/V0HAACAX/gUbvr27avx48frtdde05UrV/xdEwAAgM98CjdHjhzRkCFDNH/+fMXFxeknP/mJPvroI3/XBgAA4DWfwk1iYqJWr16t8+fPa/369SovL9c999yjQYMGaeXKlbp48aK/6wQAAPBIizYUt2/fXg899JA2b96s5cuX6+TJk3rmmWeUkJCg9PR0lZeX+6tOAAAAj7Qo3Bw6dEj//u//rvj4eK1cuVLPPPOMTp06pfz8fJ0/f17Tpk3zV50AAAAeae/LSStXrtSGDRt04sQJTZkyRa+++qqmTJmisLBvslKfPn2Um5ur3r17+7NWAACAG/Ip3Kxdu1Y//vGP9fjjjys+Pr7RPjExMfr973/fouIAAAC85VO4+eyzz27YJzw8XBkZGb5cHgAAwGc+7bnZsGGDNm/e3KB98+bN+sMf/tDiogAAAHzlU7jJyspS165dG7THxMTot7/9bYuLAgAA8JVP4aasrEx9+vRp0N6rVy+VlZW1uCgAAABf+RRuYmJidPTo0Qbtn3zyiW655ZYWFwUAAOArn8LNjBkz9B//8R96//33VV9fr/r6eu3Zs0eZmZn64Q9/6O8aAQAAPObTp6WWLl2qL774QhMmTFD79t9cwul0Kj09nT03AAAgoHwKN+Hh4dq0aZOWLl2qTz75RB07dtTgwYPVq1cvf9cHAADgFZ/CzTX9+/dX//79/VULAABAi/kUburr65Wbm6uCggJduHBBTqfT7f49e/b4pTgAAABv+RRuMjMzlZubq9TUVA0aNEg2m83fdQEAAPjEp3CTl5enN954Q1OmTPF3PQAAAC3i00fBw8PD1bdvX3/XAgAA0GI+hZunn35aq1evlmVZ/q4HAACgRXx6W+rAgQN6//33tXPnTt15553q0KGD2/1btmzxS3EAAADe8inc3HzzzXrwwQf9XQsAAECL+RRuNmzY4O86AAAA/MKnPTeS9Pe//13vvfee1q1bp8uXL0uSzp8/r5qaGr8VBwAA4C2fVm5KS0v1wAMPqKysTA6HQ/fff78iIyO1fPlyORwO5eTk+LtOAAAAj/i0cpOZmamRI0fq0qVL6tixo6v9wQcfVEFBgd+KAwAA8JZPKzf/8z//o4MHDyo8PNytvXfv3jp37pxfCgMAAPCFTys3TqdT9fX1DdrPnj2ryMjIFhcFAADgK5/CzaRJk/TCCy+4bttsNtXU1Gjx4sX8JAMAAAgon96Wys7OVkpKigYOHKgrV67okUce0WeffaauXbvq9ddf93eNAAAAHvMp3PTs2VOffPKJ8vLydPToUdXU1GjWrFl69NFH3TYYAwAAtDWfwo0ktW/fXo899pg/awEAAGgxn8LNq6++2uz96enpPhUDAADQUj6Fm8zMTLfbdXV1+utf/6rw8HB16tSJcAMAAALGp09LXbp0ye2oqanRiRMndM8997ChGAAABJTPvy11vX79+mnZsmUNVnWas3//fk2dOlXdu3eXzWbTtm3bmu2/d+9e2Wy2BkdFRUULqwcAAKbwW7iRvtlkfP78eY/719bWaujQoVqzZo1Xj3PixAmVl5e7jpiYGG9LBQAAhvJpz81///d/u922LEvl5eV68cUXdffdd3t8ncmTJ2vy5MleP35MTIxuvvlmr88DAADm8yncpKWlud222Wzq1q2b7rvvPmVnZ/ujrmYlJibK4XBo0KBB+uUvf9lsoHI4HHI4HK7b1dXVrV4fAAAIHJ/CjdPp9HcdHomPj1dOTo5Gjhwph8OhV155RcnJyfrTn/6k4cOHN3pOVlaWlixZ0saVAgCAQPH5S/wCYcCAARowYIDr9tixY3Xq1CmtWrVKf/zjHxs9Z9GiRZo/f77rdnV1tRISElq9VgAAEBg+hZtvh4UbWblypS8P4bFRo0bpwIEDTd5vt9tlt9tbtQYAABA8fAo3RUVFKioqUl1dnWsl5dNPP1W7du3c3h6y2Wz+qbIZxcXFio+Pb/XHAQAAocGncDN16lRFRkbqD3/4gzp37izpmy/2mzlzpu699149/fTTHl2npqZGJ0+edN0+ffq0iouL1aVLF916661atGiRzp075/q5hxdeeEF9+vTRnXfeqStXruiVV17Rnj179O677/oyDAAAYCCfwk12drbeffddV7CRpM6dO+vXv/61Jk2a5HG4OXTokMaPH++6fe3troyMDOXm5qq8vFxlZWWu+69evaqnn35a586dU6dOnTRkyBC99957btcAAADfbT6Fm+rqal28eLFB+8WLF3X58mWPr5OcnCzLspq8Pzc31+32ggULtGDBAo+vDwAAvnt8+obiBx98UDNnztSWLVt09uxZnT17Vm+99ZZmzZqlhx56yN81AgAAeMynlZucnBw988wzeuSRR1RXV/fNhdq316xZs7RixQq/FggAAOANn8JNp06d9NJLL2nFihU6deqUJOm2227T9773Pb8WBwAA4K0W/XDmtR+u7Nevn773ve81u38GAACgLfgUbv7v//5PEyZMUP/+/TVlyhSVl5dLkmbNmuXxJ6UAAABag0/h5qc//ak6dOigsrIyderUydU+ffp07dq1y2/FAQAAeMunPTfvvvuudu/erZ49e7q19+vXT6WlpX4pDAAAwBc+rdzU1ta6rdhc89VXX/E7TgAAIKB8Cjf33nuv6ycRpG9+Q8rpdOr555/n24IBAEBA+fS21PPPP68JEybo0KFDunr1qhYsWKDjx4/rq6++0gcffODvGgEAADzm08rNoEGD9Omnn+qee+7RtGnTVFtbq4ceekhFRUW67bbb/F0jAACAx7xeuamrq9MDDzygnJwc/fznP2+NmgAAAHzm9cpNhw4ddPTo0daoBQAAoMV8elvqscce0+9//3t/1wIAANBiPm0o/vvf/67169frvffe04gRIxr8ptTKlSv9UhwAAIC3vAo3n3/+uXr37q1jx45p+PDhkqRPP/3UrY/NZvNfdQAAAF7yKtz069dP5eXlev/99yV983ML//Vf/6XY2NhWKQ4AAMBbXu25uf5Xv3fu3Kna2lq/FgQAANASPm0ovub6sAMAABBoXoUbm83WYE8Ne2wAAEAw8WrPjWVZevzxx10/jnnlyhU9+eSTDT4ttWXLFv9VCAAA4AWvwk1GRobb7ccee8yvxQAAALSUV+Fmw4YNrVUHAACAX7RoQzEAAECwIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGCWi42b9/v6ZOnaru3bvLZrNp27ZtNzxn7969Gj58uOx2u/r27avc3NxWrxMAAISOgIab2tpaDR06VGvWrPGo/+nTp5Wamqrx48eruLhY8+bN0xNPPKHdu3e3cqUAACBUtA/kg0+ePFmTJ0/2uH9OTo769Omj7OxsSdIdd9yhAwcOaNWqVUpJSWmtMgEAQAgJqT03hYWFmjhxoltbSkqKCgsLmzzH4XCourra7QAAAOYK6MqNtyoqKhQbG+vWFhsbq+rqav3tb39Tx44dG5yTlZWlJUuWtFWJDfRe+HaDti+WpfrlOp7w5LGuv7Yv5/jzsXypx5+CvUZ//ZsKRW059u/y8wx4IxhfKyG1cuOLRYsWqaqqynWcOXMm0CUBAIBWFFIrN3FxcaqsrHRrq6ysVFRUVKOrNpJkt9tlt9vbojwAABAEQmrlJikpSQUFBW5t+fn5SkpKClBFAAAg2AQ03NTU1Ki4uFjFxcWSvvmod3FxscrKyiR985ZSenq6q/+TTz6pzz//XAsWLND//u//6qWXXtIbb7yhn/70p4EoHwAABKGAhptDhw5p2LBhGjZsmCRp/vz5GjZsmJ577jlJUnl5uSvoSFKfPn309ttvKz8/X0OHDlV2drZeeeUVPgYOAABcArrnJjk5WZZlNXl/Y98+nJycrKKiolasCgAAhLKQ2nMDAABwI4QbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAowRFuFmzZo169+6tiIgIjR49Wh999FGTfXNzc2Wz2dyOiIiINqwWAAAEs4CHm02bNmn+/PlavHixjhw5oqFDhyolJUUXLlxo8pyoqCiVl5e7jtLS0jasGAAABLOAh5uVK1dq9uzZmjlzpgYOHKicnBx16tRJ69evb/Icm82muLg41xEbG9uGFQMAgGAW0HBz9epVHT58WBMnTnS1hYWFaeLEiSosLGzyvJqaGvXq1UsJCQmaNm2ajh8/3mRfh8Oh6upqtwMAAJgroOHmyy+/VH19fYOVl9jYWFVUVDR6zoABA7R+/Xpt375dr732mpxOp8aOHauzZ8822j8rK0vR0dGuIyEhwe/jAAAAwSPgb0t5KykpSenp6UpMTNS4ceO0ZcsWdevWTevWrWu0/6JFi1RVVeU6zpw508YVAwCAttQ+kA/etWtXtWvXTpWVlW7tlZWViouL8+gaHTp00LBhw3Ty5MlG77fb7bLb7S2uFQAAhIaArtyEh4drxIgRKigocLU5nU4VFBQoKSnJo2vU19erpKRE8fHxrVUmAAAIIQFduZGk+fPnKyMjQyNHjtSoUaP0wgsvqLa2VjNnzpQkpaenq0ePHsrKypIk/epXv9KYMWPUt29fff3111qxYoVKS0v1xBNPBHIYAAAgSAQ83EyfPl0XL17Uc889p4qKCiUmJmrXrl2uTcZlZWUKC/vnAtOlS5c0e/ZsVVRUqHPnzhoxYoQOHjyogQMHBmoIAAAgiAQ83EjS3LlzNXfu3Ebv27t3r9vtVatWadWqVW1QFQAACEUh92kpAACA5hBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACMQrgBAABGIdwAAACjEG4AAIBRCDcAAMAohBsAAGAUwg0AADAK4QYAABiFcAMAAIxCuAEAAEYh3AAAAKMQbgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARgmKcLNmzRr17t1bERERGj16tD766KNm+2/evFm33367IiIiNHjwYL3zzjttVCkAAAh2AQ83mzZt0vz587V48WIdOXJEQ4cOVUpKii5cuNBo/4MHD2rGjBmaNWuWioqKlJaWprS0NB07dqyNKwcAAMEo4OFm5cqVmj17tmbOnKmBAwcqJydHnTp10vr16xvtv3r1aj3wwAN69tlndccdd2jp0qUaPny4XnzxxTauHAAABKP2gXzwq1ev6vDhw1q0aJGrLSwsTBMnTlRhYWGj5xQWFmr+/PlubSkpKdq2bVuj/R0OhxwOh+t2VVWVJKm6urqF1TfO6fjrDftc/9iNneNJH094Ms7rr93YOb6My1+P5Wk9/ppTf9Xor8e60TktefxQ05Zj/y4/z4A32uq1cu2almXduLMVQOfOnbMkWQcPHnRrf/bZZ61Ro0Y1ek6HDh2sjRs3urWtWbPGiomJabT/4sWLLUkcHBwcHBwcBhxnzpy5Yb4I6MpNW1i0aJHbSo/T6dRXX32lW265RTabzatrVVdXKyEhQWfOnFFUVJS/Sw0Kpo/R9PFJ5o/R9PFJ5o/R9PFJ5o8xEOOzLEuXL19W9+7db9g3oOGma9euateunSorK93aKysrFRcX1+g5cXFxXvW32+2y2+1ubTfffLPvRUuKiooy8h/rt5k+RtPHJ5k/RtPHJ5k/RtPHJ5k/xrYeX3R0tEf9ArqhODw8XCNGjFBBQYGrzel0qqCgQElJSY2ek5SU5NZfkvLz85vsDwAAvlsC/rbU/PnzlZGRoZEjR2rUqFF64YUXVFtbq5kzZ0qS0tPT1aNHD2VlZUmSMjMzNW7cOGVnZys1NVV5eXk6dOiQXn755UAOAwAABImAh5vp06fr4sWLeu6551RRUaHExETt2rVLsbGxkqSysjKFhf1zgWns2LHauHGjfvGLX+hnP/uZ+vXrp23btmnQoEGtXqvdbtfixYsbvM1lEtPHaPr4JPPHaPr4JPPHaPr4JPPHGOzjs1mWJ5+pAgAACA0B/xI/AAAAfyLcAAAAoxBuAACAUQg3AADAKISb66xZs0a9e/dWRESERo8erY8++qjJvrm5ubLZbG5HREREG1brnf3792vq1Knq3r27bDZbk7/H9W179+7V8OHDZbfb1bdvX+Xm5rZ6nS3h7Rj37t3bYA5tNpsqKirapmAvZWVl6a677lJkZKRiYmKUlpamEydO3PC8zZs36/bbb1dERIQGDx6sd955pw2q9Z4v4wu11+HatWs1ZMgQ15efJSUlaefOnc2eEyrzJ3k/vlCbv8YsW7ZMNptN8+bNa7ZfKM3jt3kyvmCbR8LNt2zatEnz58/X4sWLdeTIEQ0dOlQpKSm6cOFCk+dERUWpvLzcdZSWlrZhxd6pra3V0KFDtWbNGo/6nz59WqmpqRo/fryKi4s1b948PfHEE9q9e3crV+o7b8d4zYkTJ9zmMSYmppUqbJl9+/Zpzpw5+vDDD5Wfn6+6ujpNmjRJtbW1TZ5z8OBBzZgxQ7NmzVJRUZHS0tKUlpamY8eOtWHlnvFlfFJovQ579uypZcuW6fDhwzp06JDuu+8+TZs2TcePH2+0fyjNn+T9+KTQmr/rffzxx1q3bp2GDBnSbL9Qm8drPB2fFGTz6MkPXH5XjBo1ypozZ47rdn19vdW9e3crKyur0f4bNmywoqOj26g6/5Jkbd26tdk+CxYssO688063tunTp1spKSmtWJn/eDLG999/35JkXbp0qU1q8rcLFy5Ykqx9+/Y12edf//VfrdTUVLe20aNHWz/5yU9au7wW82R8ofw6vKZz587WK6+80uh9oTx/1zQ3vlCev8uXL1v9+vWz8vPzrXHjxlmZmZlN9g3FefRmfME2j6zc/MPVq1d1+PBhTZw40dUWFhamiRMnqrCwsMnzampq1KtXLyUkJNzwfyehprCw0O35kKSUlJRmn49QlZiYqPj4eN1///364IMPAl2Ox6qqqiRJXbp0abJPKM+jJ+OTQvd1WF9fr7y8PNXW1jb5EzKhPH+ejE8K3fmbM2eOUlNTG8xPY0JxHr0ZnxRc80i4+Ycvv/xS9fX1rm9GviY2NrbJ/RcDBgzQ+vXrtX37dr322mtyOp0aO3aszp492xYlt7qKiopGn4/q6mr97W9/C1BV/hUfH6+cnBy99dZbeuutt5SQkKDk5GQdOXIk0KXdkNPp1Lx583T33Xc3+w3dTc1jsO4rusbT8YXi67CkpEQ33XST7Ha7nnzySW3dulUDBw5stG8ozp834wvF+ZOkvLw8HTlyxPXTQDcSavPo7fiCbR4D/vMLoSwpKcntfyNjx47VHXfcoXXr1mnp0qUBrAyeGjBggAYMGOC6PXbsWJ06dUqrVq3SH//4xwBWdmNz5szRsWPHdODAgUCX0io8HV8ovg4HDBig4uJiVVVV6c0331RGRob27dvXZAAINd6MLxTn78yZM8rMzFR+fn7IbX72hC/jC7Z5JNz8Q9euXdWuXTtVVla6tVdWViouLs6ja3To0EHDhg3TyZMnW6PENhcXF9fo8xEVFaWOHTsGqKrWN2rUqKAPDHPnztWOHTu0f/9+9ezZs9m+Tc2jp/+uA8Gb8V0vFF6H4eHh6tu3ryRpxIgR+vjjj7V69WqtW7euQd9QnD9vxne9UJi/w4cP68KFCxo+fLirrb6+Xvv379eLL74oh8Ohdu3auZ0TSvPoy/iuF+h55G2pfwgPD9eIESNUUFDganM6nSooKGj2veJvq6+vV0lJieLj41urzDaVlJTk9nxIUn5+vsfPR6gqLi4O2jm0LEtz587V1q1btWfPHvXp0+eG54TSPPoyvuuF4uvQ6XTK4XA0el8ozV9Tmhvf9UJh/iZMmKCSkhIVFxe7jpEjR+rRRx9VcXFxo3/4Q2kefRnf9QI+j4He0RxM8vLyLLvdbuXm5lp//vOfrX/7t3+zbr75ZquiosKyLMv60Y9+ZC1cuNDVf8mSJdbu3butU6dOWYcPH7Z++MMfWhEREdbx48cDNYRmXb582SoqKrKKioosSdbKlSutoqIiq7S01LIsy1q4cKH1ox/9yNX/888/tzp16mQ9++yz1l/+8hdrzZo1Vrt27axdu3YFagg35O0YV61aZW3bts367LPPrJKSEiszM9MKCwuz3nvvvUANoVlPPfWUFR0dbe3du9cqLy93HX/9619dfa7/d/rBBx9Y7du3t/7zP//T+stf/mItXrzY6tChg1VSUhKIITTLl/GF2utw4cKF1r59+6zTp09bR48etRYuXGjZbDbr3XfftSwrtOfPsrwfX6jNX1Ou/zRRqM/j9W40vmCbR8LNdX73u99Zt956qxUeHm6NGjXK+vDDD133jRs3zsrIyHDdnjdvnqtvbGysNWXKFOvIkSMBqNoz1z72fP1xbUwZGRnWuHHjGpyTmJhohYeHW9///vetDRs2tHnd3vB2jMuXL7duu+02KyIiwurSpYuVnJxs7dmzJzDFe6CxsUlym5fr/51almW98cYbVv/+/a3w8HDrzjvvtN5+++22LdxDvowv1F6HP/7xj61evXpZ4eHhVrdu3awJEya4/vBbVmjPn2V5P75Qm7+mXP/HP9Tn8Xo3Gl+wzaPNsiyr7daJAAAAWhd7bgAAgFEINwAAwCiEGwAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAv9i/f7+mTp2q7t27y2azadu2bV5f44033lBiYqI6deqkXr16acWKFV5fg3ADAAD8ora2VkOHDtWaNWt8On/nzp169NFH9eSTT+rYsWN66aWXtGrVKr344oteXYdvKAYAAH5ns9m0detWpaWludocDod+/vOf6/XXX9fXX3+tQYMGafny5UpOTpYkPfLII6qrq9PmzZtd5/zud7/T888/r7KyMtlsNo8em5UbAADQJubOnavCwkLl5eXp6NGjevjhh/XAAw/os88+k/RN+ImIiHA7p2PHjjp79qxKS0s9fhzCDQAAaHVlZWXasGGDNm/erHvvvVe33XabnnnmGd1zzz3asGGDJCklJUVbtmxRQUGBnE6nPv30U2VnZ0uSysvLPX6s9q0yAgAAgG8pKSlRfX29+vfv79bucDh0yy23SJJmz56tU6dO6V/+5V9UV1enqKgoZWZm6pe//KXCwjxfjyHcAACAVldTU6N27drp8OHDateundt9N910k6Rv9uksX75cv/3tb1VRUaFu3bqpoKBAkvT973/f48ci3AAAgFY3bNgw1dfX68KFC7r33nub7duuXTv16NFDkvT6668rKSlJ3bp18/ixCDcAAMAvampqdPLkSdft06dPq7i4WF26dFH//v316KOPKj09XdnZ2Ro2bJguXryogoICDRkyRKmpqfryyy/15ptvKjk5WVeuXHHt0dm3b59XdfBRcAAA4Bd79+7V+PHjG7RnZGQoNzdXdXV1+vWvf61XX31V586dU9euXTVmzBgtWbJEgwcP1pdffqmpU6eqpKRElmUpKSlJv/nNbzR69Giv6iDcAAAAo/BRcAAAYBTCDQAAMArhBgAAGIVwAwAAjEK4AQAARiHcAAAAoxBuAACAUQg3AADAKIQbAABgFMINAAAwCuEGAAAYhXADAACM8v/ACWMjK75tTgAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "resources_df[resources_df['pg_relation_size'] > 10485760 * 50]['pg_relation_size'].plot.hist(bins=100)\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Explore Package Data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "data": {
      "text/plain": "                                                                                   name  \\\nid                                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c                        baum-center-zoning-overlay1   \n6916c51e-4ab2-47c8-bd18-90ef306841f7        allegheny-county-private-schools-locations2   \n46fb3ca4-e844-4b42-b034-e87291d34699                 pittsburgh-public-school-locations   \n76250dae-cad5-401e-acf4-b771dd30ae9b     allegheny-county-building-footprint-locations1   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a          residential-permit-parking-program-areas0   \n...                                                                                 ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292       pittsburgh-parking-meters-and-payment-points   \nd660edf8-9157-45ad-a282-50822badfaae                   pittsburgh-pli-violations-report   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39                               condemned-properties   \nac32c80c-de99-4444-a9a8-48c0515ac7bb                               parking-transactions   \n8744b4f6-5525-49be-9054-401a2c4c2fac  allegheny-county-restaurant-food-facility-insp...   \n\n                                                                                  title  \\\nid                                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c                         Baum Center Zoning Overlay   \n6916c51e-4ab2-47c8-bd18-90ef306841f7         Allegheny County Private Schools Locations   \n46fb3ca4-e844-4b42-b034-e87291d34699                 Pittsburgh Public School Locations   \n76250dae-cad5-401e-acf4-b771dd30ae9b      Allegheny County Building Footprint Locations   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a           Residential Permit Parking Program Areas   \n...                                                                                 ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292       Pittsburgh Parking Meters and Payment Points   \nd660edf8-9157-45ad-a282-50822badfaae              Pittsburgh PLI/DOMI Violations Report   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39                  Condemned and Dead-End Properties   \nac32c80c-de99-4444-a9a8-48c0515ac7bb                    Aggregated Parking Transactions   \n8744b4f6-5525-49be-9054-401a2c4c2fac  Allegheny County Restaurant/Food Facility Insp...   \n\n                                      version  \\\nid                                              \n7699b899-6e01-44cd-841a-7962cd43311c      NaN   \n6916c51e-4ab2-47c8-bd18-90ef306841f7      NaN   \n46fb3ca4-e844-4b42-b034-e87291d34699      NaN   \n76250dae-cad5-401e-acf4-b771dd30ae9b      NaN   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a      NaN   \n...                                       ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292      NaN   \nd660edf8-9157-45ad-a282-50822badfaae      NaN   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39      NaN   \nac32c80c-de99-4444-a9a8-48c0515ac7bb      NaN   \n8744b4f6-5525-49be-9054-401a2c4c2fac      NaN   \n\n                                                                                    url  \\\nid                                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c  http://pghgis-pittsburghpa.opendata.arcgis.com...   \n6916c51e-4ab2-47c8-bd18-90ef306841f7  https://openac-alcogis.opendata.arcgis.com/map...   \n46fb3ca4-e844-4b42-b034-e87291d34699                                                NaN   \n76250dae-cad5-401e-acf4-b771dd30ae9b  https://openac-alcogis.opendata.arcgis.com/dat...   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a  http://pghgis-pittsburghpa.opendata.arcgis.com...   \n...                                                                                 ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292                                                NaN   \nd660edf8-9157-45ad-a282-50822badfaae                                                NaN   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39                                                NaN   \nac32c80c-de99-4444-a9a8-48c0515ac7bb                                                NaN   \n8744b4f6-5525-49be-9054-401a2c4c2fac                                                NaN   \n\n                                                                                  notes  \\\nid                                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c              Pittsburgh Baum Centre Zoning Overlay   \n6916c51e-4ab2-47c8-bd18-90ef306841f7  <span style='font-size:10.0pt; font-family:&qu...   \n46fb3ca4-e844-4b42-b034-e87291d34699  Pittsburgh Public School locations, including ...   \n76250dae-cad5-401e-acf4-b771dd30ae9b  <span style='font-size:10.0pt; font-family:&qu...   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a             PGH Residential Permit Parking Program   \n...                                                                                 ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292  This dataset contains information on the Pitts...   \nd660edf8-9157-45ad-a282-50822badfaae  This dataset originally housed Department of P...   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39  This dataset contains condemned properties in ...   \nac32c80c-de99-4444-a9a8-48c0515ac7bb  This dataset records Pittsburgh Parking Author...   \n8744b4f6-5525-49be-9054-401a2c4c2fac  Facilities located within Allegheny County tha...   \n\n                                        license_id  \\\nid                                                   \n7699b899-6e01-44cd-841a-7962cd43311c  notspecified   \n6916c51e-4ab2-47c8-bd18-90ef306841f7           NaN   \n46fb3ca4-e844-4b42-b034-e87291d34699      other-pd   \n76250dae-cad5-401e-acf4-b771dd30ae9b           NaN   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a  notspecified   \n...                                            ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292         cc-by   \nd660edf8-9157-45ad-a282-50822badfaae       cc-zero   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39         cc-by   \nac32c80c-de99-4444-a9a8-48c0515ac7bb         cc-by   \n8744b4f6-5525-49be-9054-401a2c4c2fac       cc-zero   \n\n                                                               revision_id  \\\nid                                                                           \n7699b899-6e01-44cd-841a-7962cd43311c  37700bb2-dc97-4a01-855c-7d9996088b40   \n6916c51e-4ab2-47c8-bd18-90ef306841f7  d59b6057-424b-49b9-aa0a-65a69f55d50b   \n46fb3ca4-e844-4b42-b034-e87291d34699  387edaca-4185-47f5-807e-770613478ba3   \n76250dae-cad5-401e-acf4-b771dd30ae9b  cb5c7ebd-f0ba-47a5-90cc-26503ec8b994   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a  c5936c71-b1ad-4345-9e76-b9753726b2ff   \n...                                                                    ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292  0bb14f0f-b32d-4483-a187-6280110198cc   \nd660edf8-9157-45ad-a282-50822badfaae  1a58b667-e8c9-4587-9227-f7027a430fc5   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39  a4a6ea9b-7358-4e7d-a4a6-40ca50bb380b   \nac32c80c-de99-4444-a9a8-48c0515ac7bb  57d450d3-e3a3-4930-aa1d-158f9792373d   \n8744b4f6-5525-49be-9054-401a2c4c2fac  142ff16f-a004-419b-b1d1-cb7de3434594   \n\n                                     author author_email maintainer  \\\nid                                                                    \n7699b899-6e01-44cd-841a-7962cd43311c    NaN          NaN        NaN   \n6916c51e-4ab2-47c8-bd18-90ef306841f7    NaN          NaN        NaN   \n46fb3ca4-e844-4b42-b034-e87291d34699    NaN          NaN        NaN   \n76250dae-cad5-401e-acf4-b771dd30ae9b    NaN          NaN        NaN   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a    NaN          NaN        NaN   \n...                                     ...          ...        ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292    NaN          NaN        NaN   \nd660edf8-9157-45ad-a282-50822badfaae    NaN          NaN        NaN   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39    NaN          NaN        NaN   \nac32c80c-de99-4444-a9a8-48c0515ac7bb    NaN          NaN        NaN   \n8744b4f6-5525-49be-9054-401a2c4c2fac    NaN          NaN        NaN   \n\n                                     maintainer_email    state     type  \\\nid                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c              NaN  deleted  dataset   \n6916c51e-4ab2-47c8-bd18-90ef306841f7              NaN  deleted  dataset   \n46fb3ca4-e844-4b42-b034-e87291d34699              NaN   active  dataset   \n76250dae-cad5-401e-acf4-b771dd30ae9b              NaN  deleted  dataset   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a              NaN  deleted  dataset   \n...                                               ...      ...      ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292              NaN   active  dataset   \nd660edf8-9157-45ad-a282-50822badfaae              NaN   active  dataset   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39              NaN   active  dataset   \nac32c80c-de99-4444-a9a8-48c0515ac7bb              NaN   active  dataset   \n8744b4f6-5525-49be-9054-401a2c4c2fac              NaN   active  dataset   \n\n                                                                 owner_org  \\\nid                                                                           \n7699b899-6e01-44cd-841a-7962cd43311c  9ecaff80-fb4a-457b-8141-e53f7c991890   \n6916c51e-4ab2-47c8-bd18-90ef306841f7  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n46fb3ca4-e844-4b42-b034-e87291d34699  4b16ad54-d76c-47fc-b223-323950e38a14   \n76250dae-cad5-401e-acf4-b771dd30ae9b  f6dcf8fb-2051-4226-8c74-de9fbc699684   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a  9ecaff80-fb4a-457b-8141-e53f7c991890   \n...                                                                    ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292  7e3450fb-7b3e-41c9-aa1e-ca24980ec85a   \nd660edf8-9157-45ad-a282-50822badfaae  9ecaff80-fb4a-457b-8141-e53f7c991890   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39  9ecaff80-fb4a-457b-8141-e53f7c991890   \nac32c80c-de99-4444-a9a8-48c0515ac7bb  7e3450fb-7b3e-41c9-aa1e-ca24980ec85a   \n8744b4f6-5525-49be-9054-401a2c4c2fac  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n\n                                     private          metadata_modified  \\\nid                                                                        \n7699b899-6e01-44cd-841a-7962cd43311c       f 2021-02-04 02:53:44.641226   \n6916c51e-4ab2-47c8-bd18-90ef306841f7       f 2022-04-27 16:53:09.666020   \n46fb3ca4-e844-4b42-b034-e87291d34699       f 2020-06-09 15:35:57.506390   \n76250dae-cad5-401e-acf4-b771dd30ae9b       f 2021-03-09 17:00:26.028171   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a       f 2021-03-10 03:08:32.107392   \n...                                      ...                        ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292       f 2023-04-09 11:56:09.328649   \nd660edf8-9157-45ad-a282-50822badfaae       f 2023-04-09 07:49:05.888904   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39       f 2023-04-09 07:55:39.684565   \nac32c80c-de99-4444-a9a8-48c0515ac7bb       f 2023-04-10 01:20:42.318872   \n8744b4f6-5525-49be-9054-401a2c4c2fac       f 2023-04-09 08:21:09.720142   \n\n                                                           creator_user_id  \\\nid                                                                           \n7699b899-6e01-44cd-841a-7962cd43311c  90e6b24a-729c-43ea-9a24-2e060d24d07d   \n6916c51e-4ab2-47c8-bd18-90ef306841f7  90e6b24a-729c-43ea-9a24-2e060d24d07d   \n46fb3ca4-e844-4b42-b034-e87291d34699  9c1c3a99-0232-4b6c-b105-4a1ac5074184   \n76250dae-cad5-401e-acf4-b771dd30ae9b  90e6b24a-729c-43ea-9a24-2e060d24d07d   \nae42bbb0-4aee-441c-aabb-594c5b4fd79a  90e6b24a-729c-43ea-9a24-2e060d24d07d   \n...                                                                    ...   \n1d38b5ef-fa53-4c15-9a50-905d4d573292  085f4393-3645-4ace-b4ad-6fa60a5b14c0   \nd660edf8-9157-45ad-a282-50822badfaae  9195f191-1e0a-4b81-ba44-ddb02fd4ee0d   \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39  c04d83b9-8d45-45c6-b6c3-f82aa39ae539   \nac32c80c-de99-4444-a9a8-48c0515ac7bb  085f4393-3645-4ace-b4ad-6fa60a5b14c0   \n8744b4f6-5525-49be-9054-401a2c4c2fac  ed442faf-d3dc-48fb-8964-133201905237   \n\n                                               metadata_created  \nid                                                               \n7699b899-6e01-44cd-841a-7962cd43311c 2018-05-23 14:13:21.426230  \n6916c51e-4ab2-47c8-bd18-90ef306841f7 2022-04-27 15:36:51.210709  \n46fb3ca4-e844-4b42-b034-e87291d34699 2019-04-29 20:41:15.237263  \n76250dae-cad5-401e-acf4-b771dd30ae9b 2020-10-20 17:00:11.621894  \nae42bbb0-4aee-441c-aabb-594c5b4fd79a 2018-05-23 14:12:57.560308  \n...                                                         ...  \n1d38b5ef-fa53-4c15-9a50-905d4d573292 2018-07-28 22:46:01.308980  \nd660edf8-9157-45ad-a282-50822badfaae 2015-12-04 16:15:11.183903  \nf9006efd-eac0-467a-a0d2-3b3a9a59cd39 2023-01-27 18:33:04.492449  \nac32c80c-de99-4444-a9a8-48c0515ac7bb 2018-07-30 14:01:14.674776  \n8744b4f6-5525-49be-9054-401a2c4c2fac 2016-09-14 13:55:57.761681  \n\n[530 rows x 18 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>name</th>\n      <th>title</th>\n      <th>version</th>\n      <th>url</th>\n      <th>notes</th>\n      <th>license_id</th>\n      <th>revision_id</th>\n      <th>author</th>\n      <th>author_email</th>\n      <th>maintainer</th>\n      <th>maintainer_email</th>\n      <th>state</th>\n      <th>type</th>\n      <th>owner_org</th>\n      <th>private</th>\n      <th>metadata_modified</th>\n      <th>creator_user_id</th>\n      <th>metadata_created</th>\n    </tr>\n    <tr>\n      <th>id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>7699b899-6e01-44cd-841a-7962cd43311c</th>\n      <td>baum-center-zoning-overlay1</td>\n      <td>Baum Center Zoning Overlay</td>\n      <td>NaN</td>\n      <td>http://pghgis-pittsburghpa.opendata.arcgis.com...</td>\n      <td>Pittsburgh Baum Centre Zoning Overlay</td>\n      <td>notspecified</td>\n      <td>37700bb2-dc97-4a01-855c-7d9996088b40</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>deleted</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2021-02-04 02:53:44.641226</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2018-05-23 14:13:21.426230</td>\n    </tr>\n    <tr>\n      <th>6916c51e-4ab2-47c8-bd18-90ef306841f7</th>\n      <td>allegheny-county-private-schools-locations2</td>\n      <td>Allegheny County Private Schools Locations</td>\n      <td>NaN</td>\n      <td>https://openac-alcogis.opendata.arcgis.com/map...</td>\n      <td>&lt;span style='font-size:10.0pt; font-family:&amp;qu...</td>\n      <td>NaN</td>\n      <td>d59b6057-424b-49b9-aa0a-65a69f55d50b</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>deleted</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2022-04-27 16:53:09.666020</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2022-04-27 15:36:51.210709</td>\n    </tr>\n    <tr>\n      <th>46fb3ca4-e844-4b42-b034-e87291d34699</th>\n      <td>pittsburgh-public-school-locations</td>\n      <td>Pittsburgh Public School Locations</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Pittsburgh Public School locations, including ...</td>\n      <td>other-pd</td>\n      <td>387edaca-4185-47f5-807e-770613478ba3</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>4b16ad54-d76c-47fc-b223-323950e38a14</td>\n      <td>f</td>\n      <td>2020-06-09 15:35:57.506390</td>\n      <td>9c1c3a99-0232-4b6c-b105-4a1ac5074184</td>\n      <td>2019-04-29 20:41:15.237263</td>\n    </tr>\n    <tr>\n      <th>76250dae-cad5-401e-acf4-b771dd30ae9b</th>\n      <td>allegheny-county-building-footprint-locations1</td>\n      <td>Allegheny County Building Footprint Locations</td>\n      <td>NaN</td>\n      <td>https://openac-alcogis.opendata.arcgis.com/dat...</td>\n      <td>&lt;span style='font-size:10.0pt; font-family:&amp;qu...</td>\n      <td>NaN</td>\n      <td>cb5c7ebd-f0ba-47a5-90cc-26503ec8b994</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>deleted</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2021-03-09 17:00:26.028171</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2020-10-20 17:00:11.621894</td>\n    </tr>\n    <tr>\n      <th>ae42bbb0-4aee-441c-aabb-594c5b4fd79a</th>\n      <td>residential-permit-parking-program-areas0</td>\n      <td>Residential Permit Parking Program Areas</td>\n      <td>NaN</td>\n      <td>http://pghgis-pittsburghpa.opendata.arcgis.com...</td>\n      <td>PGH Residential Permit Parking Program</td>\n      <td>notspecified</td>\n      <td>c5936c71-b1ad-4345-9e76-b9753726b2ff</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>deleted</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2021-03-10 03:08:32.107392</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2018-05-23 14:12:57.560308</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1d38b5ef-fa53-4c15-9a50-905d4d573292</th>\n      <td>pittsburgh-parking-meters-and-payment-points</td>\n      <td>Pittsburgh Parking Meters and Payment Points</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>This dataset contains information on the Pitts...</td>\n      <td>cc-by</td>\n      <td>0bb14f0f-b32d-4483-a187-6280110198cc</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>7e3450fb-7b3e-41c9-aa1e-ca24980ec85a</td>\n      <td>f</td>\n      <td>2023-04-09 11:56:09.328649</td>\n      <td>085f4393-3645-4ace-b4ad-6fa60a5b14c0</td>\n      <td>2018-07-28 22:46:01.308980</td>\n    </tr>\n    <tr>\n      <th>d660edf8-9157-45ad-a282-50822badfaae</th>\n      <td>pittsburgh-pli-violations-report</td>\n      <td>Pittsburgh PLI/DOMI Violations Report</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>This dataset originally housed Department of P...</td>\n      <td>cc-zero</td>\n      <td>1a58b667-e8c9-4587-9227-f7027a430fc5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2023-04-09 07:49:05.888904</td>\n      <td>9195f191-1e0a-4b81-ba44-ddb02fd4ee0d</td>\n      <td>2015-12-04 16:15:11.183903</td>\n    </tr>\n    <tr>\n      <th>f9006efd-eac0-467a-a0d2-3b3a9a59cd39</th>\n      <td>condemned-properties</td>\n      <td>Condemned and Dead-End Properties</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>This dataset contains condemned properties in ...</td>\n      <td>cc-by</td>\n      <td>a4a6ea9b-7358-4e7d-a4a6-40ca50bb380b</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2023-04-09 07:55:39.684565</td>\n      <td>c04d83b9-8d45-45c6-b6c3-f82aa39ae539</td>\n      <td>2023-01-27 18:33:04.492449</td>\n    </tr>\n    <tr>\n      <th>ac32c80c-de99-4444-a9a8-48c0515ac7bb</th>\n      <td>parking-transactions</td>\n      <td>Aggregated Parking Transactions</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>This dataset records Pittsburgh Parking Author...</td>\n      <td>cc-by</td>\n      <td>57d450d3-e3a3-4930-aa1d-158f9792373d</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>7e3450fb-7b3e-41c9-aa1e-ca24980ec85a</td>\n      <td>f</td>\n      <td>2023-04-10 01:20:42.318872</td>\n      <td>085f4393-3645-4ace-b4ad-6fa60a5b14c0</td>\n      <td>2018-07-30 14:01:14.674776</td>\n    </tr>\n    <tr>\n      <th>8744b4f6-5525-49be-9054-401a2c4c2fac</th>\n      <td>allegheny-county-restaurant-food-facility-insp...</td>\n      <td>Allegheny County Restaurant/Food Facility Insp...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Facilities located within Allegheny County tha...</td>\n      <td>cc-zero</td>\n      <td>142ff16f-a004-419b-b1d1-cb7de3434594</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2023-04-09 08:21:09.720142</td>\n      <td>ed442faf-d3dc-48fb-8964-133201905237</td>\n      <td>2016-09-14 13:55:57.761681</td>\n    </tr>\n  </tbody>\n</table>\n<p>530 rows × 18 columns</p>\n</div>"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('package_list.csv', index_col='id', parse_dates=['metadata_created', 'metadata_modified'])\n",
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "data": {
      "text/plain": "       version\ncount      0.0\nmean       NaN\nstd        NaN\nmin        NaN\n25%        NaN\n50%        NaN\n75%        NaN\nmax        NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>version</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "data": {
      "text/plain": "count                              530\nmean     2022-02-20 21:59:47.114592512\nmin         2015-07-14 19:16:10.527033\n25%      2021-03-09 17:00:25.319997440\n50%      2022-04-27 16:53:08.873945088\n75%      2023-03-15 12:04:26.334837248\nmax         2023-04-10 01:23:05.024854\nName: metadata_modified, dtype: object"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['metadata_modified'].describe(datetime_is_numeric=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Compare Tables on New and Old DB"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [],
   "source": [
    "# details from current deployment\n",
    "# \\COPY (SELECT R.*, P.private FROM resource R JOIN package P on R.package_id = P.id WHERE P.private = 'f' AND R.state = 'active') TO '/home/ubuntu/resource_list.csv' CSV HEADER;\n",
    "old_resource_list_df = pd.read_csv('resource_list.csv').set_index('id')\n",
    "# \\COPY (SELECT * FROM package) TO '/home/ubuntu/package_list.csv' CSV HEADER;\n",
    "old_package_list_df = pd.read_csv('package_list.csv').set_index('id')\n",
    "\n",
    "# \\COPY (select  table_name,  pg_size_pretty(pg_relation_size(quote_ident(table_name))),  pg_relation_size(quote_ident(table_name)) from information_schema.tables where table_schema = 'public' order by 3 desc) TO '/path/to/table_sizes.csv' (format CSV);\n",
    "new_resource_sizes_df = pd.read_csv('dumped_resource_sizes.csv').set_index('id')\n",
    "old_resource_sizes_df = pd.read_csv('resource_sizes.csv').set_index('id')\n",
    "\n",
    "#  \\COPY (SELECT nspname AS schemaname,relname,reltuples FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE nspname NOT IN ('pg_catalog', 'information_schema') AND relkind='r' ORDER BY reltuples DESC) TO '/path/to/table_row_counts.csv' (format CSV);\n",
    "new_resource_rows_df = pd.read_csv('new_row_counts.csv').set_index('id')\n",
    "old_resource_rows_df = pd.read_csv('old_row_counts.csv').set_index('id')\n",
    "\n",
    "# join all sources of data\n",
    "size_comparison_df = old_resource_list_df.join(old_resource_sizes_df.join(new_resource_sizes_df).join(old_resource_rows_df).join(new_resource_rows_df))\n",
    "\n",
    "# the uploads were batched by IDs beginning with the same char\n",
    "size_comparison_df['group'] = size_comparison_df.index.str[0]\n",
    "\n",
    "# calculate size comparison columns\n",
    "size_comparison_df['size_diff'] = size_comparison_df['new_pg_relation_size'] - size_comparison_df['pg_relation_size']\n",
    "size_comparison_df['size_new/old'] = size_comparison_df['new_pg_relation_size'] / size_comparison_df['pg_relation_size']\n",
    "\n",
    "# calculate tuple (row?) counts\n",
    "size_comparison_df['tuples_diff'] = size_comparison_df['new_tuple_count'] - size_comparison_df['old_tuple_count']\n",
    "size_comparison_df['tuples_new/old'] = size_comparison_df['new_tuple_count'] / size_comparison_df['old_tuple_count']\n",
    "\n",
    "# add package meta data\n",
    "size_comparison_df = size_comparison_df.join(old_package_list_df, on='package_id', rsuffix='_package')\n",
    "\n",
    "# filter out private, and deleted datasets\n",
    "size_comparison_df = size_comparison_df[(size_comparison_df['state_package'] == 'active') & (size_comparison_df['private'] == 'f') & (size_comparison_df['state'] == 'active')]\n",
    "size_comparison_df = size_comparison_df.drop('notes', axis=1)\n",
    "size_comparison_df.to_csv('size_comparisons.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "data": {
      "text/plain": "                                                                                    url  \\\nid                                                                                        \nfd4d0f47-5a05-4716-861e-b0b328effe8b  https://data.wprdc.org/dataset/e41c0a67-837f-4...   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  https://data.wprdc.org/dataset/cfaffd5c-ab5a-4...   \nc926cfe2-e085-4808-94bc-b33895089a74  https://data.wprdc.org/dataset/8f92ae09-4cfa-4...   \nb378b3ba-71e9-418b-9697-ecdff6f62724                    http://www.makemytripcount.org/   \n8bda6be7-e90e-46d0-8577-693a27d8812b  https://data.wprdc.org/dataset/6bb2a968-761d-4...   \n...                                                                                 ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  https://data.wprdc.org/dataset/22fe57da-f5b8-4...   \n4af05575-052d-40ff-9311-d578319e810a  https://data.wprdc.org/datastore/dump/4af05575...   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  https://data.wprdc.org/datastore/dump/b4899889...   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  https://data.wprdc.org/datastore/dump/21b22189...   \nadffb94b-6175-4612-8628-44e7c9baa620  https://data.wprdc.org/datastore/dump/adffb94b...   \n\n                                       format  \\\nid                                              \nfd4d0f47-5a05-4716-861e-b0b328effe8b  GeoJSON   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc      CSV   \nc926cfe2-e085-4808-94bc-b33895089a74  GeoJSON   \nb378b3ba-71e9-418b-9697-ecdff6f62724     HTML   \n8bda6be7-e90e-46d0-8577-693a27d8812b      ZIP   \n...                                       ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b      ZIP   \n4af05575-052d-40ff-9311-d578319e810a      CSV   \nb4899889-fbf4-43a5-a5ba-ae178fa07347      CSV   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0      CSV   \nadffb94b-6175-4612-8628-44e7c9baa620      CSV   \n\n                                                                            description  \\\nid                                                                                        \nfd4d0f47-5a05-4716-861e-b0b328effe8b                                                NaN   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc                                                NaN   \nc926cfe2-e085-4808-94bc-b33895089a74                                                NaN   \nb378b3ba-71e9-418b-9697-ecdff6f62724                         Make My Trip Count Website   \n8bda6be7-e90e-46d0-8577-693a27d8812b                                                NaN   \n...                                                                                 ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  This is a compressed CSV file version of the d...   \n4af05575-052d-40ff-9311-d578319e810a                                                NaN   \nb4899889-fbf4-43a5-a5ba-ae178fa07347                                                NaN   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0                                                NaN   \nadffb94b-6175-4612-8628-44e7c9baa620                                                NaN   \n\n                                      position  \\\nid                                               \nfd4d0f47-5a05-4716-861e-b0b328effe8b         2   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc         0   \nc926cfe2-e085-4808-94bc-b33895089a74         2   \nb378b3ba-71e9-418b-9697-ecdff6f62724         2   \n8bda6be7-e90e-46d0-8577-693a27d8812b         1   \n...                                        ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b        20   \n4af05575-052d-40ff-9311-d578319e810a         4   \nb4899889-fbf4-43a5-a5ba-ae178fa07347         5   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0         1   \nadffb94b-6175-4612-8628-44e7c9baa620         0   \n\n                                                               revision_id  \\\nid                                                                           \nfd4d0f47-5a05-4716-861e-b0b328effe8b  1e037a30-3976-4f1e-b86c-3209201d3a58   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  7164a66f-53d5-43bc-aa08-0e2bbcda83b6   \nc926cfe2-e085-4808-94bc-b33895089a74  a36de255-fd63-4469-97f1-70340dbb9aaa   \nb378b3ba-71e9-418b-9697-ecdff6f62724  803276bf-0a92-4028-89e6-c7c2aa6a3c54   \n8bda6be7-e90e-46d0-8577-693a27d8812b  3bc21588-61f8-4f71-a2d2-26372c44eeca   \n...                                                                    ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  3bb798bb-f087-4d80-b7dc-7ee60a37cf83   \n4af05575-052d-40ff-9311-d578319e810a  633648a8-bcdf-4ef5-88b0-005f0df7b2c6   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  e1b42481-07ca-46e6-be9b-f06f46a984a3   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  422d4204-1324-41cb-b8e9-2ca1be8ec838   \nadffb94b-6175-4612-8628-44e7c9baa620  5cc1e0e7-9170-47f5-aa69-08f14234e985   \n\n                                      hash   state  \\\nid                                                   \nfd4d0f47-5a05-4716-861e-b0b328effe8b   NaN  active   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc   NaN  active   \nc926cfe2-e085-4808-94bc-b33895089a74   NaN  active   \nb378b3ba-71e9-418b-9697-ecdff6f62724   NaN  active   \n8bda6be7-e90e-46d0-8577-693a27d8812b   NaN  active   \n...                                    ...     ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b   NaN  active   \n4af05575-052d-40ff-9311-d578319e810a   NaN  active   \nb4899889-fbf4-43a5-a5ba-ae178fa07347   NaN  active   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0   NaN  active   \nadffb94b-6175-4612-8628-44e7c9baa620   NaN  active   \n\n                                                           extras  \\\nid                                                                  \nfd4d0f47-5a05-4716-861e-b0b328effe8b  {\"datastore_active\": false}   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc   {\"datastore_active\": true}   \nc926cfe2-e085-4808-94bc-b33895089a74  {\"datastore_active\": false}   \nb378b3ba-71e9-418b-9697-ecdff6f62724  {\"datastore_active\": false}   \n8bda6be7-e90e-46d0-8577-693a27d8812b  {\"datastore_active\": false}   \n...                                                           ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  {\"datastore_active\": false}   \n4af05575-052d-40ff-9311-d578319e810a   {\"datastore_active\": true}   \nb4899889-fbf4-43a5-a5ba-ae178fa07347   {\"datastore_active\": true}   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0   {\"datastore_active\": true}   \nadffb94b-6175-4612-8628-44e7c9baa620   {\"datastore_active\": true}   \n\n                                                                                   name  \\\nid                                                                                        \nfd4d0f47-5a05-4716-861e-b0b328effe8b                                            GeoJSON   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc                 Public Works - Block Party Summary   \nc926cfe2-e085-4808-94bc-b33895089a74                                            GeoJSON   \nb378b3ba-71e9-418b-9697-ecdff6f62724                         Make My Trip Count Website   \n8bda6be7-e90e-46d0-8577-693a27d8812b               City of Pittsburgh Parcel Boundaries   \n...                                                                                 ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  Raw tax-lien records (beta) [API-only version]...   \n4af05575-052d-40ff-9311-d578319e810a                              Archive Sales Results   \nb4899889-fbf4-43a5-a5ba-ae178fa07347                                   Archive Bid List   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0                                   Current Bid List   \nadffb94b-6175-4612-8628-44e7c9baa620                              Current Sales Results   \n\n                                     resource_type  ...  \\\nid                                                  ...   \nfd4d0f47-5a05-4716-861e-b0b328effe8b           NaN  ...   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc           NaN  ...   \nc926cfe2-e085-4808-94bc-b33895089a74           NaN  ...   \nb378b3ba-71e9-418b-9697-ecdff6f62724           NaN  ...   \n8bda6be7-e90e-46d0-8577-693a27d8812b           NaN  ...   \n...                                            ...  ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b           NaN  ...   \n4af05575-052d-40ff-9311-d578319e810a           NaN  ...   \nb4899889-fbf4-43a5-a5ba-ae178fa07347           NaN  ...   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0           NaN  ...   \nadffb94b-6175-4612-8628-44e7c9baa620           NaN  ...   \n\n                                                     author_email  maintainer  \\\nid                                                                              \nfd4d0f47-5a05-4716-861e-b0b328effe8b                          NaN         NaN   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  lee.haller@pittsburghpa.gov  Koury Lape   \nc926cfe2-e085-4808-94bc-b33895089a74                          NaN         NaN   \nb378b3ba-71e9-418b-9697-ecdff6f62724                          NaN         NaN   \n8bda6be7-e90e-46d0-8577-693a27d8812b                          NaN         NaN   \n...                                                           ...         ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b                          NaN         NaN   \n4af05575-052d-40ff-9311-d578319e810a                          NaN         NaN   \nb4899889-fbf4-43a5-a5ba-ae178fa07347                          NaN         NaN   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0                          NaN         NaN   \nadffb94b-6175-4612-8628-44e7c9baa620                          NaN         NaN   \n\n                                         maintainer_email state_package  \\\nid                                                                        \nfd4d0f47-5a05-4716-861e-b0b328effe8b                  NaN        active   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  kourylape@gmail.com        active   \nc926cfe2-e085-4808-94bc-b33895089a74                  NaN        active   \nb378b3ba-71e9-418b-9697-ecdff6f62724                  NaN        active   \n8bda6be7-e90e-46d0-8577-693a27d8812b                  NaN        active   \n...                                                   ...           ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b                  NaN        active   \n4af05575-052d-40ff-9311-d578319e810a                  NaN        active   \nb4899889-fbf4-43a5-a5ba-ae178fa07347                  NaN        active   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0                  NaN        active   \nadffb94b-6175-4612-8628-44e7c9baa620                  NaN        active   \n\n                                         type  \\\nid                                              \nfd4d0f47-5a05-4716-861e-b0b328effe8b  dataset   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  dataset   \nc926cfe2-e085-4808-94bc-b33895089a74  dataset   \nb378b3ba-71e9-418b-9697-ecdff6f62724  dataset   \n8bda6be7-e90e-46d0-8577-693a27d8812b  dataset   \n...                                       ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  dataset   \n4af05575-052d-40ff-9311-d578319e810a  dataset   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  dataset   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  dataset   \nadffb94b-6175-4612-8628-44e7c9baa620  dataset   \n\n                                                                 owner_org  \\\nid                                                                           \nfd4d0f47-5a05-4716-861e-b0b328effe8b  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  9ecaff80-fb4a-457b-8141-e53f7c991890   \nc926cfe2-e085-4808-94bc-b33895089a74  9ecaff80-fb4a-457b-8141-e53f7c991890   \nb378b3ba-71e9-418b-9697-ecdff6f62724  e35882ba-af3c-4e57-8204-6dea559678a0   \n8bda6be7-e90e-46d0-8577-693a27d8812b  4eb88a01-738a-4ca1-96e4-b0bf8d3ec3dc   \n...                                                                    ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n4af05575-052d-40ff-9311-d578319e810a  f6dcf8fb-2051-4226-8c74-de9fbc699684   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  f6dcf8fb-2051-4226-8c74-de9fbc699684   \nadffb94b-6175-4612-8628-44e7c9baa620  f6dcf8fb-2051-4226-8c74-de9fbc699684   \n\n                                      private_package  \\\nid                                                      \nfd4d0f47-5a05-4716-861e-b0b328effe8b                f   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc                f   \nc926cfe2-e085-4808-94bc-b33895089a74                f   \nb378b3ba-71e9-418b-9697-ecdff6f62724                f   \n8bda6be7-e90e-46d0-8577-693a27d8812b                f   \n...                                               ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b                f   \n4af05575-052d-40ff-9311-d578319e810a                f   \nb4899889-fbf4-43a5-a5ba-ae178fa07347                f   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0                f   \nadffb94b-6175-4612-8628-44e7c9baa620                f   \n\n                                               metadata_modified  \\\nid                                                                 \nfd4d0f47-5a05-4716-861e-b0b328effe8b   2023-04-08 17:27:58.59801   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  2021-02-04 17:47:03.968772   \nc926cfe2-e085-4808-94bc-b33895089a74  2023-03-29 12:02:07.901705   \nb378b3ba-71e9-418b-9697-ecdff6f62724  2021-02-04 01:55:54.679227   \n8bda6be7-e90e-46d0-8577-693a27d8812b  2021-02-04 01:55:34.808311   \n...                                                          ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b   2023-04-02 23:06:11.78898   \n4af05575-052d-40ff-9311-d578319e810a  2022-12-16 09:15:32.900946   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  2022-12-16 09:15:32.900946   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  2022-12-16 09:15:32.900946   \nadffb94b-6175-4612-8628-44e7c9baa620  2022-12-16 09:15:32.900946   \n\n                                                           creator_user_id  \\\nid                                                                           \nfd4d0f47-5a05-4716-861e-b0b328effe8b  90e6b24a-729c-43ea-9a24-2e060d24d07d   \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  0b3083b1-b5ef-4dbb-b23c-58e623476698   \nc926cfe2-e085-4808-94bc-b33895089a74  90e6b24a-729c-43ea-9a24-2e060d24d07d   \nb378b3ba-71e9-418b-9697-ecdff6f62724  9e43a726-2b62-4205-8f4c-aca5d8521b78   \n8bda6be7-e90e-46d0-8577-693a27d8812b  f40ecfea-0195-4903-8b7d-cc99f25d7d8c   \n...                                                                    ...   \nfba9b0ad-b572-46c7-9f08-4119435f913b  ed442faf-d3dc-48fb-8964-133201905237   \n4af05575-052d-40ff-9311-d578319e810a  e7f3f84d-571d-45a8-a84a-b79ce88bc083   \nb4899889-fbf4-43a5-a5ba-ae178fa07347  e7f3f84d-571d-45a8-a84a-b79ce88bc083   \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  e7f3f84d-571d-45a8-a84a-b79ce88bc083   \nadffb94b-6175-4612-8628-44e7c9baa620  e7f3f84d-571d-45a8-a84a-b79ce88bc083   \n\n                                                metadata_created  \nid                                                                \nfd4d0f47-5a05-4716-861e-b0b328effe8b  2017-05-23 14:32:40.593774  \n4ae88ba4-f535-44d7-9a96-08c6f38a34fc  2015-04-14 18:16:01.072372  \nc926cfe2-e085-4808-94bc-b33895089a74  2021-03-09 21:02:18.724974  \nb378b3ba-71e9-418b-9697-ecdff6f62724  2017-01-06 19:47:17.247589  \n8bda6be7-e90e-46d0-8577-693a27d8812b  2016-10-07 17:49:36.820423  \n...                                                          ...  \nfba9b0ad-b572-46c7-9f08-4119435f913b  2016-04-05 15:40:27.020076  \n4af05575-052d-40ff-9311-d578319e810a  2015-07-22 19:23:23.399359  \nb4899889-fbf4-43a5-a5ba-ae178fa07347  2015-07-22 19:23:23.399359  \n21b22189-2aa4-419f-a9e9-c518bbe25fe0  2015-07-22 19:23:23.399359  \nadffb94b-6175-4612-8628-44e7c9baa620  2015-07-22 19:23:23.399359  \n\n[2217 rows x 50 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>url</th>\n      <th>format</th>\n      <th>description</th>\n      <th>position</th>\n      <th>revision_id</th>\n      <th>hash</th>\n      <th>state</th>\n      <th>extras</th>\n      <th>name</th>\n      <th>resource_type</th>\n      <th>...</th>\n      <th>author_email</th>\n      <th>maintainer</th>\n      <th>maintainer_email</th>\n      <th>state_package</th>\n      <th>type</th>\n      <th>owner_org</th>\n      <th>private_package</th>\n      <th>metadata_modified</th>\n      <th>creator_user_id</th>\n      <th>metadata_created</th>\n    </tr>\n    <tr>\n      <th>id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>fd4d0f47-5a05-4716-861e-b0b328effe8b</th>\n      <td>https://data.wprdc.org/dataset/e41c0a67-837f-4...</td>\n      <td>GeoJSON</td>\n      <td>NaN</td>\n      <td>2</td>\n      <td>1e037a30-3976-4f1e-b86c-3209201d3a58</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": false}</td>\n      <td>GeoJSON</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2023-04-08 17:27:58.59801</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2017-05-23 14:32:40.593774</td>\n    </tr>\n    <tr>\n      <th>4ae88ba4-f535-44d7-9a96-08c6f38a34fc</th>\n      <td>https://data.wprdc.org/dataset/cfaffd5c-ab5a-4...</td>\n      <td>CSV</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>7164a66f-53d5-43bc-aa08-0e2bbcda83b6</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": true}</td>\n      <td>Public Works - Block Party Summary</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>lee.haller@pittsburghpa.gov</td>\n      <td>Koury Lape</td>\n      <td>kourylape@gmail.com</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2021-02-04 17:47:03.968772</td>\n      <td>0b3083b1-b5ef-4dbb-b23c-58e623476698</td>\n      <td>2015-04-14 18:16:01.072372</td>\n    </tr>\n    <tr>\n      <th>c926cfe2-e085-4808-94bc-b33895089a74</th>\n      <td>https://data.wprdc.org/dataset/8f92ae09-4cfa-4...</td>\n      <td>GeoJSON</td>\n      <td>NaN</td>\n      <td>2</td>\n      <td>a36de255-fd63-4469-97f1-70340dbb9aaa</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": false}</td>\n      <td>GeoJSON</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>9ecaff80-fb4a-457b-8141-e53f7c991890</td>\n      <td>f</td>\n      <td>2023-03-29 12:02:07.901705</td>\n      <td>90e6b24a-729c-43ea-9a24-2e060d24d07d</td>\n      <td>2021-03-09 21:02:18.724974</td>\n    </tr>\n    <tr>\n      <th>b378b3ba-71e9-418b-9697-ecdff6f62724</th>\n      <td>http://www.makemytripcount.org/</td>\n      <td>HTML</td>\n      <td>Make My Trip Count Website</td>\n      <td>2</td>\n      <td>803276bf-0a92-4028-89e6-c7c2aa6a3c54</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": false}</td>\n      <td>Make My Trip Count Website</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>e35882ba-af3c-4e57-8204-6dea559678a0</td>\n      <td>f</td>\n      <td>2021-02-04 01:55:54.679227</td>\n      <td>9e43a726-2b62-4205-8f4c-aca5d8521b78</td>\n      <td>2017-01-06 19:47:17.247589</td>\n    </tr>\n    <tr>\n      <th>8bda6be7-e90e-46d0-8577-693a27d8812b</th>\n      <td>https://data.wprdc.org/dataset/6bb2a968-761d-4...</td>\n      <td>ZIP</td>\n      <td>NaN</td>\n      <td>1</td>\n      <td>3bc21588-61f8-4f71-a2d2-26372c44eeca</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": false}</td>\n      <td>City of Pittsburgh Parcel Boundaries</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>4eb88a01-738a-4ca1-96e4-b0bf8d3ec3dc</td>\n      <td>f</td>\n      <td>2021-02-04 01:55:34.808311</td>\n      <td>f40ecfea-0195-4903-8b7d-cc99f25d7d8c</td>\n      <td>2016-10-07 17:49:36.820423</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>fba9b0ad-b572-46c7-9f08-4119435f913b</th>\n      <td>https://data.wprdc.org/dataset/22fe57da-f5b8-4...</td>\n      <td>ZIP</td>\n      <td>This is a compressed CSV file version of the d...</td>\n      <td>20</td>\n      <td>3bb798bb-f087-4d80-b7dc-7ee60a37cf83</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": false}</td>\n      <td>Raw tax-lien records (beta) [API-only version]...</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2023-04-02 23:06:11.78898</td>\n      <td>ed442faf-d3dc-48fb-8964-133201905237</td>\n      <td>2016-04-05 15:40:27.020076</td>\n    </tr>\n    <tr>\n      <th>4af05575-052d-40ff-9311-d578319e810a</th>\n      <td>https://data.wprdc.org/datastore/dump/4af05575...</td>\n      <td>CSV</td>\n      <td>NaN</td>\n      <td>4</td>\n      <td>633648a8-bcdf-4ef5-88b0-005f0df7b2c6</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": true}</td>\n      <td>Archive Sales Results</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2022-12-16 09:15:32.900946</td>\n      <td>e7f3f84d-571d-45a8-a84a-b79ce88bc083</td>\n      <td>2015-07-22 19:23:23.399359</td>\n    </tr>\n    <tr>\n      <th>b4899889-fbf4-43a5-a5ba-ae178fa07347</th>\n      <td>https://data.wprdc.org/datastore/dump/b4899889...</td>\n      <td>CSV</td>\n      <td>NaN</td>\n      <td>5</td>\n      <td>e1b42481-07ca-46e6-be9b-f06f46a984a3</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": true}</td>\n      <td>Archive Bid List</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2022-12-16 09:15:32.900946</td>\n      <td>e7f3f84d-571d-45a8-a84a-b79ce88bc083</td>\n      <td>2015-07-22 19:23:23.399359</td>\n    </tr>\n    <tr>\n      <th>21b22189-2aa4-419f-a9e9-c518bbe25fe0</th>\n      <td>https://data.wprdc.org/datastore/dump/21b22189...</td>\n      <td>CSV</td>\n      <td>NaN</td>\n      <td>1</td>\n      <td>422d4204-1324-41cb-b8e9-2ca1be8ec838</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": true}</td>\n      <td>Current Bid List</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2022-12-16 09:15:32.900946</td>\n      <td>e7f3f84d-571d-45a8-a84a-b79ce88bc083</td>\n      <td>2015-07-22 19:23:23.399359</td>\n    </tr>\n    <tr>\n      <th>adffb94b-6175-4612-8628-44e7c9baa620</th>\n      <td>https://data.wprdc.org/datastore/dump/adffb94b...</td>\n      <td>CSV</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>5cc1e0e7-9170-47f5-aa69-08f14234e985</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>{\"datastore_active\": true}</td>\n      <td>Current Sales Results</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>active</td>\n      <td>dataset</td>\n      <td>f6dcf8fb-2051-4226-8c74-de9fbc699684</td>\n      <td>f</td>\n      <td>2022-12-16 09:15:32.900946</td>\n      <td>e7f3f84d-571d-45a8-a84a-b79ce88bc083</td>\n      <td>2015-07-22 19:23:23.399359</td>\n    </tr>\n  </tbody>\n</table>\n<p>2217 rows × 50 columns</p>\n</div>"
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size_comparison_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Compare Table Counts"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [],
   "source": [
    "old_resource_sizes_df['group'] = old_resource_sizes_df.index.str[0]\n",
    "new_resource_rows_df['group'] = new_resource_rows_df.index.str[0]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "data": {
      "text/plain": "       pg_size_pretty  pg_relation_size\ngroup                                  \n(                   0                 0\n0                 191               191\n1                 168               168\n2                 196               196\n3                 181               181\n4                 206               206\n5                 185               185\n6                 203               203\n7                 215               215\n8                 188               188\n9                 193               193\nC                   1                 1\n_                   1                 1\na                 187               187\nb                 172               172\nc                 188               188\nd                 194               194\ne                 198               198\nf                 203               203\ng                   2                 2\nr                   2                 2\ns                   1                 1\nt                   1                 1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>pg_size_pretty</th>\n      <th>pg_relation_size</th>\n    </tr>\n    <tr>\n      <th>group</th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>(</th>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>191</td>\n      <td>191</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>168</td>\n      <td>168</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>196</td>\n      <td>196</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>181</td>\n      <td>181</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>206</td>\n      <td>206</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>185</td>\n      <td>185</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>203</td>\n      <td>203</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>215</td>\n      <td>215</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>188</td>\n      <td>188</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>193</td>\n      <td>193</td>\n    </tr>\n    <tr>\n      <th>C</th>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>_</th>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>a</th>\n      <td>187</td>\n      <td>187</td>\n    </tr>\n    <tr>\n      <th>b</th>\n      <td>172</td>\n      <td>172</td>\n    </tr>\n    <tr>\n      <th>c</th>\n      <td>188</td>\n      <td>188</td>\n    </tr>\n    <tr>\n      <th>d</th>\n      <td>194</td>\n      <td>194</td>\n    </tr>\n    <tr>\n      <th>e</th>\n      <td>198</td>\n      <td>198</td>\n    </tr>\n    <tr>\n      <th>f</th>\n      <td>203</td>\n      <td>203</td>\n    </tr>\n    <tr>\n      <th>g</th>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>r</th>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>s</th>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>t</th>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_resource_sizes_df.groupby('group').count()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "data": {
      "text/plain": "       new_tuple_count\ngroup                 \n0                  191\n1                  168\n2                  195\n3                  180\n4                  206\n5                  185\n6                  203\n7                  214\n8                  188\n9                  192\na                  187\nb                  170\nc                  188\nd                  193\ne                  197\nf                  203\ns                    1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>new_tuple_count</th>\n    </tr>\n    <tr>\n      <th>group</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>191</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>168</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>195</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>180</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>206</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>185</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>203</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>214</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>188</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>192</td>\n    </tr>\n    <tr>\n      <th>a</th>\n      <td>187</td>\n    </tr>\n    <tr>\n      <th>b</th>\n      <td>170</td>\n    </tr>\n    <tr>\n      <th>c</th>\n      <td>188</td>\n    </tr>\n    <tr>\n      <th>d</th>\n      <td>193</td>\n    </tr>\n    <tr>\n      <th>e</th>\n      <td>197</td>\n    </tr>\n    <tr>\n      <th>f</th>\n      <td>203</td>\n    </tr>\n    <tr>\n      <th>s</th>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_resource_rows_df.groupby('group').count()\n"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}