unzip working area only and cleanup after

osmand_osm/osm/processing.py

@@ -102,7 +102,10 @@ def update_oa(token):
     output: none
     '''
     run(['wget', '--backups=1', '--header', 'Authorization: Bearer ' + token, 'https://batch.openaddresses.io/api/collections/1/data'])
-    run(['unzip', '-o', 'data'])
+
+def decompress_oa(working_area):
+    run(['unzip', '-o', 'data',working_area.directory.as_posix() + '/*'])
+
 
 def pg2osm(source, id_start, working_area, db_name):
     '''
@@ -193,6 +196,7 @@ def load_oa(working_area, db_name):
             run(f'ogr2ogr PG:dbname={db_name} {source.path} -nln {source.table_temp} -overwrite -lco OVERWRITE=YES', shell=True, capture_output=True, check=True, encoding='utf8')
         except CalledProcessError as error:
             logging.warning(working_area.name + ' ' + error.stderr)
+        run(['rm', source.path.as_posix()])
     logging.info(working_area.name + ' ' + 'Load Finished')
 
 def filter_data(working_area, db_name):
@@ -487,8 +491,9 @@ def run_all(area, args):
     pbf_output = root.parent
     working_area = WorkingArea(area)
     logging.debug(working_area)
-    create_master_list(working_area)
     if args.load_oa:
+        decompress_oa(working_area)
+        create_master_list(working_area)
         load_oa(working_area, db_name)
     if args.filter_data:
         filter_data(working_area, db_name)

osmand_osm/osm/tests.py

@@ -42,39 +42,19 @@ def test_create_master_list(self):
         processing.create_master_list(working_area)
         self.assertEqual(1, len(working_area.master_list))
 
-    def test_load_oa_first_run(self):
-        # cleanup postgres table
-        self.cur.execute('drop table if exists aa_load_oa_addresses_city')
-        self.conn.commit()
-        working_area = processing.WorkingArea('aa')
-        working_area.master_list = [processing.Source(Path('aa/load-oa-addresses-city.geojson'))]
-        processing.load_oa(working_area, 'gis')
-        self.cur.execute('select * from aa_load_oa_addresses_city_temp')
-        data = self.cur.fetchall()
-        # check for street
-        self.assertRegex(data[0][4],'Di Mario Dr')
-        # check for number
-        self.assertRegex(data[0][3],'1')
-
     def test_load_oa(self):
-        '''
-        ensure overwriting table works
-        '''
         # cleanup postgres table
-        self.cur.execute('drop table if exists aa_load_oa_addresses_city')
-        self.conn.commit()
-        # load data into postgres
-        self.cur.execute("create table aa_load_oa_addresses_city (ogc_fid integer NOT NULL, \
-                id character varying, number character varying, street character varying, \
-                city character varying, district character varying, region character varying, \
-                postcode character varying, hash character varying, wkb_geometry public.geometry(Point, 4326));")
-        self.cur.execute("insert into aa_load_oa_addresses_city (ogc_fid, number, street) values (%s, %s, %s)", (3, '2', 'Luigi Dr'))
-        #cself.cur.execute("CREATE INDEX aa_load_oa_addresses_city_temp_wkb_geometry_geom_idx ON public.aa_load_oa_addresses_city_temp USING gist (wkb_geometry)")
         self.cur.execute('drop table if exists aa_load_oa_addresses_city_temp')
         self.conn.commit()
         working_area = processing.WorkingArea('aa')
         working_area.master_list = [processing.Source(Path('aa/load-oa-addresses-city.geojson'))]
+        run(['cp', 'aa/load-oa-addresses-city.geojson', 'aa/load-oa-addresses-city.geojson.bak'])
+        run(['mv', 'data', 'data.bak'])
+        run(['cp', '-n', 'aa/data.zip', 'data'])
+        processing.decompress_oa(working_area)
         processing.load_oa(working_area, 'gis')
+        run(['mv', 'aa/load-oa-addresses-city.geojson.bak', 'aa/load-oa-addresses-city.geojson'])
+        run(['mv', 'data.bak', 'data'])
         self.cur.execute('select * from aa_load_oa_addresses_city_temp')
         data = self.cur.fetchall()
         # check for street