Merge, support unix filename pattern matching on zip archive

Author: frodrigo

analysers/Analyser_Merge.py

@@ -34,6 +34,7 @@
 import tempfile
 import json
 import re
+import fnmatch
 from typing import Optional, Dict, Union, Callable
 from collections import defaultdict
 from .Analyser_Osmosis import Analyser_Osmosis
@@ -322,7 +323,7 @@ def __init__(self, attribution = None, millesime = None, encoding = "utf-8", fil
         @param file: file name in storage
         @param urlFile: remote URL of source file
         @param fileUrlCache: days for file in cache
-        @param zip: extract file from zip
+        @param zip: extract a file from zip. Unix filename pattern matching.
         @param extract: extract file from any archive format
         @param gzip: uncompress from bz2
         @param gzip: uncompress from gzip
@@ -350,13 +351,23 @@ def __init__(self, attribution = None, millesime = None, encoding = "utf-8", fil
         if self.attribution and "{0}" in self.attribution:
             self.attribution_re = re.compile(self.attribution.replace("{0}", ".*"))
 
+    def zipFile(self):
+        if self.file:
+            f = open(self.file, 'rb')
+        elif self.fileUrl:
+            f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
+
+        z = zipfile.ZipFile(f, 'r')
+        print(z.namelist())
+        filename = next(filter(lambda zipinfo: fnmatch.fnmatch(zipinfo.filename, self.zip), z.infolist()))
+        return filename
+
     def time(self):
         if self.file:
             return int(os.path.getmtime(self.file)+.5)
         elif self.fileUrl:
-            if self.zip:
-                f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
-                date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
+            if self.zipFile():
+                date_time = self.zipFile().date_time
                 return int(time.mktime(date_time + (0, 0, -1))+.5)
             else:
                 return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
@@ -374,8 +385,8 @@ def open(self, binary = False):
         elif self.fileUrl:
             f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
 
-        if self.zip:
-            z = zipfile.ZipFile(f, 'r').open(self.zip)
+        if self.zipFile():
+            z = zipfile.ZipFile(f, 'r').open(self.zipFile().filename)
             f = io.BytesIO(z.read())
             f.seek(0)
         elif self.extract:
@@ -689,7 +700,7 @@ def import_(self, table, srid, osmosis):
             self.source.encoding,
             srid,
             tmp_file.name,
-            self.source.zip,
+            self.source.zipFile().filename,
             table,
             tmp_file.name
         )

analysers/analyser_merge_public_transport_FR_idfm.py

@@ -38,7 +38,7 @@ def __init__(self, config, logger, clas, conflationDistance, select, osmTags, de
             "Référentiel des arrêts : fichiers SIG",
             SHP(Source(attribution = 'Île-de-France Mobilités', millesime = '03/2022',
                     fileUrl = 'https://eu.ftp.opendatasoft.com/stif/Reflex/REF_ZDE.zip',
-                    zip = 'PT_ZDE_R_02_03_2022.shp')),
+                    zip = '*.shp')),
             LoadGeomCentroid(srid = 2154,
                 select = {"type_arret": select}),
             Conflate(

doc/4-Merge.md

@@ -138,7 +138,7 @@ Remote files are fetched and saved in the Osmose-QA Backend cache, the delay can
 The remote file could be compressed or an archive:
 - `bz2` (boolean): the content is compressed in BZip2 format.
 - `gzip` (boolean): the content is compressed in GZip format.
-- `zip`: the remote URL is a ZIP archive and the data is at this path inside the archive.
+- `zip`: the remote URL is a ZIP archive and the data is at this path inside the archive. Path can use unix filename pattern matching.
 - `extract`: same as `zip`, but for all archive formats.
 
 Assuming the resource is a text file: