Merge remote-tracking branch 'frodrigo/master'

* frodrigo/master:
  Merge, support unix filename pattern matching on zip archive
  Disable FR_CAPP merge, no more data source
  Disable analyser_merge_tmc_point_FR, no more data source
  Data schema update in analyser_merge_post_box_FR (changing column names every month, that fine)
  Update data source in analyser_merge_healthcare_FR_finess
  Use std source function in analyser_merge_geodesie_support_FR

Author: jocelynj

analysers/Analyser_Merge.py

@@ -34,6 +34,7 @@
 import tempfile
 import json
 import re
+import fnmatch
 from typing import Optional, Dict, Union, Callable
 from collections import defaultdict
 from .Analyser_Osmosis import Analyser_Osmosis
@@ -322,7 +323,7 @@ def __init__(self, attribution = None, millesime = None, encoding = "utf-8", fil
         @param file: file name in storage
         @param urlFile: remote URL of source file
         @param fileUrlCache: days for file in cache
-        @param zip: extract file from zip
+        @param zip: extract a file from zip. Unix filename pattern matching.
         @param extract: extract file from any archive format
         @param gzip: uncompress from bz2
         @param gzip: uncompress from gzip
@@ -350,13 +351,23 @@ def __init__(self, attribution = None, millesime = None, encoding = "utf-8", fil
         if self.attribution and "{0}" in self.attribution:
             self.attribution_re = re.compile(self.attribution.replace("{0}", ".*"))
 
+    def zipFile(self):
+        if self.file:
+            f = open(self.file, 'rb')
+        elif self.fileUrl:
+            f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
+
+        z = zipfile.ZipFile(f, 'r')
+        print(z.namelist())
+        filename = next(filter(lambda zipinfo: fnmatch.fnmatch(zipinfo.filename, self.zip), z.infolist()))
+        return filename
+
     def time(self):
         if self.file:
             return int(os.path.getmtime(self.file)+.5)
         elif self.fileUrl:
-            if self.zip:
-                f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
-                date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
+            if self.zipFile():
+                date_time = self.zipFile().date_time
                 return int(time.mktime(date_time + (0, 0, -1))+.5)
             else:
                 return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
@@ -374,8 +385,8 @@ def open(self, binary = False):
         elif self.fileUrl:
             f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
 
-        if self.zip:
-            z = zipfile.ZipFile(f, 'r').open(self.zip)
+        if self.zipFile():
+            z = zipfile.ZipFile(f, 'r').open(self.zipFile().filename)
             f = io.BytesIO(z.read())
             f.seek(0)
         elif self.extract:
@@ -689,7 +700,7 @@ def import_(self, table, srid, osmosis):
             self.source.encoding,
             srid,
             tmp_file.name,
-            self.source.zip,
+            self.source.zipFile().filename,
             table,
             tmp_file.name
         )

analysers/analyser_merge_geodesie_support_FR.py

@@ -72,5 +72,5 @@ def __init__(self, config, error_file, logger, item, classs, level, topic, osmTa
                 conflationDistance = 200,
                 mapping = Mapping(
                     static1 = dict(dict(**osmTags), **defaultTags),
-                    static2 = {"source": lambda a: a.parser.source.attribution},
+                    static2 = {"source": self.source},
                 text = lambda tags, fields: {"en": fields["description"]} )))

analysers/analyser_merge_healthcare_FR_finess.py

@@ -78,7 +78,7 @@ def __init__(self, config, error_file, logger, srid, is_in, categories, items, m
                 attribution="Le ministère des solidarités et de la santé",
                 encoding="ISO-8859-1",
                 dataset="53699569a3a729239d2046eb",
-                resource="2ce43ade-8d2c-4d1d-81da-ca06c82abc68")),
+                resource="98f3161f-79ff-4f16-8f6a-6d571a80fea2")),
             Load("coordxet", "coordyet", srid = srid,
                 select = {"categetab": categories},
                 where = lambda res: is_in(res["departement"])),

analysers/analyser_merge_post_box_FR.py

@@ -54,5 +54,5 @@ def __init__(self, config, logger = None):
                         "amenity": "post_box",
                         "operator": "La Poste"},
                     static2 = {"source": self.source},
-                    mapping1 = {"ref": "CO_EXT"},
-                text = lambda tags, fields: {"en": ", ".join(filter(lambda x: x, [fields["NUM_VOIE"], fields["LB_CPL_NUM_ADR"], fields["LB_VOIE"], fields["CO_POS"], fields["LB_CMN"]]))} )))
+                    mapping1 = {"ref": "CO_MUP"},
+                text = lambda tags, fields: {"en": ", ".join(filter(lambda x: x, [fields["VA_NO_VOIE"], fields["LB_EXTENSION"], fields["LB_VOIE_EXT"], fields["CO_POSTAL"], fields["LB_COM"]]))} )))

analysers/analyser_merge_public_transport_FR_idfm.py

@@ -38,7 +38,7 @@ def __init__(self, config, logger, clas, conflationDistance, select, osmTags, de
             "Référentiel des arrêts : fichiers SIG",
             SHP(Source(attribution = 'Île-de-France Mobilités', millesime = '03/2022',
                     fileUrl = 'https://eu.ftp.opendatasoft.com/stif/Reflex/REF_ZDE.zip',
-                    zip = 'PT_ZDE_R_02_03_2022.shp')),
+                    zip = '*.shp')),
             LoadGeomCentroid(srid = 2154,
                 select = {"type_arret": select}),
             Conflate(