From 20b283052c3934be08166acb6ccf32e2d8e583ae Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Fri, 5 Aug 2022 16:11:29 +0200 Subject: [PATCH 01/10] wxGUI/gui_core: fix import Rasterlite DB raster --- gui/wxpython/gui_core/gselect.py | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index aa15d48d2d8..a82d942a39c 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2370,6 +2370,17 @@ def getProjMatchCaption(projectionMatch): ) data.append((layerId, raster, int(projectionMatch), grassName)) layerId += 1 + elif self.dbWidgets["format"].GetStringSelection() == "Rasterlite": + rasters = self.GetRasterliteDBRasters(dsn) + for raster in rasters: + grassName = GetValidLayerName(raster) + projectionMatch = hasRastSameProjAsLocation(dsn) + projectionMatchCaption = getProjMatchCaption(projectionMatch) + listData.append( + (layerId, raster, projectionMatchCaption, grassName) + ) + data.append((layerId, raster, int(projectionMatch), grassName)) + layerId += 1 # emit signal self.reloadDataRequired.emit(listData=listData, data=data) @@ -2652,6 +2663,75 @@ def _getPGDBRasters(self, dsn): Debug.msg(3, f"GdalSelect._getPGDBRasters(): return {rasters}") return rasters + def GetRasterliteDBRasters(self, dsn): + """Get Rasterlite DB rasters + + :param str dsn: Rasterlite DB data source name + + :return list: list of Rasterlite DB rasters + """ + # For each raster, there are 2 corresponding SQLite tables, + # suffixed with _rasters and _metadata. + raster_tables_suffix = { + "table_suffix1": "_rasters", + "table_suffix2": "_metadata", + } + return grass.parse_command( + "db.select", + flags="c", + sql=( + "SELECT name FROM sqlite_schema WHERE type ='table'" + " AND name NOT LIKE 'sqlite_%'" + f" AND name LIKE '%{raster_tables_suffix['table_suffix1']}'" + f" OR name LIKE '%{raster_tables_suffix['table_suffix2']}';" + ), + database=dsn, + parse=( + self.RasterliteDBRastersParser, + {"raster_tables_suffix": raster_tables_suffix}, + ), + ) + + def RasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): + """Rasterlite DB raster tables names parser + + :param str raster_tables: raster tables names (output of the + db.select module) + :param dict raster_tables_suffix: Rasterlite DB raster table + suffixes, for each raster + there are 2 corresponding SQLite + tables, suffixed with _rasters + and _metadata + + :return list: list of Rasterlite raster names + """ + import re + + raster_table_sep = "," + raster_tables_count = {} + table_names_without_suffix = re.sub( + rf"{raster_tables_suffix['table_suffix1']}{os.linesep}" + f"|{raster_tables_suffix['table_suffix2']}{os.linesep}", + raster_table_sep, + raster_tables, + ) + if table_names_without_suffix: + from collections import Counter + + # [:-1] filter out last separator + raster_tables_count = Counter( + table_names_without_suffix[:-1].split(raster_table_sep) + ) + # Filter out other tables which is not raster tables (has only + # one of table suffixed with _rasters or _metadata + other_tables = [ + i for i in raster_tables_count if raster_tables_count[i] == 1 + ] + if other_tables: + for table in other_tables: + del raster_tables_count[table] + return list(raster_tables_count.keys()) + class ProjSelect(wx.ComboBox): """Widget for selecting input raster/vector map used by From 9be46847aae215b4ec83ce625f68cc5ffaa0a3d4 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Fri, 5 Aug 2022 17:05:15 +0200 Subject: [PATCH 02/10] Make methods private --- gui/wxpython/gui_core/gselect.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index a82d942a39c..875769327a7 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2371,7 +2371,7 @@ def getProjMatchCaption(projectionMatch): data.append((layerId, raster, int(projectionMatch), grassName)) layerId += 1 elif self.dbWidgets["format"].GetStringSelection() == "Rasterlite": - rasters = self.GetRasterliteDBRasters(dsn) + rasters = self._getRasterliteDBRasters(dsn) for raster in rasters: grassName = GetValidLayerName(raster) projectionMatch = hasRastSameProjAsLocation(dsn) @@ -2663,7 +2663,7 @@ def _getPGDBRasters(self, dsn): Debug.msg(3, f"GdalSelect._getPGDBRasters(): return {rasters}") return rasters - def GetRasterliteDBRasters(self, dsn): + def _getRasterliteDBRasters(self, dsn): """Get Rasterlite DB rasters :param str dsn: Rasterlite DB data source name @@ -2687,12 +2687,12 @@ def GetRasterliteDBRasters(self, dsn): ), database=dsn, parse=( - self.RasterliteDBRastersParser, + self._rasterliteDBRastersParser, {"raster_tables_suffix": raster_tables_suffix}, ), ) - def RasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): + def _rasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): """Rasterlite DB raster tables names parser :param str raster_tables: raster tables names (output of the From a995328dac151124a125e49b4976c886fe949384 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sat, 6 Aug 2022 14:12:33 +0200 Subject: [PATCH 03/10] Remove the unnecessary condition --- gui/wxpython/gui_core/gselect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index 875769327a7..2ed7d6a0380 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2727,9 +2727,8 @@ def _rasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): other_tables = [ i for i in raster_tables_count if raster_tables_count[i] == 1 ] - if other_tables: - for table in other_tables: - del raster_tables_count[table] + for table in other_tables: + del raster_tables_count[table] return list(raster_tables_count.keys()) From f76ec52d22572f6bace321ee26dca704ca28dde1 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sun, 7 Aug 2022 09:49:10 +0200 Subject: [PATCH 04/10] Add additional check if raster table has raster BLOB column type --- gui/wxpython/gui_core/gselect.py | 58 ++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index 2ed7d6a0380..69c07ff751d 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2688,11 +2688,56 @@ def _getRasterliteDBRasters(self, dsn): database=dsn, parse=( self._rasterliteDBRastersParser, - {"raster_tables_suffix": raster_tables_suffix}, + { + "raster_tables_suffix": raster_tables_suffix, + "dsn": dsn, + }, ), ) - def _rasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): + def _checkRasterliteDBRasterBlobColumnExists( + self, + database, + tables, + raster_suffix, + ): + """Check if Rasterlite DB raster table has raster BLOB column + + :param str database: database path + :param list tables: raster tables + :param str raster_suffix: Rasterlite DB raster table suffix, + which is _rasters usually + + :return list other_tables: list of Rasterlite DB tables which + aren't rasters + """ + other_tables = [] + tabs_with_quotes = [f"'{t}'" for t in tables] + ret = grass.read_command( + "db.select", + flags="c", + sql=( + "SELECT sql FROM sqlite_master" + f" WHERE tbl_name IN ({', '.join(tabs_with_quotes)})" + " AND type = 'table'" + ), + database=database, + ) + for line in ret.split(os.linesep): + if "raster BLOB" not in line: + for table in tables: + if table in line: + other_tables.append( + table.replace(raster_suffix, ""), + ) + return other_tables + + def _rasterliteDBRastersParser( + self, + raster_tables, + raster_tables_suffix, + dsn, + ): """Rasterlite DB raster tables names parser :param str raster_tables: raster tables names (output of the @@ -2702,6 +2747,7 @@ def _rasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): there are 2 corresponding SQLite tables, suffixed with _rasters and _metadata + :param str dsn: Rasterlite DB data source name :return list: list of Rasterlite raster names """ @@ -2729,6 +2775,14 @@ def _rasterliteDBRastersParser(self, raster_tables, raster_tables_suffix): ] for table in other_tables: del raster_tables_count[table] + # Check if raster BLOB column type exists + raster_suffix = raster_tables_suffix["table_suffix1"] + for table in self._checkRasterliteDBRasterBlobColumnExists( + database=dsn, + tables=[f"{t}{raster_suffix}" for t in raster_tables_count], + raster_suffix=raster_suffix, + ): + del raster_tables_count[table] return list(raster_tables_count.keys()) From c0668a97ccad9129785252ed63404f8ec8bee3e3 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Mon, 19 Sep 2022 08:44:29 +0200 Subject: [PATCH 05/10] Set idsn var value if DB is Rasterlite --- gui/wxpython/modules/import_export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gui/wxpython/modules/import_export.py b/gui/wxpython/modules/import_export.py index 97a646037ed..72cddb5e011 100644 --- a/gui/wxpython/modules/import_export.py +++ b/gui/wxpython/modules/import_export.py @@ -470,6 +470,7 @@ def OnRun(self, event): if self.dsnInput.GetType() == "dir": idsn = os.path.join(dsn, layer) elif self.dsnInput.GetType() == "db": + idsn = dsn if "PG:" in dsn: idsn = f"{dsn} table={layer}" else: From 3ef684ebfec1945b0f2bab37c9a4c28cf17d599d Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Thu, 28 Sep 2023 16:05:13 +0200 Subject: [PATCH 06/10] Use gdalinfo for getting Rasterlite DB list of rasters --- gui/wxpython/gui_core/gselect.py | 129 ++++---------------------- gui/wxpython/modules/import_export.py | 22 +++++ 2 files changed, 39 insertions(+), 112 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index 69c07ff751d..99f3f54ba1e 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2670,120 +2670,25 @@ def _getRasterliteDBRasters(self, dsn): :return list: list of Rasterlite DB rasters """ - # For each raster, there are 2 corresponding SQLite tables, - # suffixed with _rasters and _metadata. - raster_tables_suffix = { - "table_suffix1": "_rasters", - "table_suffix2": "_metadata", - } - return grass.parse_command( - "db.select", - flags="c", - sql=( - "SELECT name FROM sqlite_schema WHERE type ='table'" - " AND name NOT LIKE 'sqlite_%'" - f" AND name LIKE '%{raster_tables_suffix['table_suffix1']}'" - f" OR name LIKE '%{raster_tables_suffix['table_suffix2']}';" - ), - database=dsn, - parse=( - self._rasterliteDBRastersParser, - { - "raster_tables_suffix": raster_tables_suffix, - "dsn": dsn, - }, - ), - ) - - def _checkRasterliteDBRasterBlobColumnExists( - self, - database, - tables, - raster_suffix, - ): - """Check if Rasterlite DB raster table has raster BLOB column - - :param str database: database path - :param list tables: raster tables - :param str raster_suffix: Rasterlite DB raster table suffix, - which is _rasters usually - - :return list other_tables: list of Rasterlite DB tables which - aren't rasters - """ - other_tables = [] - tabs_with_quotes = [f"'{t}'" for t in tables] - ret = grass.read_command( - "db.select", - flags="c", - sql=( - "SELECT sql FROM sqlite_master" - f" WHERE tbl_name IN ({', '.join(tabs_with_quotes)})" - " AND type = 'table'" - ), - database=database, - ) - for line in ret.split(os.linesep): - if "raster BLOB" not in line: - for table in tables: - if table in line: - other_tables.append( - table.replace(raster_suffix, ""), - ) - return other_tables - - def _rasterliteDBRastersParser( - self, - raster_tables, - raster_tables_suffix, - dsn, - ): - """Rasterlite DB raster tables names parser - - :param str raster_tables: raster tables names (output of the - db.select module) - :param dict raster_tables_suffix: Rasterlite DB raster table - suffixes, for each raster - there are 2 corresponding SQLite - tables, suffixed with _rasters - and _metadata - :param str dsn: Rasterlite DB data source name + import subprocess - :return list: list of Rasterlite raster names - """ - import re - - raster_table_sep = "," - raster_tables_count = {} - table_names_without_suffix = re.sub( - rf"{raster_tables_suffix['table_suffix1']}{os.linesep}" - f"|{raster_tables_suffix['table_suffix2']}{os.linesep}", - raster_table_sep, - raster_tables, + rasterlite_info = subprocess.run( + ["gdalinfo", "-json", dsn], + capture_output=True, ) - if table_names_without_suffix: - from collections import Counter - - # [:-1] filter out last separator - raster_tables_count = Counter( - table_names_without_suffix[:-1].split(raster_table_sep) - ) - # Filter out other tables which is not raster tables (has only - # one of table suffixed with _rasters or _metadata - other_tables = [ - i for i in raster_tables_count if raster_tables_count[i] == 1 - ] - for table in other_tables: - del raster_tables_count[table] - # Check if raster BLOB column type exists - raster_suffix = raster_tables_suffix["table_suffix1"] - for table in self._checkRasterliteDBRasterBlobColumnExists( - database=dsn, - tables=[f"{t}{raster_suffix}" for t in raster_tables_count], - raster_suffix=raster_suffix, - ): - del raster_tables_count[table] - return list(raster_tables_count.keys()) + if rasterlite_info.stderr: + GError(parent=self, message=grass.decode(rasterlite_info.stderr)) + return + if rasterlite_info.stdout: + import json + + rasterlite_info = json.loads(grass.decode(rasterlite_info.stdout)) + rasters = rasterlite_info["metadata"].get("SUBDATASETS") + if rasters: + return [ + v.rsplit("table=")[-1] for k, v in rasters.items() if "NAME" in k + ] + return [os.path.basename(rasterlite_info["files"][0]).rsplit(".")[0]] class ProjSelect(wx.ComboBox): diff --git a/gui/wxpython/modules/import_export.py b/gui/wxpython/modules/import_export.py index 72cddb5e011..9e62aab2567 100644 --- a/gui/wxpython/modules/import_export.py +++ b/gui/wxpython/modules/import_export.py @@ -473,6 +473,28 @@ def OnRun(self, event): idsn = dsn if "PG:" in dsn: idsn = f"{dsn} table={layer}" + elif os.path.exists(idsn): + import subprocess + + dataset_info = subprocess.run( + ["gdalinfo", "-json", dsn], + capture_output=True, + ) + if dataset_info.stderr: + GError( + parent=self, + message=grass.decode(dataset_info.stderr), + ) + return + if dataset_info.stdout: + import json + + dataset_info = json.loads( + grass.decode(dataset_info.stdout), + ) + # Rasterlite DB + if "Rasterlite" in dataset_info["driverShortName"]: + idsn = f"RASTERLITE:{dsn},table={layer}" else: idsn = dsn From da15c4eda7bb80ce22e7f7806d8e11a592ebd76c Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Fri, 29 Sep 2023 05:57:34 +0200 Subject: [PATCH 07/10] Use Python GDAL binding instead of parsing gdalinfo program output --- gui/wxpython/gui_core/gselect.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index 99f3f54ba1e..413dee0bad4 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2670,25 +2670,19 @@ def _getRasterliteDBRasters(self, dsn): :return list: list of Rasterlite DB rasters """ - import subprocess - - rasterlite_info = subprocess.run( - ["gdalinfo", "-json", dsn], - capture_output=True, - ) - if rasterlite_info.stderr: - GError(parent=self, message=grass.decode(rasterlite_info.stderr)) - return - if rasterlite_info.stdout: - import json - - rasterlite_info = json.loads(grass.decode(rasterlite_info.stdout)) - rasters = rasterlite_info["metadata"].get("SUBDATASETS") - if rasters: - return [ - v.rsplit("table=")[-1] for k, v in rasters.items() if "NAME" in k - ] - return [os.path.basename(rasterlite_info["files"][0]).rsplit(".")[0]] + try: + from osgeo import gdal + except ImportError: + GError( + parent=self, + message=_("The GDAL library is missing. Please install it."), + ) + return [] + rasterlite = gdal.Open(dsn) + rasters = rasterlite.GetSubDatasets() + if rasters: + return [r[0].rsplit("table=")[-1] for r in rasters] + return [os.path.basename(rasterlite.GetFileList()[0]).rsplit(".")[0]] class ProjSelect(wx.ComboBox): From 0f2fccaa1efe22293793f4d22bfd787757fd980f Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Tue, 3 Oct 2023 21:01:52 +0200 Subject: [PATCH 08/10] Use the Python Osgeo GDAL package to get dataset driver name --- gui/wxpython/gui_core/gselect.py | 2 +- gui/wxpython/modules/import_export.py | 27 ++++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index 413dee0bad4..f4896c5a06b 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2675,7 +2675,7 @@ def _getRasterliteDBRasters(self, dsn): except ImportError: GError( parent=self, - message=_("The GDAL library is missing. Please install it."), + message=_("The Python osgeo package is missing. Please install it."), ) return [] rasterlite = gdal.Open(dsn) diff --git a/gui/wxpython/modules/import_export.py b/gui/wxpython/modules/import_export.py index 9e62aab2567..d3f9d82010d 100644 --- a/gui/wxpython/modules/import_export.py +++ b/gui/wxpython/modules/import_export.py @@ -474,27 +474,20 @@ def OnRun(self, event): if "PG:" in dsn: idsn = f"{dsn} table={layer}" elif os.path.exists(idsn): - import subprocess - - dataset_info = subprocess.run( - ["gdalinfo", "-json", dsn], - capture_output=True, - ) - if dataset_info.stderr: + try: + from osgeo import gdal + except ImportError: GError( parent=self, - message=grass.decode(dataset_info.stderr), + message=_( + "The Python osgeo package is missing." + " Please install it." + ), ) return - if dataset_info.stdout: - import json - - dataset_info = json.loads( - grass.decode(dataset_info.stdout), - ) - # Rasterlite DB - if "Rasterlite" in dataset_info["driverShortName"]: - idsn = f"RASTERLITE:{dsn},table={layer}" + dataset = gdal.Open(dsn) + if "Rasterlite" in dataset.GetDriver().ShortName: + idsn = f"RASTERLITE:{dsn},table={layer}" else: idsn = dsn From 685cb7af6288604fd4acd84549f5befc6d9402a6 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sat, 23 Dec 2023 18:06:38 +0100 Subject: [PATCH 09/10] Fix Python GDAL package name --- gui/wxpython/gui_core/gselect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/wxpython/gui_core/gselect.py b/gui/wxpython/gui_core/gselect.py index f4896c5a06b..7639eda6bf4 100644 --- a/gui/wxpython/gui_core/gselect.py +++ b/gui/wxpython/gui_core/gselect.py @@ -2675,7 +2675,7 @@ def _getRasterliteDBRasters(self, dsn): except ImportError: GError( parent=self, - message=_("The Python osgeo package is missing. Please install it."), + message=_("The Python GDAL package is missing. Please install it."), ) return [] rasterlite = gdal.Open(dsn) From a40c1e3f2c353758ffbb39c8959ab0823df9e050 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 1 May 2024 23:00:17 -0400 Subject: [PATCH 10/10] Update gui/wxpython/modules/import_export.py --- gui/wxpython/modules/import_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/wxpython/modules/import_export.py b/gui/wxpython/modules/import_export.py index d3f9d82010d..9028ba5c8c4 100644 --- a/gui/wxpython/modules/import_export.py +++ b/gui/wxpython/modules/import_export.py @@ -480,7 +480,7 @@ def OnRun(self, event): GError( parent=self, message=_( - "The Python osgeo package is missing." + "The Python GDAL package is missing." " Please install it." ), )