From b8b31c5a1d992f139b08522b7ada3e0220b00b36 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Sat, 14 Dec 2024 13:25:18 +0100 Subject: [PATCH 1/4] python3Packages.python-mapnik: disable broken test --- pkgs/development/python-modules/python-mapnik/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/python-mapnik/default.nix b/pkgs/development/python-modules/python-mapnik/default.nix index 33770c3d82917..d360fb2ce0f1c 100644 --- a/pkgs/development/python-modules/python-mapnik/default.nix +++ b/pkgs/development/python-modules/python-mapnik/default.nix @@ -114,6 +114,7 @@ buildPythonPackage rec { "test_geometry_type" "test_passing_pycairo_context_pdf" "test_pdf_printing" + "test_raster_warping" "test_render_with_scale_factor" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ @@ -134,6 +135,5 @@ buildPythonPackage rec { maintainers = [ ]; homepage = "https://mapnik.org"; license = licenses.lgpl21Plus; - broken = true; # At 2024-11-13, test_raster_warping fails. }; } From b70af9ef2f7060f537cd369b35d08640ce6d50a7 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Tue, 17 Dec 2024 23:36:08 +0100 Subject: [PATCH 2/4] openstreetmap-carto: init at 5.9.0-unstable-2024-11-27 --- .../op/openstreetmap-carto/package.nix | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 pkgs/by-name/op/openstreetmap-carto/package.nix diff --git a/pkgs/by-name/op/openstreetmap-carto/package.nix b/pkgs/by-name/op/openstreetmap-carto/package.nix new file mode 100644 index 0000000000000..5fb16ce9cc68d --- /dev/null +++ b/pkgs/by-name/op/openstreetmap-carto/package.nix @@ -0,0 +1,199 @@ +{ + lib, + stdenvNoCC, + fetchFromGitHub, + fetchurl, + + carto, + gdal, + hanazono, + noto-fonts, + python3, + runCommand, + symlinkJoin, +}: + +let + generate_env = python3.withPackages ( + p: with p; [ + colormath + lxml + pyyaml + ] + ); + get-external-data_env = python3.withPackages ( + p: with p; [ + psycopg2 + pyyaml + requests + ] + ); + + mkArchive = + { + url, + archive_time, + hash, + }: + { + inherit url; + archive = fetchurl { + url = "https://web.archive.org/web/${archive_time}/${url}"; + inherit hash; + }; + }; + simplified_water_polygons = mkArchive { + url = "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip"; + archive_time = "20241201150706"; + hash = "sha256-vVVbHBCbeaTaYAZRvIEQ5N1Dy5E+RaOSgZS3Uf59QkU="; + }; + water_polygons = mkArchive { + url = "https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip"; + archive_time = "20241201150707"; + hash = "sha256-YV3KCcVGZoSrZVjASUCPEcTCn0C39l0pG4gBl5AJm5M="; + }; + icesheet_polygons = mkArchive { + url = "https://osmdata.openstreetmap.de/download/antarctica-icesheet-polygons-3857.zip"; + archive_time = "20241201150705"; + hash = "sha256-QvaegTQuiju8B8IyT6qq3QDjBWk2D02bu6g07G8VPMA="; + }; + icesheet_outlines = mkArchive { + url = "https://osmdata.openstreetmap.de/download/antarctica-icesheet-outlines-3857.zip"; + archive_time = "20241201150707"; + hash = "sha256-d09olTATtgLO4VOAAnbO6XOrSm89DOLbSzP4id7njtU="; + }; + ne_110m_admin_0_boundary_lines_land = mkArchive { + url = "https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_boundary_lines_land.zip"; + archive_time = "20241201150708"; + hash = "sha256-uylUmBVwSYpLQ0T46D+lTKpYWOFegACUESQH0r0vlA0="; + }; +in +stdenvNoCC.mkDerivation (finalAttrs: { + pname = "openstreetmap-carto"; + version = "5.9.0-unstable-2024-11-27"; + + src = fetchFromGitHub { + owner = "gravitystorm"; + repo = "openstreetmap-carto"; + rev = "5791e79934164f8d5ff73460ad993f3d9f64222c"; + hash = "sha256-0kPgX09oJprnTDnXdLhse8VWz/crjN5d40yKS0WhsHc="; + }; + + postPatch = '' + substituteInPlace external-data.yml \ + --replace-fail '${simplified_water_polygons.url}' 'file://${simplified_water_polygons.archive}' \ + --replace-fail '${water_polygons.url}' 'file://${water_polygons.archive}' \ + --replace-fail '${icesheet_polygons.url}' 'file://${icesheet_polygons.archive}' \ + --replace-fail '${icesheet_outlines.url}' 'file://${icesheet_outlines.archive}' \ + --replace-fail '${ne_110m_admin_0_boundary_lines_land.url}' 'file://${ne_110m_admin_0_boundary_lines_land.archive}' + + if grep http external-data.yml; then + echo 'Not all URLs were patched!' + exit 1 + fi + + substituteInPlace style/fonts.mss --replace-fail \ + "url('fonts')" \ + "url('${finalAttrs.passthru.fonts}')" + + substituteInPlace scripts/get-external-data.py \ + --replace-fail \ + 'ogrcommand = ["ogr2ogr",' \ + 'ogrcommand = ["${lib.getExe' gdal "ogr2ogr"}",' \ + --replace-fail \ + '#!/usr/bin/env python3' \ + '#!${lib.getExe get-external-data_env}' + + substituteInPlace scripts/generate_road_colours.py scripts/generate_shields.py scripts/generate_unpaved_patterns.py scripts/indexes.py \ + --replace-fail \ + '#!/usr/bin/env python3' \ + '#!${lib.getExe generate_env}' + ''; + + outputs = [ + "out" + "get_external_data" + ]; + + buildPhase = '' + runHook preBuild + + # Generate some files ourselves instead of relying on the pre-generated ones + mv symbols/unpaved/unpaved.svg unpaved.svg + mv style/road-colors-generated.mss road-colors-generated.orig.mss + rm -r indexes.sql symbols/shields symbols/unpaved + mkdir -p symbols/shields symbols/unpaved + mv unpaved.svg symbols/unpaved/unpaved.svg + + scripts/indexes.py > indexes.sql + scripts/generate_road_colours.py > style/road-colors-generated.mss + scripts/generate_shields.py + scripts/generate_unpaved_patterns.py + + # We don't copy this file to the output, so we can only check it here + if ! diff 'road-colors-generated.orig.mss' 'style/road-colors-generated.mss'; then + echo 'The file style/road-colors-generated.mss we generated differs from the pre-generated one!' + exit 1 + fi + + mkdir -p "$out/" + '${lib.getExe carto}' project.mml > "$out/mapnik.xml" + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p "$get_external_data/" + cp -r patterns/ symbols/ *.sql *.lua "$out/" + cp external-data.yml "$get_external_data/" + install -D scripts/get-external-data.py --target-directory="$get_external_data/bin/" + # There should be no need for the get-fonts.sh script as we patch `style/fonts.mss` to depend directly on the fonts. + + runHook postInstall + ''; + + strictDeps = true; + + passthru = { + fonts = symlinkJoin { + name = "osm-fonts"; + paths = [ + # Some fonts are probably missing, please add + "${noto-fonts}/share/fonts/noto" + "${hanazono}/share/fonts/truetype" + ]; + }; + + tests = { + inherit (finalAttrs.finalPackage.passthru) fonts; + + # Check that we generated the exact same files as the pre-generated ones + compare-generated-files = runCommand "compare-generated-files" { } '' + if ! diff '${finalAttrs.src}/indexes.sql' '${finalAttrs.finalPackage}/indexes.sql'; then + echo 'The file indexes.sql we generated differs from the pre-generated one!' + exit 1 + fi + if ! diff -r '${finalAttrs.src}/symbols/shields/' '${finalAttrs.finalPackage}/symbols/shields/'; then + echo 'The directory symbols/shields/ we generated differs from the pre-generated one!' + exit 1 + fi + if ! diff -r --exclude=unpaved.md '${finalAttrs.src}/symbols/unpaved/' '${finalAttrs.finalPackage}/symbols/unpaved/'; then + echo 'The directory symbols/unpaved/ we generated differs from the pre-generated one!' + exit 1 + fi + + touch "$out" + ''; + }; + }; + + meta = { + description = "General-purpose OpenStreetMap mapnik style, in CartoCSS"; + homepage = "https://github.com/gravitystorm/openstreetmap-carto"; + license = lib.licenses.cc0; + maintainers = lib.teams.geospatial.members ++ (with lib.maintainers; [ Luflosi ]); + platforms = lib.platforms.linux; + }; +}) From f3ee67efe3740c8417cd50fa0ecee017ee895c37 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Wed, 28 Jun 2023 13:41:50 +0200 Subject: [PATCH 3/4] nik4: init at 1.8 --- pkgs/by-name/ni/nik4/package.nix | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 pkgs/by-name/ni/nik4/package.nix diff --git a/pkgs/by-name/ni/nik4/package.nix b/pkgs/by-name/ni/nik4/package.nix new file mode 100644 index 0000000000000..d712d1e96da7e --- /dev/null +++ b/pkgs/by-name/ni/nik4/package.nix @@ -0,0 +1,38 @@ +{ + lib, + fetchFromGitHub, + + python3Packages, +}: + +python3Packages.buildPythonPackage rec { + pname = "nik4"; + version = "1.8"; + pyproject = true; + + src = fetchFromGitHub { + owner = "Zverik"; + repo = "Nik4"; + tag = "v${version}"; + hash = "sha256-ICNy+H1N8YY/5P4gCX/laAn+G2JNQOKhvh4fsgcM0kM="; + }; + + build-system = with python3Packages; [ setuptools ]; + + dependencies = with python3Packages; [ + pycairo + python-mapnik + ]; + + strictDeps = true; + + meta = { + changelog = "https://github.com/Zverik/Nik4/releases/tag/v${version}"; + description = "Mapnik to image export"; + homepage = "https://github.com/Zverik/Nik4"; + license = lib.licenses.wtfpl; + mainProgram = "nik4.py"; + maintainers = lib.teams.geospatial.members ++ (with lib.maintainers; [ Luflosi ]); + platforms = lib.platforms.linux; + }; +} From cc192b69143cfbd4b90be765112d8cc698714550 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Tue, 17 Dec 2024 23:59:42 +0100 Subject: [PATCH 4/4] nixos/tests/nik4: init --- nixos/tests/all-tests.nix | 1 + nixos/tests/nik4.nix | 69 +++++++++++++++++++ pkgs/by-name/ni/nik4/package.nix | 5 ++ .../op/openstreetmap-carto/package.nix | 2 + 4 files changed, 77 insertions(+) create mode 100644 nixos/tests/nik4.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 622d43d4080bd..9f0cb278d585b 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -667,6 +667,7 @@ in { nix-channel = pkgs.callPackage ../modules/config/nix-channel/test.nix { }; nebula = handleTest ./nebula.nix {}; netbird = handleTest ./netbird.nix {}; + nik4 = runTest ./nik4.nix; nimdow = handleTest ./nimdow.nix {}; neo4j = handleTest ./neo4j.nix {}; netdata = handleTest ./netdata.nix {}; diff --git a/nixos/tests/nik4.nix b/nixos/tests/nik4.nix new file mode 100644 index 0000000000000..069f8e09218da --- /dev/null +++ b/nixos/tests/nik4.nix @@ -0,0 +1,69 @@ +# This test downloads some example OpenStreetMap data, imports it into a database and then renders a portion of it. The resulting PNG image will be in the output. +# Importing the data takes some time, at least a couple minutes. + +{ lib, pkgs, ... }: +let + # Fetch a tiny example region + andorra = pkgs.fetchurl { + # The OpenStreetMap data from the first of January seems to be kept for at least 10 years + url = "https://download.geofabrik.de/europe/andorra-240101.osm.pbf"; + hash = "sha256-9RylVK1C28e9qnc8Lu1VGcxS3lNlmjPTWcWvJJlmAf4="; + }; +in +{ + name = "nik4"; + meta.maintainers = lib.teams.geospatial.members ++ (with lib.maintainers; [ Luflosi ]); + + nodes.machine = { + virtualisation = { + cores = 2; + diskSize = 8 * 1024; + memorySize = 2 * 1024; + }; + + services.postgresql = { + enable = true; + extensions = ps: with ps; [ postgis ]; + ensureUsers = lib.singleton { + name = "osmuser"; + ensureClauses.createdb = true; + }; + }; + + users.users.osmuser = { + isSystemUser = true; + group = "osmuser"; + }; + users.groups.osmuser = { }; + + environment.systemPackages = with pkgs; [ + osm2pgsql + nik4 + ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + + with subtest("Create database"): + machine.succeed("sudo -u postgres createdb --encoding=UTF8 --owner=osmuser gis") + machine.succeed("sudo -u postgres psql gis --command='CREATE EXTENSION postgis;'") + machine.succeed("sudo -u postgres psql gis --command='CREATE EXTENSION hstore;'") + + with subtest("Insert data into database"): + machine.succeed("mkdir /osm >&2") + machine.succeed("chown osmuser /osm >&2") + + machine.succeed("sudo -u osmuser '${pkgs.openstreetmap-carto.get_external_data}/bin/get-external-data.py' --config '${pkgs.openstreetmap-carto.get_external_data}/external-data.yml' --data '/osm/get-external-data/'") + machine.succeed("sudo -u osmuser osm2pgsql --output=flex --style='${pkgs.openstreetmap-carto}/openstreetmap-carto-flex.lua' --database=gis --create '${andorra}'") + machine.succeed("sudo -u osmuser psql -d gis -f '${pkgs.openstreetmap-carto}/indexes.sql'") + machine.succeed("sudo -u osmuser psql -d gis -f '${pkgs.openstreetmap-carto}/functions.sql'") + + with subtest("Execute nik4.py"): + machine.succeed("sudo -u osmuser nik4.py --url 'https://www.openstreetmap.org/#map=17/42.506650/1.525828' --ppi 300 -a 4 '${pkgs.openstreetmap-carto}/mapnik.xml' /tmp/map.png >&2") + machine.copy_from_vm("/tmp/map.png") + import os + image_size = os.stat(machine.out_dir / "map.png").st_size + assert image_size >= 2 * 1024 * 1024, f"The rendered map image was too small ({image_size} bytes). This indicates, that the map was not rendered correctly." + ''; +} diff --git a/pkgs/by-name/ni/nik4/package.nix b/pkgs/by-name/ni/nik4/package.nix index d712d1e96da7e..b6a265fc6c42d 100644 --- a/pkgs/by-name/ni/nik4/package.nix +++ b/pkgs/by-name/ni/nik4/package.nix @@ -2,6 +2,7 @@ lib, fetchFromGitHub, + nixosTests, python3Packages, }: @@ -26,6 +27,10 @@ python3Packages.buildPythonPackage rec { strictDeps = true; + passthru.tests = { + inherit (nixosTests) nik4; + }; + meta = { changelog = "https://github.com/Zverik/Nik4/releases/tag/v${version}"; description = "Mapnik to image export"; diff --git a/pkgs/by-name/op/openstreetmap-carto/package.nix b/pkgs/by-name/op/openstreetmap-carto/package.nix index 5fb16ce9cc68d..084ccd6125afa 100644 --- a/pkgs/by-name/op/openstreetmap-carto/package.nix +++ b/pkgs/by-name/op/openstreetmap-carto/package.nix @@ -7,6 +7,7 @@ carto, gdal, hanazono, + nixosTests, noto-fonts, python3, runCommand, @@ -167,6 +168,7 @@ stdenvNoCC.mkDerivation (finalAttrs: { }; tests = { + inherit (nixosTests) nik4; inherit (finalAttrs.finalPackage.passthru) fonts; # Check that we generated the exact same files as the pre-generated ones