diff --git a/.github/workflows/frontend-overpass.yaml b/.github/workflows/frontend-overpass.yaml index 5b710de1..4b2f7c46 100644 --- a/.github/workflows/frontend-overpass.yaml +++ b/.github/workflows/frontend-overpass.yaml @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@v6 with: repository: OpenHistoricalMap/overpass-turbo - ref: f7c69c8bb16f37f88d55de44fd5854a1485890aa + ref: 527c3accf412f73766b70193f4577f2d2021d534 path: overpass-turbo - name: Enable Corepack diff --git a/hetzner/tiler/tiler.production.yml b/hetzner/tiler/tiler.production.yml index 173e500e..693fb461 100644 --- a/hetzner/tiler/tiler.production.yml +++ b/hetzner/tiler/tiler.production.yml @@ -18,7 +18,7 @@ services: tiler_imposm: container_name: tiler_imposm - image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.3349.h2739b4b1 + image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.3371.hef98f958 volumes: - tiler_imposm_data:/mnt/data env_file: @@ -79,7 +79,7 @@ services: tiler_server_martin: container_name: tiler_server_martin - image: ghcr.io/openhistoricalmap/tiler-server-martin:0.0.1-0.dev.git.3359.h756000d2 + image: ghcr.io/openhistoricalmap/tiler-server-martin:0.0.1-0.dev.git.3371.hef98f958 restart: always environment: - OHM_DOMAIN=${OHM_DOMAIN:-openhistoricalmap.org} @@ -132,13 +132,13 @@ services: volumes: tiler_pgdata: driver: local - name: tiler_db_2104 + name: tiler_db_2105 tiler_imposm_data: driver: local - name: tiler_imposm_2104 + name: tiler_imposm_2105 tiler_monitor_data: driver: local - name: tiler_monitor_2104 + name: tiler_monitor_2105 networks: ohm_network: external: true diff --git a/images/tiler-imposm/config/imposm3.template.json b/images/tiler-imposm/config/imposm3.template.json index 4f29a2b4..5500a4cd 100644 --- a/images/tiler-imposm/config/imposm3.template.json +++ b/images/tiler-imposm/config/imposm3.template.json @@ -73,11 +73,6 @@ "odbl:note", "history", - "roof:*", - "building:levels", - "building:part", - "building:material", - "building:colour", "generator:*", "tactile_paving", "crossing:markings", diff --git a/images/tiler-imposm/config/layers/buildings.json b/images/tiler-imposm/config/layers/buildings.json index 5a41f21b..5a97a294 100644 --- a/images/tiler-imposm/config/layers/buildings.json +++ b/images/tiler-imposm/config/layers/buildings.json @@ -41,6 +41,66 @@ "name": "height", "key": "height" }, + { + "type": "string", + "name": "min_height", + "key": "min_height" + }, + { + "type": "string", + "name": "building_height", + "key": "building:height" + }, + { + "type": "string", + "name": "building_min_level", + "key": "building:min_level" + }, + { + "type": "string", + "name": "building_use", + "key": "building:use" + }, + { + "type": "string", + "name": "building_material", + "key": "building:material" + }, + { + "type": "string", + "name": "building_levels", + "key": "building:levels" + }, + { + "type": "string", + "name": "building_colour", + "key": "building:colour" + }, + { + "type": "string", + "name": "building_part", + "key": "building:part" + }, + { + "type": "string", + "name": "roof_material", + "key": "roof:material" + }, + { + "type": "string", + "name": "roof_colour", + "key": "roof:colour" + }, + { + "type": "string", + "name": "roof_shape", + "key": "roof:shape" + }, + { + "type": "string", + "name": "roof_height", + "key": "roof:height" + }, { "type": "string", "name": "start_date", @@ -96,7 +156,16 @@ "mapping": { "building": [ "__any__" + ], + "building:part": [ + "__any__" ] + }, + "filters": { + "reject": { + "building": ["no", "none", "No"], + "building:part": ["no", "none", "No"] + } } } } diff --git a/images/tiler-imposm/config/layers/buildings_points.json b/images/tiler-imposm/config/layers/buildings_points.json index ea0ba667..7d562c51 100644 --- a/images/tiler-imposm/config/layers/buildings_points.json +++ b/images/tiler-imposm/config/layers/buildings_points.json @@ -36,6 +36,11 @@ "name": "class", "key": null }, + { + "type": "string", + "name": "building_use", + "key": "building:use" + }, { "type": "string", "name": "start_date", diff --git a/images/tiler-imposm/config/layers/buildings_relation_members.json b/images/tiler-imposm/config/layers/buildings_relation_members.json new file mode 100644 index 00000000..63b97dbc --- /dev/null +++ b/images/tiler-imposm/config/layers/buildings_relation_members.json @@ -0,0 +1,72 @@ +{ + "tags": { + "load_all": true, + "exclude": [ + "created_by", + "source", + "source:datetime" + ] + }, + "generalized_tables": {}, + "tables": { + "buildings_relation_members": { + "type": "relation_member", + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "name": "geometry", + "type": "geometry" + }, + { + "name": "member", + "type": "member_id" + }, + { + "name": "role", + "type": "member_role" + }, + { + "name": "me_building", + "type": "string", + "key": "building", + "from_member": true + }, + { + "name": "me_building_part", + "type": "string", + "key": "building:part", + "from_member": true + }, + { + "type": "hstore_tags", + "name": "me_tags", + "from_member": true + } + ], + "mapping": { + "type": [ + "building" + ] + } + } + } +} diff --git a/images/tiler-imposm/config/layers/water_multilines.json b/images/tiler-imposm/config/layers/water_multilines.json new file mode 100644 index 00000000..93c99a46 --- /dev/null +++ b/images/tiler-imposm/config/layers/water_multilines.json @@ -0,0 +1,89 @@ +{ + "tags": { + "load_all": true, + "exclude": [ + "created_by", + "source", + "source:datetime" + ] + }, + "generalized_tables": {}, + "tables": { + "water_multilines": { + "type": "relation_member", + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + }, + { + "type": "string", + "name": "bridge", + "key": "bridge" + }, + { + "type": "string", + "name": "start_date", + "key": "start_date" + }, + { + "type": "string", + "name": "end_date", + "key": "end_date" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "member_id", + "name": "member" + }, + { + "type": "hstore_tags", + "name": "me_tags", + "from_member": true + }, + { + "type": "string", + "name": "me_name", + "key": "name", + "from_member": true + }, + { + "type": "string", + "name": "me_waterway", + "key": "waterway", + "from_member": true + } + ], + "mapping": { + "type": [ + "waterway" + ] + } + } + } +} diff --git a/images/tiler-imposm/queries/ohm_mviews/buildings.sql b/images/tiler-imposm/queries/ohm_mviews/buildings.sql index 9f7d4f8b..350b3721 100644 --- a/images/tiler-imposm/queries/ohm_mviews/buildings.sql +++ b/images/tiler-imposm/queries/ohm_mviews/buildings.sql @@ -1,66 +1,93 @@ -- ============================================================================ --- Prepare points materialized view for higher zoom levels (12+) --- Add height and height_fixed columns +-- Index on building relation outline members. +-- Speeds up the hide_3d EXISTS subquery in mv_buildings_areas_z16_20. +-- ============================================================================ +CREATE INDEX IF NOT EXISTS osm_buildings_relation_members_outline_idx + ON osm_buildings_relation_members (member) + WHERE role = 'outline'; + +-- ============================================================================ +-- Points mview for zoom 12+. Building and roof attributes come from imposm +-- in osm_buildings_points. Point buildings don't render in 3D, so +-- render_height and render_min_height are NULL placeholders. They're here +-- so the UNION with polygon centroids in mv_buildings_points_centroids_* +-- lines up column-for-column. -- ============================================================================ SELECT create_points_mview( 'osm_buildings_points', 'mv_buildings_points', 'id, source, osm_id', - ARRAY['NULL as height'], + ARRAY[ + 'NULL::double precision AS render_height', + 'NULL::double precision AS render_min_height', + 'NULL::double precision AS roof_height', + 'NULL::text AS building_material', + 'NULL::text AS building_colour', + 'NULL::text AS building_part', + 'FALSE::boolean AS is_building_part', + 'FALSE::boolean AS hide_3d', + 'NULL::text AS roof_material', + 'NULL::text AS roof_colour', + 'NULL::text AS roof_shape' + ], NULL ); - -- ============================================================================ --- Zoom 14-15: --- Very low simplification (5m) --- Very small areas (>5K m² = 0.005 km²) --- Add height_fixed column +-- Base areas mview, zoom 16-20. +-- render_height and render_min_height use the openmaptiles fallback: +-- parsed height first, then building:height (deprecated), then +-- building:levels * 3m. Raw height and level columns are dropped, so +-- anything downstream reads render_height / render_min_height instead. +-- roof_height stays separate (parsed, not combined) since roof rendering +-- needs it on its own. Lower zooms inherit this schema. -- ============================================================================ + SELECT create_areas_mview( 'osm_buildings', - 'mv_buildings_areas_z14_15', - 5, - 5000, + 'mv_buildings_areas_z16_20', + 0, + 0, 'id, osm_id, type', NULL, - NULL, - NULL + '(class = ''building:part'') AS is_building_part, + EXISTS (SELECT 1 FROM osm_buildings_relation_members obrm WHERE obrm.member = ABS(osm_buildings.osm_id) AND obrm.role = ''outline'') AS hide_3d, + render_height(height, building_height, building_levels) AS render_height, + render_min_height(min_height, building_min_level) AS render_min_height', + '{"roof_height": "parse_to_meters(roof_height)"}'::jsonb, + ARRAY['height', 'min_height', 'building_height', 'building_levels', 'building_min_level'] ); -SELECT create_points_centroids_mview( - 'mv_buildings_areas_z14_15', - 'mv_buildings_points_centroids_z14_15', - 'mv_buildings_points' -); -- ============================================================================ --- Zoom 16-20: --- No simplification --- All areas --- Add height_fixed column +-- Areas z14-15, derived from z16-20. +-- Light 5m simplification, drops buildings under 5,000 m². -- ============================================================================ - -SELECT create_areas_mview( - 'osm_buildings', +SELECT create_area_mview_from_mview( 'mv_buildings_areas_z16_20', - 0, - 0, - 'id, osm_id, type', - NULL, - NULL, + 'mv_buildings_areas_z14_15', + 5, + 5000, NULL ); + +-- ============================================================================ +-- Centroid mviews per zoom. Each one UNIONs polygon centroids with +-- point-tagged buildings. +-- ============================================================================ SELECT create_points_centroids_mview( 'mv_buildings_areas_z16_20', 'mv_buildings_points_centroids_z16_20', 'mv_buildings_points' ); +SELECT create_points_centroids_mview( + 'mv_buildings_areas_z14_15', + 'mv_buildings_points_centroids_z14_15', + 'mv_buildings_points' +); + --- Refresh areas views -- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_buildings_areas_z14_15; -- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_buildings_areas_z16_20; - --- Refresh centroids views -- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_buildings_points_centroids_z14_15; -- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_buildings_points_centroids_z16_20; diff --git a/images/tiler-imposm/queries/utils/create_01_areas_mview.sql b/images/tiler-imposm/queries/utils/create_01_areas_mview.sql index 4d8c3b50..096fd8e1 100644 --- a/images/tiler-imposm/queries/utils/create_01_areas_mview.sql +++ b/images/tiler-imposm/queries/utils/create_01_areas_mview.sql @@ -19,6 +19,7 @@ -- where_filter TEXT - Optional WHERE clause filter (e.g., "type != 'barrier'" or "class NOT IN ('power', 'military')"). NULL = no filter -- tag_columns TEXT - Optional extra columns from tags (e.g., "tags->'religion' AS religion, tags->'denomination' AS denomination"). NULL = none -- column_overrides JSONB - Optional mapping {column_name: sql_expression} to override how an existing column is selected (e.g., '{"ref": "COALESCE(faa, iata, icao, NULLIF(ref, ''''))"}'). NULL = no overrides +-- exclude_columns TEXT[] - Optional list of source-table column names to omit from the mview output (e.g., ARRAY['building_height'] to suppress a deprecated column). NULL = no exclusions -- -- Notes: -- - Creates the materialized view using a temporary swap mechanism @@ -30,6 +31,7 @@ DROP FUNCTION IF EXISTS create_areas_mview(TEXT, TEXT, DOUBLE PRECISION, DOUBLE PRECISION, TEXT, TEXT); DROP FUNCTION IF EXISTS create_areas_mview(TEXT, TEXT, DOUBLE PRECISION, DOUBLE PRECISION, TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS create_areas_mview(TEXT, TEXT, DOUBLE PRECISION, DOUBLE PRECISION, TEXT, TEXT, TEXT, JSONB); +DROP FUNCTION IF EXISTS create_areas_mview(TEXT, TEXT, DOUBLE PRECISION, DOUBLE PRECISION, TEXT, TEXT, TEXT, JSONB, TEXT[]); CREATE OR REPLACE FUNCTION create_areas_mview( source_table TEXT, @@ -39,7 +41,8 @@ CREATE OR REPLACE FUNCTION create_areas_mview( unique_columns TEXT DEFAULT 'id, osm_id, type', where_filter TEXT DEFAULT NULL, tag_columns TEXT DEFAULT NULL, - column_overrides JSONB DEFAULT NULL + column_overrides JSONB DEFAULT NULL, + exclude_columns TEXT[] DEFAULT NULL ) RETURNS void AS $$ DECLARE @@ -97,7 +100,8 @@ BEGIN FROM information_schema.columns WHERE table_schema = 'public' AND table_name = source_table - AND column_name NOT IN ('start_decdate', 'end_decdate'); + AND column_name NOT IN ('start_decdate', 'end_decdate') + AND (exclude_columns IS NULL OR NOT (column_name = ANY(exclude_columns))); -- Always add calculated date columns all_cols := all_cols || ', public.isodatetodecimaldate(public.pad_date(start_date, ''start''), FALSE) AS start_decdate'; diff --git a/images/tiler-imposm/queries/utils/postgis_helpers.sql b/images/tiler-imposm/queries/utils/postgis_helpers.sql index 41f9d85d..e4393d35 100644 --- a/images/tiler-imposm/queries/utils/postgis_helpers.sql +++ b/images/tiler-imposm/queries/utils/postgis_helpers.sql @@ -16,4 +16,125 @@ END; $$ STRICT LANGUAGE plpgsql IMMUTABLE; +-- Parse free-form OSM/OHM height-like values (height=*, min_height=*, roof:height=*, +-- building:height=*) to meters as double precision. +-- +-- Accepts (canonical units per https://wiki.openstreetmap.org/wiki/Map_features/Units): +-- "20", "20.5" -> meters (OSM default unit) +-- "20 m", "20m" -> meters with explicit unit (case-sensitive) +-- "85'" -> feet -> meters +-- "8'5\"", "8'5" -> feet + inches -> meters +-- "8 1/2'" -> fractional feet (US notation) -> meters +-- +-- Hardening: +-- - Normalizes Unicode curly quotes/primes to ASCII straight quotes +-- (’ -> ', ” -> ", ′ -> ', ″ -> ") and NBSP to space. +-- - Collapses whitespace runs (tabs, newlines, multiple spaces) before parsing. +-- - Rejects strings longer than 64 chars early (likely garbage / URLs). +-- - Regex pre-validates before any cast, so the function never raises and +-- no EXCEPTION block is needed (avoids per-call subtransaction overhead +-- in bulk queries). +-- +-- Returns NULL for null / empty / unparseable / non-positive / >1000m input. +-- Why NULL on 0: 0 in MVT short-circuits style COALESCE chains and renders +-- 3D extrusions flat. NULL is stripped from MVT properties, so consumers can +-- use ["coalesce", ["get","height"], ] as intended. +CREATE OR REPLACE FUNCTION parse_to_meters(input text) RETURNS double precision AS $$ +DECLARE + s text; + ft numeric; + ft_int numeric; + ft_num numeric; + ft_den numeric; + inch numeric; + result double precision; +BEGIN + IF input IS NULL THEN + RETURN NULL; + END IF; + + -- Normalize Unicode quotes/primes to ASCII; map NBSP to space. + s := translate(input, + E'‘’‛′“”‟″ ', + '''''''''""""' || ' '); + -- Collapse whitespace runs and trim. + s := btrim(regexp_replace(s, '\s+', ' ', 'g')); + + IF s = '' OR length(s) > 64 THEN + RETURN NULL; + END IF; + + -- Fractional feet (US notation): 8 1/2' -> 8.5' + IF s ~ '^\d+ \d+/\d+''$' THEN + ft_int := (regexp_match(s, '^(\d+) '))[1]::numeric; + ft_num := (regexp_match(s, ' (\d+)/'))[1]::numeric; + ft_den := (regexp_match(s, '/(\d+)'''))[1]::numeric; + IF ft_den = 0 THEN + RETURN NULL; + END IF; + result := (ft_int + ft_num / ft_den) * 0.3048; + + -- Feet + optional inches: 8'5" / 8'5 / 8' + ELSIF s ~ '^\d+(\.\d+)?''( ?\d+(\.\d+)?"?)?$' THEN + ft := (regexp_match(s, '^(\d+(\.\d+)?)'''))[1]::numeric; + inch := COALESCE((regexp_match(s, ''' ?(\d+(\.\d+)?)"?$'))[1], '0')::numeric; + result := (ft * 0.3048) + (inch * 0.0254); + + -- Meters (default or with explicit "m" suffix). + -- Symbols case-sensitive per OSM Units spec. + ELSIF s ~ '^-?\d+(\.\d+)?( ?m)?$' THEN + result := regexp_replace(s, ' ?m$', '')::double precision; + + ELSE + RETURN NULL; + END IF; + + IF result IS NULL OR result <= 0 OR result > 1000 THEN + RETURN NULL; + END IF; + RETURN result; +END; +$$ STRICT +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; + +-- Parse a numeric string to numeric, or NULL if malformed. +-- Regex from OpenMapTiles CleanNumeric. +CREATE OR REPLACE FUNCTION clean_numeric(input text) RETURNS numeric AS $$ + SELECT substring(input from '^\s*([-+]?(?=\d|\.\d)\d*(?:\.\d*)?(?:[Ee][-+]?\d+)?)\s*$')::numeric; +$$ STRICT +LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +-- Resolve a building height in meters, using OpenMapTiles fallback order: +-- 1. height=* (parsed via parse_to_meters) +-- 2. building:height=* (deprecated alias, parsed) +-- 3. building:levels=* * 3m (one level assumed 3m) +-- Returns NULL when no source is parseable. Mirrors the openmaptiles +-- building layer logic so render_height stays consistent across consumers. +-- Reference (OMT uses 3.66m per level; we use 3m): +-- https://github.com/openmaptiles/openmaptiles/blob/master/layers/building/building.sql#L101-L102 +CREATE OR REPLACE FUNCTION render_height( + height_text text, + building_height_text text, + building_levels_text text +) RETURNS double precision AS $$ + SELECT COALESCE( + public.parse_to_meters(height_text), + public.parse_to_meters(building_height_text), + public.clean_numeric(building_levels_text)::double precision * 3 + ); +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +-- Resolve a building min_height in meters using the same fallback pattern: +-- 1. min_height=* (parsed via parse_to_meters) +-- 2. building:min_level=* * 3m (one level assumed 3m) +CREATE OR REPLACE FUNCTION render_min_height( + min_height_text text, + building_min_level_text text +) RETURNS double precision AS $$ + SELECT COALESCE( + public.parse_to_meters(min_height_text), + public.clean_numeric(building_min_level_text)::double precision * 3 + ); +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + COMMIT; \ No newline at end of file diff --git a/images/tiler-imposm/queries/utils/utils.sql b/images/tiler-imposm/queries/utils/utils.sql index 108bd40d..35e76fac 100644 --- a/images/tiler-imposm/queries/utils/utils.sql +++ b/images/tiler-imposm/queries/utils/utils.sql @@ -79,15 +79,22 @@ $$ LANGUAGE plpgsql; -- Function: get_language_columns() -- Description: -- Returns a comma-separated list of SQL expressions like: --- tags -> 'name:es' AS "es" --- Based on aliases found in the `languages` table. +-- tags -> 'name:zh-Hant-TW' AS "name_zh-Hant-TW" +-- Based on the `key_name` values stored in the `languages` table. +-- +-- Only the first ':' in the tag key (the one separating `name` from the +-- language tag) is rewritten to `_`. Hyphens and mixed case inside the +-- IETF BCP 47 language tag are preserved so consumers (e.g. Diplomat) can +-- discover script and country subtags. Identifiers are quoted with %I so +-- that hyphens / uppercase are accepted as PostgreSQL column names. -- -- Notes: -- - Designed for direct use when `tags` is accessed without a table alias. -- - Useful for generating multilingual columns dynamically in SQL queries. -- -- Example: --- get_language_columns() → "tags->'name:es' AS es, tags->'name:fr' AS fr, ..." +-- get_language_columns() → tags->'name:es' AS "name_es", +-- tags->'name:zh-Hant-TW' AS "name_zh-Hant-TW", ... -- ============================================================================ CREATE OR REPLACE FUNCTION get_language_columns() RETURNS TEXT AS $$ @@ -98,7 +105,7 @@ BEGIN format( 'tags -> %L AS %I', key_name, - 'name_' || regexp_replace(lower(substring(key_name from 6)), '[^a-z0-9]', '_', 'g') + 'name_' || substring(key_name from 6) ), ', ' ) diff --git a/images/tiler-imposm/scripts/create_mviews.sh b/images/tiler-imposm/scripts/create_mviews.sh index 266d078e..cc6b0e06 100755 --- a/images/tiler-imposm/scripts/create_mviews.sh +++ b/images/tiler-imposm/scripts/create_mviews.sh @@ -63,6 +63,10 @@ fi ##################### OHM ##################### log_message "Creating materialized views for OSM data" +# Always (re)load helper functions so mview SQL can rely on them after +# upgrades or reimports without re-running the full --all path. +execute_sql_file ./queries/utils/postgis_helpers.sql + ## Admin boundaries areas execute_sql_file queries/ohm_mviews/admin_boundaries_areas.sql execute_sql_file queries/ohm_mviews/admin_boundaries_centroids.sql diff --git a/images/tiler-imposm/scripts/refresh_mviews.sh b/images/tiler-imposm/scripts/refresh_mviews.sh index 62b908c2..824584a3 100755 --- a/images/tiler-imposm/scripts/refresh_mviews.sh +++ b/images/tiler-imposm/scripts/refresh_mviews.sh @@ -272,6 +272,10 @@ no_admin_boundaries_views=( mv_non_admin_boundaries_centroids_z0_2 ) +addresses_views=( + mv_address_points_z16_20 +) + log_message "Starting parallel refresh of materialized views..." @@ -292,3 +296,4 @@ refresh_mviews_group "WATER" 180 light "${water_views[@]}" & refresh_mviews_group "BUILDINGS" 180 light "${buildings_views[@]}" & refresh_mviews_group "ROUTES" 180 light "${routes_views[@]}" & refresh_mviews_group "NO_ADMIN_BOUNDARIES" 180 light "${no_admin_boundaries_views[@]}" & +refresh_mviews_group "ADDRESSES" 180 light "${addresses_views[@]}" & diff --git a/images/tiler-server-martin/scripts/generate_functions.py b/images/tiler-server-martin/scripts/generate_functions.py index ec47048d..62212fc5 100644 --- a/images/tiler-server-martin/scripts/generate_functions.py +++ b/images/tiler-server-martin/scripts/generate_functions.py @@ -17,8 +17,21 @@ import json import os +import re import psycopg2 +# Allowed column names: letters, digits, underscore, hyphen (for locale +# columns like "name_zh-Hant-TW"). Anything else is rejected before being +# interpolated into SQL. +SAFE_COL_NAME_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_-]*$') + + +def safe_col(c): + """Return c if it's a safe PostgreSQL identifier, else raise ValueError.""" + if not SAFE_COL_NAME_RE.match(c): + raise ValueError(f"Refusing to quote unsafe column name from catalog: {c!r}") + return c + BASE_DIR = os.path.join(os.path.dirname(__file__), "..") CONFIG_PATH = os.path.join(BASE_DIR, "config", "functions.json") OUTPUT_DIR = os.path.join(BASE_DIR, "pg_functions") @@ -123,17 +136,23 @@ def generate_function_sql(func_def, columns_per_table): # Build the IF/ELSIF/ELSE blocks blocks = [] + quoted_geom = f'"{geom_col}"' for i, (max_zoom, table_name) in enumerate(zoom_mapping): cols = columns_per_table[table_name] - col_list = ", ".join(f"t.{c}" for c in cols) + # Quote each column with double quotes so identifiers with hyphens or + # mixed case (e.g. localized name columns like "name_zh-Hant-TW") + # remain valid PostgreSQL identifiers. safe_col() rejects any name that + # falls outside [A-Za-z_][A-Za-z0-9_-]* to prevent SQL injection via + # crafted column names from the catalog. + col_list = ", ".join(f't."{safe_col(c)}"' for c in cols) extent, buffer = get_mvt_geom_params(max_zoom) query = ( f"SELECT ST_AsMVT(q, '{sl}', {extent}) INTO mvt FROM (\n" f" SELECT {col_list},\n" - f" ST_AsMVTGeom(t.{geom_col}, bounds, {extent}, {buffer}, true) AS geometry\n" + f" ST_AsMVTGeom(t.{quoted_geom}, bounds, {extent}, {buffer}, true) AS geometry\n" f" FROM public.{table_name} t\n" - f" WHERE t.{geom_col} && bounds\n" + f" WHERE t.{quoted_geom} && bounds\n" f" ) q;" )