diff --git a/docs/developers/install.md b/docs/developers/install.md index 49d78048..65f80712 100644 --- a/docs/developers/install.md +++ b/docs/developers/install.md @@ -94,16 +94,16 @@ psql -U postgres -d oldinsurancemaps -c "CREATE EXTENSION PostGIS;" Run migrations and create admin user to access the Django admin panel (and login to the site once it's running) ```bash -python manage.py migrate -python manage.py createsuperuser +uv run manage.py migrate +uv run manage.py createsuperuser ``` Load a few other fixtures with some default objects: ```bash -python manage.py loaddata default-region-categories -python manage.py loaddata default-layerset-categories -python manage.py loaddata default-navbar +uv run manage.py loaddata default-region-categories +uv run manage.py loaddata default-layerset-categories +uv run manage.py loaddata default-navbar ``` Alternatively, if you have set all of the `DATABASE_*` variables in `.env`, you can use the included script to perform all of the actions described above: @@ -125,7 +125,7 @@ source ./scripts/load_dev_data.sh There are a few js and css plugins that must be downloaded to the local static directory: ```bash -python manage.py get-plugins +uv run manage.py get-plugins ``` The frontend uses a suite of independently built svelte components. First install `pnpm`: https://pnpm.io/installation. Then: @@ -153,7 +153,7 @@ In production, use the `build` command instead, and then Django's `collectstatic ```bash pnpm run build cd ../../.. -python manage.py collectstatic --noinput +uv run manage.py collectstatic --noinput ``` This bash script combines all steps into one: @@ -168,7 +168,7 @@ You can now activate the virtual environment and then run the django dev server: ```bash source .venv/bin/activate -python manage.py runserver +uv run manage.py runserver ``` and view the site at `http://localhost:8000`. @@ -259,13 +259,13 @@ In production, you will already be using a webserver for static files so you wil All tests are stored in `ohmg/tests`. Make sure you have installed dev requirements, then run: ```bash -python manage.py test +uv run manage.py test ``` To skip the tests that make external calls to the LOC API, use the following command. Keep in mind that coverage numbers will be lower when you skip tests. ```bash -python manage.py test --exclude-tag=loc +uv run manage.py test --exclude-tag=loc ``` ## Load the Place scaffolding @@ -273,7 +273,7 @@ python manage.py test --exclude-tag=loc Load all the place objects to create geography scaffolding (this will take a minute or two) ```bash -python manage.py place import-all +uv run manage.py place import-all ``` ## Environment variables diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index 6bae135c..73ebb726 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -13,7 +13,7 @@ # set BASE_DIR which is used to locate log, cache, temp, static, and uploaded dirs BASE_DIR = PROJECT_DIR.parent -# the build file is generated and updated with python manage.py update_build +# the build file is generated and updated with ./manage.py update_build BUILD_FILE = BASE_DIR / ".build" BUILD_NUMBER = "" if BUILD_FILE.is_file(): diff --git a/ohmg/extensions/iiif.py b/ohmg/extensions/iiif.py index f1d0c492..14b3c37e 100644 --- a/ohmg/extensions/iiif.py +++ b/ohmg/extensions/iiif.py @@ -34,35 +34,35 @@ def get_target(self): wgs84.ImportFromEPSG(4326) mask_geojson = json.loads(self.layer.mask.geojson) coords = mask_geojson["coordinates"][0] - ct = CoordTransform(SpatialReference("WGS84"), SpatialReference("EPSG:3857")) + + target_crs = f"EPSG:{self.region.gcpgroup.crs_epsg}" + ct = CoordTransform(SpatialReference("WGS84"), SpatialReference(target_crs)) polygon = Polygon(coords) polygon.transform(ct) - g = Georeferencer( - crs=f"EPSG:{self.region.gcpgroup.crs_epsg}", - transformation=self.region.gcpgroup.transformation, - gcps_geojson=self.region.gcpgroup.as_geojson, - ) in_path = ( self.region.file.url if self.region.file.url.startswith("http") else self.region.file.path ) + + g = Georeferencer( + crs=target_crs, + transformation=self.region.gcpgroup.transformation, + gcps_geojson=self.region.gcpgroup.as_geojson, + ) g.make_gcps_vrt(in_path) ds = gdal.Open(g.gcps_vrt.get_vsi_url()) transformer = gdal.Transformer( # Source datasource - None, - # Target datasource ds, + # Target datasource + None, # Transformer options that are ultimately passed to 'GDALCreateGenImgProjTransformer2()' # https://gdal.org/api/gdal_alg.html#_CPPv432GDALCreateGenImgProjTransformer212GDALDatasetH12GDALDatasetHPPc - [ - ## need to update this, different number if thin plate spline - "MAX_GCP_ORDER=1", - ], + g.make_transformer_options(), ) - transposed, status = transformer.TransformPoints(False, polygon.coords[0]) + transposed, status = transformer.TransformPoints(True, polygon.coords[0]) coords_str = [f"{i[0]},{i[1]}" for i in transposed] coords_join = " ".join(coords_str) @@ -114,14 +114,15 @@ def get_gcps(self): return features def get_body(self): - if self.region.gcpgroup.transformation == "poly1": - transformation = {"type": "polynomial", "options": {"order": 1}} - elif self.region.gcpgroup.transformation == "tps": - transformation = { - "type": "thinPlateSpline", - } - else: - raise Exception("invalid transformation", self.region.gcpgroup.transformation) + match t := self.region.gcpgroup.transformation: + case "poly1": + transformation = {"type": "polynomial", "options": {"order": 1}} + case "tps": + transformation = {"type": "thinPlateSpline"} + case "helmert": + transformation = {"type": "helmert"} + case _: + raise Exception(f"invalid transformation: {t}") body = { "id": full_reverse("iiif_gcps_view", args=(self.layerid,)), diff --git a/ohmg/extensions/urls.py b/ohmg/extensions/urls.py index 65408ed4..e85ebc56 100644 --- a/ohmg/extensions/urls.py +++ b/ohmg/extensions/urls.py @@ -1,5 +1,6 @@ from django.urls import path, register_converter +from ohmg.extensions.feeds import PlaceFeed from ohmg.places.converters import PlaceConverter from .views import ( @@ -9,7 +10,6 @@ IIIFResourceView, IIIFSelectorView, ) -from ohmg.extensions.feeds import PlaceFeed register_converter(PlaceConverter, "place-slug") diff --git a/ohmg/frontend/svelte_components/src/components/interfaces/Georeferencer.svelte b/ohmg/frontend/svelte_components/src/components/interfaces/Georeferencer.svelte index 73a53141..1785c5fe 100644 --- a/ohmg/frontend/svelte_components/src/components/interfaces/Georeferencer.svelte +++ b/ohmg/frontend/svelte_components/src/components/interfaces/Georeferencer.svelte @@ -173,21 +173,6 @@ available: CONTEXT.user.perms.includes("core.use_helmert") }, ]; - $: { - if (gcpList.length == 3 && currentTransformation == "helmert") { - currentTransformation = "poly1" - } - if (currentTransformation == "helmert") { - minGCPs = 2; - } else { - minGCPs = 3; - } - } - $: { - if (gcpList.length <= 2 && currentTransformation != "helmert") { - previewMode = "n/a" - } - } let currentTargetProjection = 'EPSG:3857'; const availableProjections = [ @@ -590,6 +575,19 @@ note: props.note, }); }); + + // special handling for helmert, do this before the preview is regenerated + if (gcpList.length == 3 && currentTransformation == "helmert") { + currentTransformation = "poly1"; + } + if (currentTransformation == "helmert") { + minGCPs = 2; + } else { + minGCPs = 3; + } + if (gcpList.length <= 2 && currentTransformation != "helmert") { + previewMode = "n/a" + } getPreview(); } @@ -736,6 +734,16 @@ return featureCollection; }; + const preparePayload = function () { + return { + gcp_geojson: asGeoJSON(), + transformation: currentTransformation, + projection: currentTargetProjection, + sesh_id: sessionId, + last_preview_id: currentPreviewId, + } + } + function getPreview() { if (currentTransformation == "helmert") { minGCPs = 2; @@ -750,13 +758,7 @@ `/georeference/${REGION.id}/`, CONTEXT.ohmg_post_headers, 'preview', - { - gcp_geojson: asGeoJSON(), - transformation: currentTransformation, - projection: currentTargetProjection, - sesh_id: sessionId, - last_preview_id: currentPreviewId, - }, + preparePayload(), (result) => { previewMode = previewMode == 'n/a' ? 'transparent' : previewMode; // updating this variable will trigger the preview layer to be @@ -779,13 +781,7 @@ `/georeference/${REGION.id}/`, CONTEXT.ohmg_post_headers, 'submit', - { - gcp_geojson: asGeoJSON(), - transformation: currentTransformation, - projection: currentTargetProjection, - sesh_id: sessionId, - last_preview_id: currentPreviewId, - }, + preparePayload(), () => { window.location.href = `/map/${REGION.map}`; }, @@ -901,17 +897,23 @@ }} on:unload={cancelSession} /> -
- Create 3 or more ground control points to georeference this document. + Create three or more ground control points to georeference this document. First, click on a recognizable spot + in the document on the left to begin the GCP. Then, click on that same location in the web map on the right + to complete it. Learn more about the process Learn more + external={true}>in the docs.
- +

- You have only two GCPs, but it is highly advisable to have three or more. - Please add more, unless you are unable to do so. + You have only created two GCPs, but we recommend having three or more. + If possible, please add more before submitting your work.

HelmertParams: offset_y=offset_y, ) + def _get_helmert_proj_pipeline(self) -> str: + ## fit the four-parameter Helmert model in one shot + params = self._get_helmert_params() + + ## convert the rotation to arcseconds for the proj pipeline + arcseconds = (params.rotation + 90) * 3600 + + pipeline = ( + "+proj=pipeline " + "+step +proj=axisswap +order=2,1 " + f"+step +proj=helmert +x={params.offset_x} +y={params.offset_y} " + f"+theta={arcseconds} +s={params.scale}" + ) + + return pipeline + + def make_transformer_options(self) -> list[str]: + match self.transformation["id"]: + case "helmert": + pipeline = self._get_helmert_proj_pipeline() + logger.debug(f"applying: {pipeline}") + return [ + "SRC_METHOD=NO_GEOTRANSFORM", + f"COORDINATE_OPERATION={pipeline}", + f"DST_SRS={self.crs_wkt}", + ] + case _: + return [ + f"DST_SRS={self.crs_wkt}", + f"MAX_GCP_ORDER={self.transformation['gdal_code']}", + ] + def cleanup_files(self): for vrt in [ self.gcps_vrt, @@ -355,61 +388,16 @@ def make_warped_vrt( self.make_gcps_vrt(src_path, out_name) self.warped_vrt = VRTHandler(out_name, as_variant="modified") - if self.transformation["id"] == "helmert": - ## fit the four-parameter Helmert model in one shot - params = self._get_helmert_params() - - ## convert the rotation to arcseconds for the proj pipeline - arcseconds = (params.rotation + 90) * 3600 - - pipeline = ( - "+proj=pipeline " - "+step +proj=axisswap +order=2,1 " - f"+step +proj=helmert +x={params.offset_x} +y={params.offset_y} " - f"+theta={arcseconds} +s={params.scale}" - ) - - logger.debug(f"applying: {pipeline}") - - wo = gdal.WarpOptions( - creationOptions=[ - "BLOCKXSIZE=512", - "BLOCKYSIZE=512", - ], - coordinateOperation=pipeline, - format="VRT", - dstSRS=f"{self.crs_code}", - transformerOptions=["SRC_METHOD=NO_GEOTRANSFORM"], - dstAlpha=True, - resampleAlg="nearest", - ) - else: - wo = gdal.WarpOptions( - creationOptions=[ - # "NUM_THREADS=ALL_CPUS", - # ## originally used this set of flags used - # # "COMPRESS=DEFLATE", - # ## should have been used PREDICTOR=2 with DEFLATE but didn't know about it - # # "PREDICTOR=2" - # ## useful in general but not needed when using COG driver - "BLOCKXSIZE=512", - "BLOCKYSIZE=512", - # ## advisable if using JPEG with GTiff, but not supported in COG - # # "JPEG_QUALITY=75", - # # "PHOTOMETRIC=YCBCR", - # ## Use JPEG, as recommended by Paul Ramsey article: - # ## https://blog.cleverelephant.ca/2015/02/geotiff-compression-for-dummies.html - # "COMPRESS=JPEG", - ], - transformerOptions=[ - f"DST_SRS={self.crs_wkt}", - f'MAX_GCP_ORDER={self.transformation["gdal_code"]}', - ], - format="VRT", - dstSRS=f"{self.crs_code}", - dstAlpha=True, - resampleAlg="nearest", - ) + wo = gdal.WarpOptions( + creationOptions=[ + "BLOCKXSIZE=512", + "BLOCKYSIZE=512", + ], + transformerOptions=self.make_transformer_options(), + format="VRT", + dstAlpha=True, + resampleAlg="nearest", + ) try: gdal.Warp(str(self.warped_vrt.get_path()), self.gcps_vrt.get_vsi_url(), options=wo) @@ -432,15 +420,6 @@ def make_trimmed_vrt(self, src_path, multimask_json_file: Path, layer_name: str) cutlineLayer=multimask_json_file.stem, cutlineWhere=f"layer='{layer_name}'", cropToCutline=True, - # srcAlpha = True, - # dstAlpha = True, - # creationOptions= [ - # 'COMPRESS=JPEG', - # ] - # creationOptions= [ - # 'COMPRESS=DEFLATE', - # 'PREDICTOR=2', - # ] ) gdal.Warp(str(self.trimmed_vrt.get_path()), self.warped_vrt.get_vsi_url(), options=wo) @@ -458,17 +437,9 @@ def make_cog( self.cog = Path(settings.TEMP_DIR, Path(src_path).stem + "-modified.tif") to = gdal.TranslateOptions( - # format="GTiff", format="COG", - # maskBand="mask", creationOptions=[ - # 'COMPRESS=DEFLATE', - # 'PREDICTOR=2', "COMPRESS=JPEG", - # 'TILED=YES', - # 'BLOCKXSIZE=512', - # 'BLOCKYSIZE=512', - # "PHOTOMETRIC=YCBCR", "TILING_SCHEME=GoogleMapsCompatible", ], resampleAlg="nearest", diff --git a/scripts/dump_database.sh b/scripts/dump_database.sh index 08ff2257..2e033a43 100644 --- a/scripts/dump_database.sh +++ b/scripts/dump_database.sh @@ -23,7 +23,7 @@ models="accounts.user for i in $models; do echo dumping $i - python manage.py dumpdata $i > $OUTDIR/$i.json + uv run manage.py dumpdata $i > $OUTDIR/$i.json done tar -czvf $BACKUPDIR/ohmg_$DATETIME.tar.gz $OUTDIR/* diff --git a/scripts/load_dev_data.sh b/scripts/load_dev_data.sh index 6b742029..0c79b583 100644 --- a/scripts/load_dev_data.sh +++ b/scripts/load_dev_data.sh @@ -6,14 +6,14 @@ set -a source $ENV_FILE set +a -python manage.py loaddata ohmg/core/fixtures/sanborn-region-categories.json -python manage.py loaddata ohmg/core/fixtures/sanborn-layerset-categories.json +uv run manage.py loaddata ohmg/core/fixtures/sanborn-region-categories.json +uv run manage.py loaddata ohmg/core/fixtures/sanborn-layerset-categories.json -python manage.py loaddata tests/data/fixtures/places/new-iberia-la-and-parents.json -python manage.py loaddata tests/data/fixtures/core/new-iberia-1885-map.json -python manage.py loaddata tests/data/fixtures/core/new-iberia-1885-docs.json +uv run manage.py loaddata tests/data/fixtures/places/new-iberia-la-and-parents.json +uv run manage.py loaddata tests/data/fixtures/core/new-iberia-1885-map.json +uv run manage.py loaddata tests/data/fixtures/core/new-iberia-1885-docs.json -python manage.py loaddata tests/data/fixtures/core/new-iberia-1885-main-content-layerset.json +uv run manage.py loaddata tests/data/fixtures/core/new-iberia-1885-main-content-layerset.json mkdir -p ./uploaded/documents mkdir -p ./uploaded/thumbnails @@ -27,4 +27,4 @@ cp ./tests/data/files/thumbnails/new_iberia_la_1885_p2-doc-thumb.jpg ./uploaded/ cp ./tests/data/files/thumbnails/new_iberia_la_1885_p3-doc-thumb.jpg ./uploaded/thumbnails/new_iberia_la_1885_p3-doc-thumb.jpg # small hack to resave and trigger signals on the test Document objects. -python manage.py shell -c "from ohmg.core.models import Document; [i.save() for i in Document.objects.all()]" +uv run manage.py shell -c "from ohmg.core.models import Document; [i.save() for i in Document.objects.all()]" diff --git a/scripts/setup_database.sh b/scripts/setup_database.sh index 984d6467..f3ea0fbd 100644 --- a/scripts/setup_database.sh +++ b/scripts/setup_database.sh @@ -18,10 +18,10 @@ then psql -U postgres -h $DATABASE_HOST -c "DROP DATABASE IF EXISTS "$DATABASE_NAME" WITH (FORCE);" psql -U postgres -h $DATABASE_HOST -c "CREATE DATABASE "$DATABASE_NAME" WITH OWNER "$DATABASE_USER";" - python manage.py migrate + uv run manage.py migrate - python manage.py loaddata tests/data/fixtures/auth/admin-user.json - python manage.py loaddata default-region-categories - python manage.py loaddata default-layerset-categories - python manage.py loaddata default-navbar + uv run manage.py loaddata tests/data/fixtures/auth/admin-user.json + uv run manage.py loaddata default-region-categories + uv run manage.py loaddata default-layerset-categories + uv run manage.py loaddata default-navbar fi