diff --git a/docs/user_guide/examples/explanation_kernelloop.md b/docs/user_guide/examples/explanation_kernelloop.md index e415e3250..cfde89724 100644 --- a/docs/user_guide/examples/explanation_kernelloop.md +++ b/docs/user_guide/examples/explanation_kernelloop.md @@ -57,7 +57,6 @@ import parcels.tutorial # Load the CopernicusMarine data in the Agulhas region from the example_datasets ds_fields = parcels.tutorial.open_dataset("CopernicusMarine_data_for_Argo_tutorial/data") -ds_fields.load() # load the dataset into memory # Create an idealised wind field and add it to the dataset tdim, ydim, xdim = (len(ds_fields.time),len(ds_fields.latitude), len(ds_fields.longitude)) @@ -83,6 +82,9 @@ fieldset = parcels.FieldSet.from_sgrid_conventions( "Wind": ("UWind", "VWind") } ) + +# Convert the FieldSet to windowed arrays for better performance +fieldset = fieldset.to_windowed_arrays() ``` Now we define a wind Kernel that uses a forward Euler method to apply the wind forcing. Note that we update the `particles.dx` and `particles.dy` variables, rather than `particles.x` and `particles.y` directly. diff --git a/docs/user_guide/examples/tutorial_delaystart.ipynb b/docs/user_guide/examples/tutorial_delaystart.ipynb index 6ec8bb296..d8dfa2ce3 100644 --- a/docs/user_guide/examples/tutorial_delaystart.ipynb +++ b/docs/user_guide/examples/tutorial_delaystart.ipynb @@ -56,12 +56,14 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "# Convert to SGRID-compliant dataset and create FieldSet\n", "fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n", "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", - "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)" + "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", + "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()" ] }, { @@ -398,7 +400,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Parcels:docs (3.14.6)", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/docs/user_guide/examples/tutorial_diffusion.ipynb b/docs/user_guide/examples/tutorial_diffusion.ipynb index 79fe4018c..b0314fbfb 100644 --- a/docs/user_guide/examples/tutorial_diffusion.ipynb +++ b/docs/user_guide/examples/tutorial_diffusion.ipynb @@ -472,8 +472,7 @@ "# Load the CopernicusMarine data in the Agulhas region from the example_datasets\n", "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", - ").isel(depth=slice(0, 1))\n", - "ds_fields.load() # load the dataset into memory" + ").isel(depth=slice(0, 1))" ] }, { @@ -528,7 +527,10 @@ "ds_cell_areas[\"cell_areas\"] = ([\"lat\", \"lon\"], calc_cell_areas(ds_fields))\n", "fset2 = parcels.FieldSet.from_sgrid_conventions(ds_cell_areas)\n", "\n", - "fieldset += fset2" + "fieldset += fset2\n", + "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()" ] }, { diff --git a/docs/user_guide/examples/tutorial_dt_integrators.ipynb b/docs/user_guide/examples/tutorial_dt_integrators.ipynb index 9fe32fa76..9ca44604d 100644 --- a/docs/user_guide/examples/tutorial_dt_integrators.ipynb +++ b/docs/user_guide/examples/tutorial_dt_integrators.ipynb @@ -67,7 +67,6 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "OUTPUT_FOLDER = Path(\"output\")\n", "OUTPUT_FOLDER.mkdir(exist_ok=True)\n", @@ -77,6 +76,9 @@ "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()\n", + "\n", "# Check field resolution in time and space\n", "print(\n", " f\"dt = {np.unique(np.diff(ds_fields.time.values).astype('timedelta64[h]'))} hours\"\n", diff --git a/docs/user_guide/examples/tutorial_gsw_density.ipynb b/docs/user_guide/examples/tutorial_gsw_density.ipynb index c94ce0549..2a2b267f5 100644 --- a/docs/user_guide/examples/tutorial_gsw_density.ipynb +++ b/docs/user_guide/examples/tutorial_gsw_density.ipynb @@ -43,9 +43,6 @@ " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", "\n", - "# TODO check how we can get good performance without loading full dataset in memory\n", - "ds_fields.load() # load the dataset into memory\n", - "\n", "# Convert to SGRID-compliant dataset and create FieldSet\n", "fields = {\n", " \"U\": ds_fields[\"uo\"],\n", @@ -54,7 +51,10 @@ " \"so\": ds_fields[\"so\"],\n", "}\n", "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", - "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)" + "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", + "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()" ] }, { diff --git a/docs/user_guide/examples/tutorial_manipulating_field_data.ipynb b/docs/user_guide/examples/tutorial_manipulating_field_data.ipynb index a5f9dd104..e536de38b 100644 --- a/docs/user_guide/examples/tutorial_manipulating_field_data.ipynb +++ b/docs/user_guide/examples/tutorial_manipulating_field_data.ipynb @@ -49,7 +49,6 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "# Create an idealised wind field and add it to the dataset\n", "ydim, xdim = len(ds_fields.latitude), len(ds_fields.longitude)\n", @@ -122,6 +121,9 @@ "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()\n", + "\n", "npart = 10\n", "z = np.repeat(ds_fields.depth[0].values, npart)\n", "lons = np.repeat(32.2, npart)\n", diff --git a/docs/user_guide/examples/tutorial_sampling.ipynb b/docs/user_guide/examples/tutorial_sampling.ipynb index f80c49104..70752e5e8 100644 --- a/docs/user_guide/examples/tutorial_sampling.ipynb +++ b/docs/user_guide/examples/tutorial_sampling.ipynb @@ -60,12 +60,14 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "# Convert to SGRID-compliant dataset and create FieldSet\n", "fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"], \"thetao\": ds_fields[\"thetao\"]}\n", "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", - "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)" + "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", + "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()" ] }, { diff --git a/docs/user_guide/examples/tutorial_write_in_kernel.ipynb b/docs/user_guide/examples/tutorial_write_in_kernel.ipynb index 4f6ea2ad6..ede2c0823 100644 --- a/docs/user_guide/examples/tutorial_write_in_kernel.ipynb +++ b/docs/user_guide/examples/tutorial_write_in_kernel.ipynb @@ -122,12 +122,12 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "# Convert to SGRID-compliant dataset and create FieldSet\n", "fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n", "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", + "fieldset = fieldset.to_windowed_arrays()\n", "\n", "pset = parcels.ParticleSet(fieldset=fieldset, x=[31, 32], y=[-32, -31], z=[1, 1])\n", "\n", diff --git a/docs/user_guide/getting_started/tutorial_output.ipynb b/docs/user_guide/getting_started/tutorial_output.ipynb index 1a0891cf2..e6444473d 100644 --- a/docs/user_guide/getting_started/tutorial_output.ipynb +++ b/docs/user_guide/getting_started/tutorial_output.ipynb @@ -58,12 +58,14 @@ "ds_fields = parcels.tutorial.open_dataset(\n", " \"CopernicusMarine_data_for_Argo_tutorial/data\"\n", ")\n", - "ds_fields.load() # load the dataset into memory\n", "\n", "# Convert to SGRID-compliant dataset and create FieldSet\n", "fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n", "ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n", - "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)" + "fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n", + "\n", + "# Convert the FieldSet to windowed arrays for better performance\n", + "fieldset = fieldset.to_windowed_arrays()" ] }, { diff --git a/docs/user_guide/getting_started/tutorial_quickstart.md b/docs/user_guide/getting_started/tutorial_quickstart.md index 63196ab01..508ce6044 100644 --- a/docs/user_guide/getting_started/tutorial_quickstart.md +++ b/docs/user_guide/getting_started/tutorial_quickstart.md @@ -31,8 +31,6 @@ hydrodynamics fields in which the particles are tracked. Here we provide an exam ```{code-cell} ds_fields = parcels.tutorial.open_dataset("CopernicusMarine_data_for_Argo_tutorial/data") - -ds_fields.load() # load the dataset into memory ds_fields ``` @@ -52,6 +50,15 @@ ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields) fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset) ``` +Now, in order to improve performance, we can convert the `parcels.FieldSet` to windowed arrays. This is especially useful for large datasets with many timeslices, as it allows Parcels to load only the necessary timeslices into memory during the simulation. + +#TODO add link to performance explanation notebook here. + +```{code-cell} +# Convert the FieldSet to windowed arrays for better performance +fieldset = fieldset.to_windowed_arrays() +``` + You can inspect the `parcels.FieldSet` object with the `describe` method in order to see which `parcels.Field`s are included, and which grid and interpolation method is used for each field. This also gives information on the type of mesh and the time interval of the `parcels.FieldSet`: ```{code-cell} diff --git a/tests/test_sigmagrids.py b/tests/test_sigmagrids.py index ec05b81d0..fd37027a0 100644 --- a/tests/test_sigmagrids.py +++ b/tests/test_sigmagrids.py @@ -46,7 +46,6 @@ def test_conversion_3DCROCO(): def test_advection_3DCROCO(): ds_fields = parcels.tutorial.open_dataset("CROCOidealized_data/data") - ds_fields.load() fields = { "U": ds_fields["u"], @@ -61,6 +60,7 @@ def test_advection_3DCROCO(): ds_fset = parcels.convert.croco_to_sgrid(fields=fields, coords=ds_fields) fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset) + fieldset = fieldset.to_windowed_arrays() fieldset.add_context("hc", ds_fields.hc.item()) runtime = 10_000