OPeNDAP: change chunk from {} to null

There's an issue with the interworkings of dask and OPeNDAP: If dask decides to do chunks on a dataset, it will request the whole chunk from the backend storage, even if only a part of it is needed in the end. However for OPeNDAP it is best to request only the required parts from the backend. This change disables initial dask-chunking for all datasets which previously used only a single chunk. While this likely will improve performance for a lot of cases, it may not always be the best solution. If multiple datasets are to be concatenated, dask has to be used. However, in those cases, the user maybe knows about chunking an will define those manually.
eurec4a · Mar 18, 2021 · 76ee606 · 76ee606
1 parent 54b9f78
commit 76ee606
Show file tree

Hide file tree

Showing 90 changed files with 415 additions and 415 deletions.
diff --git a/ATR/main.yaml b/ATR/main.yaml
@@ -9,7 +9,7 @@ sources:
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/PRODUCTS/TRACKS/EUREC4A_tracks_ATR_v1.0.nc
       auth: null
-      chunks: {}
+      chunks: null
       engine: netcdf4
     metadata:
       tags:

diff --git a/Atalante/main.yaml b/Atalante/main.yaml
@@ -6,7 +6,7 @@ sources:
   track:
     args:
       auth: null
-      chunks: {}
+      chunks: null
       engine: netcdf4
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/PRODUCTS/TRACKS/EUREC4A_tracks_Atalante_v1.0.nc
     description: Atalante track

diff --git a/BOREAL/main.yaml b/BOREAL/main.yaml
@@ -6,7 +6,7 @@ sources:
   track:
     args:
       auth: null
-      chunks: {}
+      chunks: null
       engine: netcdf4
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/PRODUCTS/TRACKS/EUREC4A_tracks_BOREAL_v1.0.nc
     description: BOREAL track

diff --git a/CU-RAAVEN/main.yaml b/CU-RAAVEN/main.yaml
@@ -6,7 +6,7 @@ sources:
   track:
     args:
       auth: null
-      chunks: {}
+      chunks: null
       engine: netcdf4
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/PRODUCTS/TRACKS/EUREC4A_tracks_CU-RAAVEN_v1.0.nc
     description: CU-RAAVEN track

diff --git a/Caravela/main.yaml b/Caravela/main.yaml
@@ -6,7 +6,7 @@ sources:
   track:
     args:
       auth: null
-      chunks: {}
+      chunks: null
       engine: netcdf4
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/PRODUCTS/TRACKS/EUREC4A_tracks_Caravela_v1.0.nc
     description: Caravela track

diff --git a/HALO/BAHAMAS_PositionAttitude.yaml b/HALO/BAHAMAS_PositionAttitude.yaml
@@ -8,116 +8,116 @@ sources:
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200115_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0118:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200118_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0119:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200119_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0122:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200122_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0124:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200124_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0126:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200126_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0128:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200128_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0130:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200130_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0131:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200131_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0202:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200202_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0205:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200205_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0207:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200207_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0209:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200209_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0211:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200211_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0213:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200213_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0215:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200215_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0218:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/ESSD_HALO_BAHAMAS_Subset/EUREC4A_HALO_BAHAMAS_PositionAttitude_20200218_v1.1.nc
       auth: null
-      chunks: {}
+      chunks: null
diff --git a/HALO/BAHAMAS_QL.yaml b/HALO/BAHAMAS_QL.yaml
@@ -8,102 +8,102 @@ sources:
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200119a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0122:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200122a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0124:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200124a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0126:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200126a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0128:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200128a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0130:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200130a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0131:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200131a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0202:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200202a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0205:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200205a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0207:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200207a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0209:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200209a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0211:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200211a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0213:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200213a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0215:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200215a.nc
       auth: null
-      chunks: {}
+      chunks: null
 
   HALO-0218:
     driver: opendap
     args:
       urlpath: https://observations.ipsl.fr/thredds/dodsC/EUREC4A/AIRCRAFT/HALO/BAHAMAS/data/EUREC4A_HALO_BAHAMAS-QL-10Hz_20200218a.nc
       auth: null
-      chunks: {}
+      chunks: null