From 6eca1dbbd0487c4c14f033f910c5e56198f994cc Mon Sep 17 00:00:00 2001 From: tsutterley Date: Mon, 3 Jun 2024 14:07:18 -0700 Subject: [PATCH 1/8] [WIP] using datatree to read complete ATL15 granule --- .gitignore | 1 + IS2view/__init__.py | 2 +- IS2view/api.py | 308 +++++++++++++++++++++++++------ IS2view/convert.py | 11 +- IS2view/io.py | 66 +++++-- IS2view/tools.py | 67 +++++-- IS2view/utilities.py | 63 +++++-- notebooks/IS2-ATL15-Viewer.ipynb | 30 +-- 8 files changed, 429 insertions(+), 119 deletions(-) diff --git a/.gitignore b/.gitignore index 24f75cb..f8fec0a 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ _*.c *.shp *.shx *.tif +*.vrt *.zarr # Logs and databases # ###################### diff --git a/IS2view/__init__.py b/IS2view/__init__.py index e66f808..1cb4469 100644 --- a/IS2view/__init__.py +++ b/IS2view/__init__.py @@ -11,7 +11,7 @@ import IS2view.version from IS2view.api import Leaflet, basemaps, layers, image_service_layer from IS2view.convert import convert -from IS2view.io import open_dataset, from_file, from_rasterio, from_xarray +from IS2view.io import open_datatree, open_dataset, from_file, from_rasterio, from_xarray from IS2view.tools import widgets # get semantic version from setuptools-scm __version__ = IS2view.version.version diff --git a/IS2view/api.py b/IS2view/api.py index 5cd1ec3..5adf209 100644 --- a/IS2view/api.py +++ b/IS2view/api.py @@ -5,6 +5,8 @@ Plotting tools for visualizing rioxarray variables on leaflet maps PYTHON DEPENDENCIES: + datatree: Tree-like hierarchical data structure for xarray + https://xarray-datatree.readthedocs.io geopandas: Python tools for geographic data http://geopandas.readthedocs.io/ ipywidgets: interactive HTML widgets for Jupyter notebooks and IPython @@ -56,20 +58,20 @@ import collections.abc from traitlets import HasTraits, Float, Tuple, observe from traitlets.utils.bunch import Bunch +from IS2view.utilities import import_dependency # attempt imports -try: - import geopandas as gpd -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("geopandas not available") -try: - import ipywidgets -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("ipywidgets not available") -try: - import ipyleaflet -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("ipyleaflet not available") +datatree = import_dependency('datatree') +gpd = import_dependency('geopandas') +ipywidgets = import_dependency('ipywidgets') +ipyleaflet = import_dependency('ipyleaflet') +wms = import_dependency('owslib.wms') +riotransform = import_dependency('rasterio.transform') +riowarp = import_dependency('rasterio.warp') +xr = import_dependency('xarray') +xyzservices = import_dependency('xyzservices') + +# attempt matplotlib imports try: import matplotlib import matplotlib.cm as cm @@ -81,23 +83,6 @@ matplotlib.rcParams['mathtext.default'] = 'regular' except (AttributeError, ImportError, ModuleNotFoundError) as exc: logging.critical("matplotlib not available") -try: - import owslib.wms -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("owslib not available") -try: - import rasterio.transform - import rasterio.warp -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("rasterio not available") -try: - import xarray as xr -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("xarray not available") -try: - import xyzservices -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("xyzservices not available") # set environmental variable for anonymous s3 access os.environ['AWS_NO_SIGN_REQUEST'] = 'YES' @@ -230,6 +215,8 @@ def _tile_provider(provider): # create traitlets of basemap providers basemaps = _load_dict(providers) +# set default map dimensions +_default_layout = ipywidgets.Layout(width='70%', height='600px') # draw ipyleaflet map class Leaflet: @@ -241,6 +228,8 @@ class Leaflet: ``ipyleaflet.Map`` basemap : obj or NoneType Basemap for the ``ipyleaflet.Map`` + layout : obj, default ``ipywidgets.Layout(width='70%', height='600px')`` + Layout for the ``ipyleaflet.Map`` attribution : bool, default False Include layer attributes on leaflet map scale_control : bool, default False @@ -280,6 +269,7 @@ class Leaflet: def __init__(self, projection, **kwargs): # set default keyword arguments kwargs.setdefault('map', None) + kwargs.setdefault('layout', _default_layout) kwargs.setdefault('attribution', False) kwargs.setdefault('full_screen_control', False) kwargs.setdefault('scale_control', False) @@ -300,7 +290,9 @@ def __init__(self, projection, **kwargs): zoom=kwargs['zoom'], max_zoom=5, attribution_control=kwargs['attribution'], basemap=kwargs['basemap'], - crs=projections['EPSG:3413']) + crs=projections['EPSG:3413'], + layout=kwargs['layout'] + ) self.crs = 'EPSG:3413' elif (projection == 'South'): kwargs.setdefault('basemap', @@ -310,7 +302,9 @@ def __init__(self, projection, **kwargs): zoom=kwargs['zoom'], max_zoom=5, attribution_control=kwargs['attribution'], basemap=kwargs['basemap'], - crs=projections['EPSG:3031']) + crs=projections['EPSG:3031'], + layout=kwargs['layout'] + ) self.crs = 'EPSG:3031' else: # use a predefined ipyleaflet map @@ -535,8 +529,8 @@ def plot_basemap(self, ax=None, **kwargs): # https://wiki.earthdata.nasa.gov/display/GIBS # https://worldview.earthdata.nasa.gov/ url = f'https://gibs.earthdata.nasa.gov/wms/{srs}/best/wms.cgi?' - wms = owslib.wms.WebMapService(url=url, version='1.1.1') - basemap = wms.getmap(**kwargs) + mappingservice = wms.WebMapService(url=url, version='1.1.1') + basemap = mappingservice.getmap(**kwargs) # read WMS layer and plot img = plt.imread(io.BytesIO(basemap.read())) ax.imshow(img, extent=[bbox[0],bbox[2],bbox[1],bbox[3]]) @@ -777,12 +771,8 @@ def plot(self, m, **kwargs): # reduce to variable and lag self._variable = copy.copy(kwargs['variable']) self.lag = int(kwargs['lag']) - if (self._ds[self._variable].ndim == 3) and ('time' in self._ds[self._variable].dims): - self._ds_selected = self._ds[self._variable].sel(time=self._ds.time[self.lag]) - elif (self._ds[self._variable].ndim == 3) and ('band' in self._ds[self._variable].dims): - self._ds_selected = self._ds[self._variable].sel(band=1) - else: - self._ds_selected = self._ds[self._variable] + # select data variable + self.set_dataset() # get the normalization bounds self.get_norm_bounds(**kwargs) # create matplotlib normalization @@ -911,7 +901,7 @@ def get_bounds(self): """get the bounds of the leaflet map in geographical coordinates """ self.get_bbox() - lon, lat = rasterio.warp.transform( + lon, lat = riowarp.transform( self.crs['name'], 'EPSG:4326', [self.sw['x'], self.ne['x']], [self.sw['y'], self.ne['y']]) @@ -1000,7 +990,7 @@ def clip_image(self, ds): # attempt to get the coordinate reference system of the dataset self.get_crs() # convert map bounds to coordinate reference system of image - minx, miny, maxx, maxy = rasterio.warp.transform_bounds( + minx, miny, maxx, maxy = riowarp.transform_bounds( self.crs['name'], self._ds.rio.crs, self.sw['x'], self.sw['y'], self.ne['x'], self.ne['y']) @@ -1023,14 +1013,14 @@ def clip_image(self, ds): # warp image to map bounds and resolution # input and output affine transformations src_transform = ds.rio.transform() - dst_transform = rasterio.transform.from_origin(minx, maxy, + dst_transform = riotransform.from_origin(minx, maxy, self.resolution, self.resolution) # allocate for output warped image dst_width = int((maxx - minx)//self.resolution) dst_height = int((maxy - miny)//self.resolution) dst_data = np.zeros((dst_height, dst_width), dtype=ds.dtype.type) # warp image to output resolution - rasterio.warp.reproject(source=ds.values, destination=dst_data, + riowarp.reproject(source=ds.values, destination=dst_data, src_transform=src_transform, src_crs=self._ds.rio.crs, src_nodata=np.nan, @@ -1148,14 +1138,10 @@ def set_observables(self, widget, **kwargs): except (AttributeError, NameError, ValueError) as exc: pass - def set_variable(self, sender): - """update the dataframe variable for a new selected variable + def set_dataset(self): + """Select the dataset for the selected variable + and time lag """ - # only update variable if a new final - if isinstance(sender['new'], str): - self._variable = sender['new'] - else: - return # reduce to variable and lag if (self._ds[self._variable].ndim == 3) and ('time' in self._ds[self._variable].dims): self._ds_selected = self._ds[self._variable].sel(time=self._ds.time[self.lag]) @@ -1163,6 +1149,17 @@ def set_variable(self, sender): self._ds_selected = self._ds[self._variable].sel(band=1) else: self._ds_selected = self._ds[self._variable] + + def set_variable(self, sender): + """update the plotted variable + """ + # only update variable if a new final + if isinstance(sender['new'], str): + self._variable = sender['new'] + else: + return + # reduce to variable and lag + self.set_dataset() # check if dynamic normalization is enabled if self._dynamic: self.get_norm_bounds() @@ -1263,7 +1260,7 @@ def handle_click(self, **kwargs): else: self._ds.rio.set_crs(crs) # get the clicked point in dataset coordinate reference system - x, y = rasterio.warp.transform('EPSG:4326', crs, [lon], [lat]) + x, y = riowarp.transform('EPSG:4326', crs, [lon], [lat]) # find nearest point in dataset self._data = self._ds_selected.sel(x=x, y=y, method='nearest').values[0] self._units = self._ds[self._variable].attrs['units'] @@ -1362,6 +1359,209 @@ def imshow(self, ax=None, **kwargs): ax.set_ylim(self.extent[2], self.extent[3]) ax.set_aspect('equal', adjustable='box') +@datatree.register_datatree_accessor('leaflet') +class TreeLeafletMap(LeafletMap): + """A datatree.DataTree extension for interactive map plotting, based on ipyleaflet + + Parameters + ---------- + dt : obj + ``datatree.DataTree`` + + Attributes + ---------- + _ds : obj + ``xarray.Dataset`` for selected group + _ds_selected : obj + ``xarray.Dataset`` for selected variable + _variable : str + Selected variable + map : obj + ``ipyleaflet.Map`` + crs : str + Coordinate Reference System of map + left, top, right, bottom : float + Map bounds in image coordinates + sw : dict + Location of lower-left corner in projected coordinates + ne : dict + Location of upper-right corner in projected coordinates + bounds : tuple + Location of map bounds in geographical coordinates + image : obj + ``ipyleaflet.ImageService`` layer for variable + cmap : obj + Matplotlib colormap object + norm : obj + Matplotlib normalization object + opacity : float + Transparency of image service layer + colorbar : obj + ``ipyleaflet.WidgetControl`` with Matplotlib colorbar + popup : obj + ``ipyleaflet.Popup`` with value at clicked location + _data : float + Variable value at clicked location + _units : str + Units of selected variable + """ + np.seterr(invalid='ignore') + def __init__(self, dt): + super().__init__(dt) + # initialize datatree and dataset + self._dt = dt + self._ds = None + + # add imagery data to leaflet map + def plot(self, m, **kwargs): + """Creates image plots on leaflet maps + + Parameters + ---------- + m : obj + leaflet map to add the layer + group : str or NoneType, default None + DataTree group to plot + variable : str, default 'delta_h' + xarray variable to plot + lag : int, default 0 + Time lag to plot if 3-dimensional + cmap : str, default 'viridis' + matplotlib colormap + vmin : float or NoneType + Minimum value for normalization + vmax : float or NoneType + Maximum value for normalization + norm : obj or NoneType + Matplotlib color normalization object + opacity : float, default 1.0 + Opacity of image plot + enable_popups : bool, default False + Enable contextual popups + colorbar : bool, decault True + Show colorbar for rendered variable + position : str, default 'topright' + Position of colorbar on leaflet map + """ + kwargs.setdefault('group', 'delta_h') + kwargs.setdefault('variable', 'delta_h') + kwargs.setdefault('lag', 0) + kwargs.setdefault('cmap', 'viridis') + kwargs.setdefault('vmin', None) + kwargs.setdefault('vmax', None) + kwargs.setdefault('norm', None) + kwargs.setdefault('opacity', 1.0) + kwargs.setdefault('enable_popups', False) + kwargs.setdefault('colorbar', True) + kwargs.setdefault('position', 'topright') + # set map and map coordinate reference system + self.map = m + crs = m.crs['name'] + self.crs = projections[crs] + (self.left, self.top), (self.right, self.bottom) = self.map.pixel_bounds + # enable contextual popups + self.enable_popups = bool(kwargs['enable_popups']) + # reduce to variable and lag + self._group = copy.copy(kwargs['group']) + self._variable = copy.copy(kwargs['variable']) + self.lag = int(kwargs['lag']) + # select data variable + self.set_dataset() + # get the normalization bounds + self.get_norm_bounds(**kwargs) + # create matplotlib normalization + if kwargs['norm'] is None: + self.norm = colors.Normalize(vmin=self.vmin, vmax=self.vmax, clip=True) + else: + self.norm = copy.copy(kwargs['norm']) + # get colormap + self.cmap = copy.copy(cm.get_cmap(kwargs['cmap'])) + # get opacity + self.opacity = float(kwargs['opacity']) + # wait for changes + asyncio.ensure_future(self.async_wait_for_bounds()) + self._image = ipyleaflet.ImageService( + name=self._variable, + crs=self.crs, + interactive=True, + update_interval=100, + endpoint='local') + # add click handler for popups + if self.enable_popups: + self._image.on_click(self.handle_click) + # set the image url + self.set_image_url() + # add image object to map + self.add(self._image) + # add colorbar + self.colorbar = kwargs['colorbar'] + self.colorbar_position = kwargs['position'] + if self.colorbar: + self.add_colorbar( + label=self._variable, + cmap=self.cmap, + opacity=self.opacity, + norm=self.norm, + position=self.colorbar_position + ) + + # observe changes in widget parameters + def set_observables(self, widget, **kwargs): + """observe changes in widget parameters + """ + # set default keyword arguments + # to map widget changes to functions + kwargs.setdefault('group', [self.set_group]) + kwargs.setdefault('variable', [self.set_variable]) + kwargs.setdefault('timelag', [self.set_lag]) + kwargs.setdefault('range', [self.set_norm]) + kwargs.setdefault('dynamic', [self.set_dynamic]) + kwargs.setdefault('cmap', [self.set_colormap]) + kwargs.setdefault('reverse', [self.set_colormap]) + # connect each widget with a set function + for key, val in kwargs.items(): + # try to retrieve the functional + try: + observable = getattr(widget, key) + except AttributeError as exc: + continue + # assert that observable is an ipywidgets object + assert isinstance(observable, ipywidgets.widgets.widget.Widget) + assert hasattr(observable, 'observe') + # for each functional to map + for i, functional in enumerate(val): + # try to connect the widget to the functional + try: + observable.observe(functional) + except (AttributeError, NameError, ValueError) as exc: + pass + + def set_dataset(self): + """Select the dataset for the selected variable and time lag + """ + # reduce to group if applicable and convert to dataset + self._ds = self._dt[self._group].to_dataset() + # check if variable is in dataset + # if not, wait for change + if self._variable not in self._ds: + self.wait_for_change(self._variable, 'value') + # reduce to variable and lag + if (self._ds[self._variable].ndim == 3) and ('time' in self._ds[self._variable].dims): + self._ds_selected = self._ds[self._variable].sel(time=self._ds.time[self.lag]) + elif (self._ds[self._variable].ndim == 3) and ('band' in self._ds[self._variable].dims): + self._ds_selected = self._ds[self._variable].sel(band=1) + else: + self._ds_selected = self._ds[self._variable] + + def set_group(self, sender): + """update the plotted variable for a group + """ + # only update variable if a new final + if isinstance(sender['new'], str): + self._group = sender['new'] + else: + return + @xr.register_dataset_accessor('timeseries') class TimeSeries(HasTraits): """A xarray.DataArray extension for extracting and plotting a time series @@ -1575,7 +1775,7 @@ def point(self, ax, **kwargs): """ # convert point to dataset coordinate reference system lon, lat = self.geometry['coordinates'] - x, y = rasterio.warp.transform(self.crs, self._ds.rio.crs, [lon], [lat]) + x, y = riowarp.transform(self.crs, self._ds.rio.crs, [lon], [lat]) # output time series for point self._data = np.zeros_like(self._ds.time) # reduce dataset to geometry @@ -1629,7 +1829,7 @@ def transect(self, ax, **kwargs): """ # convert linestring to dataset coordinate reference system lon, lat = np.transpose(self.geometry['coordinates']) - x, y = rasterio.warp.transform(self.crs, self._ds.rio.crs, lon, lat) + x, y = riowarp.transform(self.crs, self._ds.rio.crs, lon, lat) # get coordinates of each grid cell gridx, gridy = np.meshgrid(self._ds.x, self._ds.y) # clip ice area to geometry @@ -1997,7 +2197,7 @@ def transect(self, ax, **kwargs): """ # convert linestring to dataset coordinate reference system lon, lat = np.transpose(self.geometry['coordinates']) - x, y = rasterio.warp.transform(self.crs, self._ds.rio.crs, lon, lat) + x, y = riowarp.transform(self.crs, self._ds.rio.crs, lon, lat) # get coordinates of each grid cell gridx, gridy = np.meshgrid(self._ds.x, self._ds.y) # clip variable to geometry and create mask diff --git a/IS2view/convert.py b/IS2view/convert.py index 6cabea9..4d2d08b 100644 --- a/IS2view/convert.py +++ b/IS2view/convert.py @@ -23,16 +23,11 @@ import logging import pathlib import numpy as np +from IS2view.utilities import import_dependency # attempt imports -try: - import h5netcdf -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("h5netcdf not available") -try: - import xarray as xr -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("xarray not available") +h5netcdf = import_dependency('h5netcdf') +xr = import_dependency('xarray') # default groups to skip _default_skip_groups = ('METADATA', 'orbit_info', 'quality_assessment',) diff --git a/IS2view/io.py b/IS2view/io.py index 00808c3..3e6eaef 100644 --- a/IS2view/io.py +++ b/IS2view/io.py @@ -1,9 +1,11 @@ """ io.py -Written by Tyler Sutterley (10/2023) +Written by Tyler Sutterley (05/2024) Utilities for reading gridded ICESat-2 files using rasterio and xarray PYTHON DEPENDENCIES: + datatree: Tree-like hierarchical data structure for xarray + https://xarray-datatree.readthedocs.io h5netcdf: Pythonic interface to netCDF4 via h5py https://h5netcdf.org/ numpy: Scientific Computing Tools For Python @@ -18,6 +20,8 @@ https://docs.xarray.dev/en/stable/ UPDATE HISTORY: + Updated 05/2024: use wrapper to importlib for optional dependencies + added function to use datatree to merge hierarchical datasets Updated 10/2023: use dask.delayed to read multiple files in parallel Updated 08/2023: use xarray h5netcdf to read files streaming from s3 add open_dataset function for opening multiple granules @@ -27,22 +31,14 @@ """ from __future__ import annotations import os -import logging +from IS2view.utilities import import_dependency # attempt imports -try: - import rioxarray - import rioxarray.merge -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("rioxarray not available") -try: - import dask -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("dask not available") -try: - import xarray as xr -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.critical("xarray not available") +datatree = import_dependency('datatree') +rioxarray = import_dependency('rioxarray') +riomerge = import_dependency('rioxarray.merge') +dask = import_dependency('dask') +xr = import_dependency('xarray') # set environmental variable for anonymous s3 access os.environ['AWS_NO_SIGN_REQUEST'] = 'YES' @@ -50,6 +46,44 @@ # default engine for xarray _default_engine = dict(nc='h5netcdf', zarr='zarr') +def open_datatree(granule, + name: str | None = None, + **kwargs + ): + """ + Converts a dataset dictionary to a ``DataTree`` + + Parameters + ---------- + granule: str or list + presigned url or path for granule(s) as a s3fs object + name: str or NoneType, default None + Name of the ``DataTree`` + kwargs: dict + Keyword arguments to pass to ``open_dataset`` + + Returns + ------- + dt: object + merged ``DataTree`` + """ + # create a dictionary of datasets + d = {} + # open the primary groups + for group in ['delta_h', 'dhdt_lag1']: + d[group] = open_dataset(granule, group=group, **kwargs) + # extend possible time lags to 16 years post-launch + for timelag in range(4, 68, 4): + group = f'dhdt_lag{timelag:d}' + # attempt to open the group + try: + d[group] = open_dataset(granule, group=group, **kwargs) + except: + break + # create a datatree object + dt = datatree.DataTree.from_dict(d, name=name) + return dt + def open_dataset(granule, group: str | None = None, format: str = 'nc', @@ -100,7 +134,7 @@ def open_dataset(granule, if parallel: datasets, closers = dask.compute(datasets, closers) # merge datasets - ds = rioxarray.merge.merge_datasets(datasets) + ds = riomerge.merge_datasets(datasets) else: # read a single granule ds = from_file(granule, diff --git a/IS2view/tools.py b/IS2view/tools.py index 8e2c89a..de06fb4 100644 --- a/IS2view/tools.py +++ b/IS2view/tools.py @@ -28,16 +28,12 @@ import copy import logging import numpy as np +from IS2view.utilities import import_dependency # attempt imports -try: - import ipywidgets -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("ipywidgets not available") -try: - import matplotlib.cm as cm -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("matplotlib not available") +datatree = import_dependency('datatree') +ipywidgets = import_dependency('ipywidgets') +cm = import_dependency('matplotlib.cm') # set environmental variable for anonymous s3 access os.environ['AWS_NO_SIGN_REQUEST'] = 'YES' @@ -166,7 +162,7 @@ def __init__(self, **kwargs): style=self.style, ) - # dropdown menu for selecting time lag to draw on map + # slider for selecting time lag to draw on map self.timelag = ipywidgets.IntSlider( description='Lag:', description_tooltip="Lag: time lag to draw on leaflet map", @@ -174,7 +170,7 @@ def __init__(self, **kwargs): style=self.style, ) - # dropdown menu for selecting time step to draw on map + # slider for selecting time step to draw on map self.timestep = ipywidgets.FloatSlider( description='Time:', description_tooltip="Time: time step to draw on leaflet map", @@ -182,6 +178,7 @@ def __init__(self, **kwargs): readout=True, readout_format='.2f', disabled=False, + continuous_update=False, style=self.style, ) @@ -199,6 +196,9 @@ def __init__(self, **kwargs): self.asset.observe(self.set_format_visibility) self.release.observe(self.set_groups) self.dynamic.observe(self.set_dynamic) + self.group.observe(self.set_variables) + self.group.observe(self.set_time_steps) + self.group.observe(self.set_atl15_defaults) self.variable.observe(self.set_time_visibility) self.timestep.observe(self.set_lag) @@ -381,7 +381,7 @@ def set_atl15_defaults(self, *args, **kwargs): # set default variable for group self.variable.value = variables[group] - def set_groups(self, sender): + def set_groups(self, *args): """sets the list of available groups for a release """ group_list = ['delta_h', 'dhdt_lag1', 'dhdt_lag4', 'dhdt_lag8'] @@ -423,11 +423,15 @@ def set_groups(self, sender): self.region.description_tooltip = description_tooltip def set_variables(self, *args): - """sets the list of available variables in a group + """sets the list of available variables """ - if any(args): + if isinstance(self.data_vars, dict): + # set list of available variables in group + group = self.group.value + self.variable.options = sorted(self.data_vars[group]) + elif isinstance(self.data_vars, list): # set list of available variables - self.variable.options = sorted(args[0].keys()) + self.variable.options = sorted(self.data_vars) else: # return to temporary defaults self.variable.options = ['delta_h', 'dhdt'] @@ -448,13 +452,44 @@ def set_dynamic(self, *args, **kwargs): self.range.value = [-5, 5] self.range.layout.display = 'inline-flex' - def set_time_steps(self, ds, epoch=2018.0): + def get_variables(self, d): + """ + Gets the available variables and time steps + + Parameters + ---------- + d : datatree.DataTree or xarray.Dataset + DataTree or xarray.Dataset object + """ + # check if a DataTree object + if isinstance(d, datatree.DataTree): + self.data_vars = {g.strip('/'):sorted(d[g].data_vars) + for g in d.groups if d[g].data_vars} + self.time_vars = {g.strip('/'):d[g].time.values + for g in d.groups if 'time' in d[g]} + else: + self.data_vars = sorted(d.data_vars) + self.time_vars = d.time.values if 'time' in d else None + # set the default groups + self.set_groups() + # set the default variables + self.set_variables() + # set the default time steps + self.set_time_steps() + + def set_time_steps(self, *args, epoch=2018.0): """sets available time range """ + if isinstance(self.time_vars, dict): + # check if a DataTree object + group = self.group.value + self.time = list(epoch + self.time_vars[group]/365.25) + elif isinstance(self.time_vars, (list, np.ndarray)): + # set list of available variables + self.time = list(epoch + self.time_vars/365.25) # try setting the min and max time step try: # convert time to units - self.time = list(epoch + ds.time.values/365.25) self.timestep.max = self.time[-1] self.timestep.min = self.time[0] self.timestep.value = self.time[0] diff --git a/IS2view/utilities.py b/IS2view/utilities.py index 3f342d4..b7d6401 100644 --- a/IS2view/utilities.py +++ b/IS2view/utilities.py @@ -1,10 +1,17 @@ #!/usr/bin/env python u""" utilities.py -Written by Tyler Sutterley (11/2023) +Written by Tyler Sutterley (05/2024) Download and management utilities +PYTHON DEPENDENCIES: + boto3: Amazon Web Services (AWS) SDK for Python + https://boto3.amazonaws.com/v1/documentation/api/latest/index.html + s3fs: FUSE-based file system backed by Amazon S3 + https://s3fs.readthedocs.io/en/latest/ + UPDATE HISTORY: + Updated 05/2024: add wrapper to importlib for optional dependencies Updated 11/2023: updated ssl context to fix deprecation error Updated 10/2023: filter CMR request type using regular expressions Updated 08/2023: added ATL14/15 Release-03 data products @@ -36,6 +43,7 @@ import pathlib import builtins import warnings +import importlib import posixpath import traceback import subprocess @@ -49,16 +57,6 @@ from urllib.parse import urlencode import urllib.request as urllib2 -# attempt imports -try: - import boto3 -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("boto3 not available") -try: - import s3fs -except (AttributeError, ImportError, ModuleNotFoundError) as exc: - logging.debug("s3fs not available") - # PURPOSE: get absolute path within a package from a relative path def get_data_path(relpath: list | str | pathlib.Path): """ @@ -78,6 +76,46 @@ def get_data_path(relpath: list | str | pathlib.Path): elif isinstance(relpath, (str, pathlib.Path)): return filepath.joinpath(relpath) +def import_dependency( + name: str, + extra: str = "", + raise_exception: bool = False + ): + """ + Import an optional dependency + + Adapted from ``pandas.compat._optional::import_optional_dependency`` + + Parameters + ---------- + name: str + Module name + extra: str, default "" + Additional text to include in the ``ImportError`` message + raise_exception: bool, default False + Raise an ``ImportError`` if the module is not found + + Returns + ------- + module: obj + Imported module + """ + # check if the module name is a string + msg = f"Invalid module name: '{name}'; must be a string" + assert isinstance(name, str), msg + # try to import the module + err = f"Missing optional dependency '{name}'. {extra}" + module = None + try: + module = importlib.import_module(name) + except (ImportError, ModuleNotFoundError) as exc: + if raise_exception: + raise ImportError(err) from exc + else: + logging.debug(err) + # return the module + return module + # PURPOSE: get the hash value of a file def get_hash( local: str | io.IOBase | pathlib.Path, @@ -265,6 +303,7 @@ def s3_client( response = urllib2.urlopen(request, timeout=timeout) cumulus = json.loads(response.read()) # get AWS client object + boto3 = import_dependency('boto3') client = boto3.client('s3', aws_access_key_id=cumulus['accessKeyId'], aws_secret_access_key=cumulus['secretAccessKey'], @@ -301,6 +340,7 @@ def s3_filesystem( response = urllib2.urlopen(request, timeout=timeout) cumulus = json.loads(response.read()) # get AWS file system session object + s3fs = import_dependency('s3fs') session = s3fs.S3FileSystem(anon=False, key=cumulus['accessKeyId'], secret=cumulus['secretAccessKey'], @@ -405,6 +445,7 @@ def generate_presigned_url( s3 presigned https url """ # generate a presigned URL for S3 object + boto3 = import_dependency('boto3') s3 = boto3.client('s3') try: response = s3.generate_presigned_url('get_object', diff --git a/notebooks/IS2-ATL15-Viewer.ipynb b/notebooks/IS2-ATL15-Viewer.ipynb index 5a9a2a8..165d8a4 100644 --- a/notebooks/IS2-ATL15-Viewer.ipynb +++ b/notebooks/IS2-ATL15-Viewer.ipynb @@ -32,7 +32,6 @@ "- Release: ATL15 data release (001, 002, 003)\n", "- Region: ATL15 data region (AA, A1, A2, A3, A4, CN, CS, GL, IS, RA, SV)\n", "- Resolution: ATL15 horizontal resolution (01km, 10km, 20km, 40km)\n", - "- Group: ATL15 data group to read from file\n", "- Format: ATL15 data format to read (nc, zarr)" ] }, @@ -49,7 +48,6 @@ " IS2widgets.release,\n", " IS2widgets.region,\n", " IS2widgets.resolution,\n", - " IS2widgets.group,\n", " IS2widgets.format,\n", "])" ] @@ -86,7 +84,7 @@ "metadata": {}, "source": [ "### Read and inspect ATL15 data\n", - "The selected group within ATL15 data will be read using [xarray](https://xarray.dev/) and [rioxarray](https://corteva.github.io/rioxarray/)." + "The ATL15 data will be read using [xarray](https://xarray.dev/) and [rioxarray](https://corteva.github.io/rioxarray/)." ] }, { @@ -95,10 +93,9 @@ "metadata": {}, "outputs": [], "source": [ - "ds = IS2view.open_dataset(granule,\n", - " group=IS2widgets.group.value,\n", + "dt = IS2view.open_datatree(granule,\n", " format=IS2widgets.format.value)\n", - "ds" + "dt" ] }, { @@ -129,10 +126,10 @@ " draw_control=True,\n", " attribution=False)\n", "# set plot attributes\n", - "IS2widgets.set_variables(ds)\n", + "IS2widgets.get_variables(dt)\n", "IS2widgets.set_atl15_defaults()\n", - "IS2widgets.set_time_steps(ds)\n", "wbox = IS2widgets.VBox([\n", + " IS2widgets.group,\n", " IS2widgets.variable,\n", " IS2widgets.timestep,\n", " IS2widgets.dynamic,\n", @@ -149,7 +146,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Add xarray dataset as an image service layer" + "### Add xarray datatree as an image service layer" ] }, { @@ -158,14 +155,15 @@ "metadata": {}, "outputs": [], "source": [ - "ds.leaflet.plot(m.map, lag=IS2widgets.lag,\n", + "dt.leaflet.plot(m.map, lag=IS2widgets.lag,\n", " vmin=IS2widgets.vmin, vmax=IS2widgets.vmax,\n", + " group=IS2widgets.group.value,\n", " variable=IS2widgets.variable.value,\n", " cmap=IS2widgets.colormap,\n", " opacity=0.75,\n", " enable_popups=False)\n", "# observe changes in widget parameters\n", - "ds.leaflet.set_observables(IS2widgets)" + "dt.leaflet.set_observables(IS2widgets)" ] }, { @@ -186,7 +184,8 @@ "outputs": [], "source": [ "for feature in m.geometries['features']:\n", - " ds.timeseries.plot(feature,\n", + " dt.timeseries.plot(feature, cmap='viridis', legend=True,\n", + " group=IS2widgets.group.value,\n", " variable=IS2widgets.variable.value,\n", " )" ] @@ -205,8 +204,13 @@ "metadata": {}, "outputs": [], "source": [ - "ds.leaflet.reset()" + "dt.leaflet.reset()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { From 83ea1041347b1cdb0915bc0464a0323c64599267 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 7 Jun 2024 10:47:59 -0700 Subject: [PATCH 2/8] feat: minor updates for R004 --- IS2view/__init__.py | 2 +- IS2view/api.py | 209 +----------------------------- IS2view/convert.py | 3 +- IS2view/io.py | 46 +------ IS2view/tools.py | 35 ++--- IS2view/utilities.py | 5 +- doc/source/user_guide/Recipes.rst | 7 + notebooks/IS2-ATL14-Viewer.ipynb | 19 +-- notebooks/IS2-ATL15-Viewer.ipynb | 40 ++---- 9 files changed, 38 insertions(+), 328 deletions(-) diff --git a/IS2view/__init__.py b/IS2view/__init__.py index 1cb4469..e66f808 100644 --- a/IS2view/__init__.py +++ b/IS2view/__init__.py @@ -11,7 +11,7 @@ import IS2view.version from IS2view.api import Leaflet, basemaps, layers, image_service_layer from IS2view.convert import convert -from IS2view.io import open_datatree, open_dataset, from_file, from_rasterio, from_xarray +from IS2view.io import open_dataset, from_file, from_rasterio, from_xarray from IS2view.tools import widgets # get semantic version from setuptools-scm __version__ = IS2view.version.version diff --git a/IS2view/api.py b/IS2view/api.py index 5adf209..c5f5701 100644 --- a/IS2view/api.py +++ b/IS2view/api.py @@ -1,12 +1,10 @@ #!/usr/bin/env python u""" api.py -Written by Tyler Sutterley (04/2024) +Written by Tyler Sutterley (06/2024) Plotting tools for visualizing rioxarray variables on leaflet maps PYTHON DEPENDENCIES: - datatree: Tree-like hierarchical data structure for xarray - https://xarray-datatree.readthedocs.io geopandas: Python tools for geographic data http://geopandas.readthedocs.io/ ipywidgets: interactive HTML widgets for Jupyter notebooks and IPython @@ -30,6 +28,7 @@ https://xyzservices.readthedocs.io/en/stable/ UPDATE HISTORY: + Updated 06/2024: use wrapper to importlib for optional dependencies Updated 04/2024: add connections and functions for changing variables and other attributes of the leaflet map visualization simplify and generalize mapping between observables and functionals @@ -61,7 +60,6 @@ from IS2view.utilities import import_dependency # attempt imports -datatree = import_dependency('datatree') gpd = import_dependency('geopandas') ipywidgets = import_dependency('ipywidgets') ipyleaflet = import_dependency('ipyleaflet') @@ -1359,209 +1357,6 @@ def imshow(self, ax=None, **kwargs): ax.set_ylim(self.extent[2], self.extent[3]) ax.set_aspect('equal', adjustable='box') -@datatree.register_datatree_accessor('leaflet') -class TreeLeafletMap(LeafletMap): - """A datatree.DataTree extension for interactive map plotting, based on ipyleaflet - - Parameters - ---------- - dt : obj - ``datatree.DataTree`` - - Attributes - ---------- - _ds : obj - ``xarray.Dataset`` for selected group - _ds_selected : obj - ``xarray.Dataset`` for selected variable - _variable : str - Selected variable - map : obj - ``ipyleaflet.Map`` - crs : str - Coordinate Reference System of map - left, top, right, bottom : float - Map bounds in image coordinates - sw : dict - Location of lower-left corner in projected coordinates - ne : dict - Location of upper-right corner in projected coordinates - bounds : tuple - Location of map bounds in geographical coordinates - image : obj - ``ipyleaflet.ImageService`` layer for variable - cmap : obj - Matplotlib colormap object - norm : obj - Matplotlib normalization object - opacity : float - Transparency of image service layer - colorbar : obj - ``ipyleaflet.WidgetControl`` with Matplotlib colorbar - popup : obj - ``ipyleaflet.Popup`` with value at clicked location - _data : float - Variable value at clicked location - _units : str - Units of selected variable - """ - np.seterr(invalid='ignore') - def __init__(self, dt): - super().__init__(dt) - # initialize datatree and dataset - self._dt = dt - self._ds = None - - # add imagery data to leaflet map - def plot(self, m, **kwargs): - """Creates image plots on leaflet maps - - Parameters - ---------- - m : obj - leaflet map to add the layer - group : str or NoneType, default None - DataTree group to plot - variable : str, default 'delta_h' - xarray variable to plot - lag : int, default 0 - Time lag to plot if 3-dimensional - cmap : str, default 'viridis' - matplotlib colormap - vmin : float or NoneType - Minimum value for normalization - vmax : float or NoneType - Maximum value for normalization - norm : obj or NoneType - Matplotlib color normalization object - opacity : float, default 1.0 - Opacity of image plot - enable_popups : bool, default False - Enable contextual popups - colorbar : bool, decault True - Show colorbar for rendered variable - position : str, default 'topright' - Position of colorbar on leaflet map - """ - kwargs.setdefault('group', 'delta_h') - kwargs.setdefault('variable', 'delta_h') - kwargs.setdefault('lag', 0) - kwargs.setdefault('cmap', 'viridis') - kwargs.setdefault('vmin', None) - kwargs.setdefault('vmax', None) - kwargs.setdefault('norm', None) - kwargs.setdefault('opacity', 1.0) - kwargs.setdefault('enable_popups', False) - kwargs.setdefault('colorbar', True) - kwargs.setdefault('position', 'topright') - # set map and map coordinate reference system - self.map = m - crs = m.crs['name'] - self.crs = projections[crs] - (self.left, self.top), (self.right, self.bottom) = self.map.pixel_bounds - # enable contextual popups - self.enable_popups = bool(kwargs['enable_popups']) - # reduce to variable and lag - self._group = copy.copy(kwargs['group']) - self._variable = copy.copy(kwargs['variable']) - self.lag = int(kwargs['lag']) - # select data variable - self.set_dataset() - # get the normalization bounds - self.get_norm_bounds(**kwargs) - # create matplotlib normalization - if kwargs['norm'] is None: - self.norm = colors.Normalize(vmin=self.vmin, vmax=self.vmax, clip=True) - else: - self.norm = copy.copy(kwargs['norm']) - # get colormap - self.cmap = copy.copy(cm.get_cmap(kwargs['cmap'])) - # get opacity - self.opacity = float(kwargs['opacity']) - # wait for changes - asyncio.ensure_future(self.async_wait_for_bounds()) - self._image = ipyleaflet.ImageService( - name=self._variable, - crs=self.crs, - interactive=True, - update_interval=100, - endpoint='local') - # add click handler for popups - if self.enable_popups: - self._image.on_click(self.handle_click) - # set the image url - self.set_image_url() - # add image object to map - self.add(self._image) - # add colorbar - self.colorbar = kwargs['colorbar'] - self.colorbar_position = kwargs['position'] - if self.colorbar: - self.add_colorbar( - label=self._variable, - cmap=self.cmap, - opacity=self.opacity, - norm=self.norm, - position=self.colorbar_position - ) - - # observe changes in widget parameters - def set_observables(self, widget, **kwargs): - """observe changes in widget parameters - """ - # set default keyword arguments - # to map widget changes to functions - kwargs.setdefault('group', [self.set_group]) - kwargs.setdefault('variable', [self.set_variable]) - kwargs.setdefault('timelag', [self.set_lag]) - kwargs.setdefault('range', [self.set_norm]) - kwargs.setdefault('dynamic', [self.set_dynamic]) - kwargs.setdefault('cmap', [self.set_colormap]) - kwargs.setdefault('reverse', [self.set_colormap]) - # connect each widget with a set function - for key, val in kwargs.items(): - # try to retrieve the functional - try: - observable = getattr(widget, key) - except AttributeError as exc: - continue - # assert that observable is an ipywidgets object - assert isinstance(observable, ipywidgets.widgets.widget.Widget) - assert hasattr(observable, 'observe') - # for each functional to map - for i, functional in enumerate(val): - # try to connect the widget to the functional - try: - observable.observe(functional) - except (AttributeError, NameError, ValueError) as exc: - pass - - def set_dataset(self): - """Select the dataset for the selected variable and time lag - """ - # reduce to group if applicable and convert to dataset - self._ds = self._dt[self._group].to_dataset() - # check if variable is in dataset - # if not, wait for change - if self._variable not in self._ds: - self.wait_for_change(self._variable, 'value') - # reduce to variable and lag - if (self._ds[self._variable].ndim == 3) and ('time' in self._ds[self._variable].dims): - self._ds_selected = self._ds[self._variable].sel(time=self._ds.time[self.lag]) - elif (self._ds[self._variable].ndim == 3) and ('band' in self._ds[self._variable].dims): - self._ds_selected = self._ds[self._variable].sel(band=1) - else: - self._ds_selected = self._ds[self._variable] - - def set_group(self, sender): - """update the plotted variable for a group - """ - # only update variable if a new final - if isinstance(sender['new'], str): - self._group = sender['new'] - else: - return - @xr.register_dataset_accessor('timeseries') class TimeSeries(HasTraits): """A xarray.DataArray extension for extracting and plotting a time series diff --git a/IS2view/convert.py b/IS2view/convert.py index 4d2d08b..cb7f8d2 100644 --- a/IS2view/convert.py +++ b/IS2view/convert.py @@ -1,6 +1,6 @@ """ convert.py -Written by Tyler Sutterley (08/2023) +Written by Tyler Sutterley (06/2024) Utilities for converting gridded ICESat-2 files from native netCDF4 PYTHON DEPENDENCIES: @@ -13,6 +13,7 @@ https://docs.xarray.dev/en/stable/ UPDATE HISTORY: + Updated 06/2024: use wrapper to importlib for optional dependencies Updated 08/2023: use h5netcdf as the netCDF4 driver for xarray Updated 07/2023: use logging instead of warnings for import attempts Updated 06/2023: using pathlib to define and expand paths diff --git a/IS2view/io.py b/IS2view/io.py index 3e6eaef..e9abed3 100644 --- a/IS2view/io.py +++ b/IS2view/io.py @@ -1,11 +1,9 @@ """ io.py -Written by Tyler Sutterley (05/2024) +Written by Tyler Sutterley (06/2024) Utilities for reading gridded ICESat-2 files using rasterio and xarray PYTHON DEPENDENCIES: - datatree: Tree-like hierarchical data structure for xarray - https://xarray-datatree.readthedocs.io h5netcdf: Pythonic interface to netCDF4 via h5py https://h5netcdf.org/ numpy: Scientific Computing Tools For Python @@ -20,8 +18,7 @@ https://docs.xarray.dev/en/stable/ UPDATE HISTORY: - Updated 05/2024: use wrapper to importlib for optional dependencies - added function to use datatree to merge hierarchical datasets + Updated 06/2024: use wrapper to importlib for optional dependencies Updated 10/2023: use dask.delayed to read multiple files in parallel Updated 08/2023: use xarray h5netcdf to read files streaming from s3 add open_dataset function for opening multiple granules @@ -34,7 +31,6 @@ from IS2view.utilities import import_dependency # attempt imports -datatree = import_dependency('datatree') rioxarray = import_dependency('rioxarray') riomerge = import_dependency('rioxarray.merge') dask = import_dependency('dask') @@ -46,44 +42,6 @@ # default engine for xarray _default_engine = dict(nc='h5netcdf', zarr='zarr') -def open_datatree(granule, - name: str | None = None, - **kwargs - ): - """ - Converts a dataset dictionary to a ``DataTree`` - - Parameters - ---------- - granule: str or list - presigned url or path for granule(s) as a s3fs object - name: str or NoneType, default None - Name of the ``DataTree`` - kwargs: dict - Keyword arguments to pass to ``open_dataset`` - - Returns - ------- - dt: object - merged ``DataTree`` - """ - # create a dictionary of datasets - d = {} - # open the primary groups - for group in ['delta_h', 'dhdt_lag1']: - d[group] = open_dataset(granule, group=group, **kwargs) - # extend possible time lags to 16 years post-launch - for timelag in range(4, 68, 4): - group = f'dhdt_lag{timelag:d}' - # attempt to open the group - try: - d[group] = open_dataset(granule, group=group, **kwargs) - except: - break - # create a datatree object - dt = datatree.DataTree.from_dict(d, name=name) - return dt - def open_dataset(granule, group: str | None = None, format: str = 'nc', diff --git a/IS2view/tools.py b/IS2view/tools.py index de06fb4..82b0116 100644 --- a/IS2view/tools.py +++ b/IS2view/tools.py @@ -15,6 +15,7 @@ https://github.com/matplotlib/matplotlib UPDATE HISTORY: + Updated 06/2024: use wrapper to importlib for optional dependencies Updated 11/2023: set time steps using decimal years rather than lags setting dynamic colormap with float64 min and max Updated 08/2023: added options for ATL14/15 Release-03 data @@ -31,7 +32,6 @@ from IS2view.utilities import import_dependency # attempt imports -datatree = import_dependency('datatree') ipywidgets = import_dependency('ipywidgets') cm = import_dependency('matplotlib.cm') @@ -196,9 +196,6 @@ def __init__(self, **kwargs): self.asset.observe(self.set_format_visibility) self.release.observe(self.set_groups) self.dynamic.observe(self.set_dynamic) - self.group.observe(self.set_variables) - self.group.observe(self.set_time_steps) - self.group.observe(self.set_atl15_defaults) self.variable.observe(self.set_time_visibility) self.timestep.observe(self.set_lag) @@ -425,11 +422,7 @@ def set_groups(self, *args): def set_variables(self, *args): """sets the list of available variables """ - if isinstance(self.data_vars, dict): - # set list of available variables in group - group = self.group.value - self.variable.options = sorted(self.data_vars[group]) - elif isinstance(self.data_vars, list): + if isinstance(self.data_vars, list): # set list of available variables self.variable.options = sorted(self.data_vars) else: @@ -458,18 +451,12 @@ def get_variables(self, d): Parameters ---------- - d : datatree.DataTree or xarray.Dataset - DataTree or xarray.Dataset object + d : xarray.Dataset + xarray.Dataset object """ - # check if a DataTree object - if isinstance(d, datatree.DataTree): - self.data_vars = {g.strip('/'):sorted(d[g].data_vars) - for g in d.groups if d[g].data_vars} - self.time_vars = {g.strip('/'):d[g].time.values - for g in d.groups if 'time' in d[g]} - else: - self.data_vars = sorted(d.data_vars) - self.time_vars = d.time.values if 'time' in d else None + # data and time variables + self.data_vars = sorted(d.data_vars) + self.time_vars = d.time.values if 'time' in d else None # set the default groups self.set_groups() # set the default variables @@ -480,16 +467,10 @@ def get_variables(self, d): def set_time_steps(self, *args, epoch=2018.0): """sets available time range """ - if isinstance(self.time_vars, dict): - # check if a DataTree object - group = self.group.value - self.time = list(epoch + self.time_vars[group]/365.25) - elif isinstance(self.time_vars, (list, np.ndarray)): - # set list of available variables - self.time = list(epoch + self.time_vars/365.25) # try setting the min and max time step try: # convert time to units + self.time = list(epoch + self.time_vars/365.25) self.timestep.max = self.time[-1] self.timestep.min = self.time[0] self.timestep.value = self.time[0] diff --git a/IS2view/utilities.py b/IS2view/utilities.py index b7d6401..a36681e 100644 --- a/IS2view/utilities.py +++ b/IS2view/utilities.py @@ -1259,7 +1259,7 @@ def query_resources(**kwargs): - ``ATL14`` : land ice height - ``ATL15`` : land ice height change - release: str, default '001' + release: str, default '003' ICESat-2 data release version: str, default '01' ICESat-2 data version @@ -1335,7 +1335,7 @@ def query_resources(**kwargs): # verify inputs assert kwargs['asset'] in _assets assert kwargs['product'] in _products - assert kwargs['release'] in ('001', '002', '003') + assert kwargs['release'] in ('001', '002', '003', '004') if kwargs['cycles'] is not None: assert (len(kwargs['cycles']) == 2), 'cycles should be length 2' for r in kwargs['region']: @@ -1433,6 +1433,7 @@ def query_resources(**kwargs): cycles['001'] = (3, 11) cycles['002'] = (3, 14) cycles['003'] = (3, 18) + cycles['004'] = (3, 21) kwargs['cycles'] = cycles[kwargs['release']] # for each requested region for region in kwargs['region']: diff --git a/doc/source/user_guide/Recipes.rst b/doc/source/user_guide/Recipes.rst index f00c54a..cc55ff1 100644 --- a/doc/source/user_guide/Recipes.rst +++ b/doc/source/user_guide/Recipes.rst @@ -195,3 +195,10 @@ Requires optional ``geopandas`` and ``owslib`` dependencies. .. figure:: ../_assets/map.png :width: 600 :align: center + +Remove Image Service Layer from Map +################################### + +.. code-block:: python + + ds.leaflet.reset() diff --git a/notebooks/IS2-ATL14-Viewer.ipynb b/notebooks/IS2-ATL14-Viewer.ipynb index e06f2e1..5982313 100644 --- a/notebooks/IS2-ATL14-Viewer.ipynb +++ b/notebooks/IS2-ATL14-Viewer.ipynb @@ -29,7 +29,7 @@ "#### Set parameters for ATL14\n", "- Asset: Location to get the data\n", "- Directory: Working data directory\n", - "- Release: ATL14 data release (001, 002, 003)\n", + "- Release: ATL14 data release (001, 002, 003, 004)\n", "- Region: ATL14 data region (AA, A1, A2, A3, A4, CN, CS, GL, IS, RA, SV)\n", "- Format: ATL14 data format to read (nc, zarr)" ] @@ -182,23 +182,6 @@ " variable=IS2widgets.variable.value,\n", " )" ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Remove the image service layer from the map" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds.leaflet.reset()" - ] } ], "metadata": { diff --git a/notebooks/IS2-ATL15-Viewer.ipynb b/notebooks/IS2-ATL15-Viewer.ipynb index 165d8a4..80ac2c3 100644 --- a/notebooks/IS2-ATL15-Viewer.ipynb +++ b/notebooks/IS2-ATL15-Viewer.ipynb @@ -29,9 +29,10 @@ "#### Set parameters for ATL15\n", "- Asset: Location to get the data\n", "- Directory: Working data directory\n", - "- Release: ATL15 data release (001, 002, 003)\n", + "- Release: ATL15 data release (001, 002, 003, 004)\n", "- Region: ATL15 data region (AA, A1, A2, A3, A4, CN, CS, GL, IS, RA, SV)\n", "- Resolution: ATL15 horizontal resolution (01km, 10km, 20km, 40km)\n", + "- Group: ATL15 data group to read from file\n", "- Format: ATL15 data format to read (nc, zarr)" ] }, @@ -48,6 +49,7 @@ " IS2widgets.release,\n", " IS2widgets.region,\n", " IS2widgets.resolution,\n", + " IS2widgets.group,\n", " IS2widgets.format,\n", "])" ] @@ -84,7 +86,7 @@ "metadata": {}, "source": [ "### Read and inspect ATL15 data\n", - "The ATL15 data will be read using [xarray](https://xarray.dev/) and [rioxarray](https://corteva.github.io/rioxarray/)." + "The selected group within ATL15 data will be read using [xarray](https://xarray.dev/) and [rioxarray](https://corteva.github.io/rioxarray/)." ] }, { @@ -93,9 +95,10 @@ "metadata": {}, "outputs": [], "source": [ - "dt = IS2view.open_datatree(granule,\n", + "ds = IS2view.open_dataset(granule,\n", + " group=IS2widgets.group.value,\n", " format=IS2widgets.format.value)\n", - "dt" + "ds" ] }, { @@ -126,10 +129,9 @@ " draw_control=True,\n", " attribution=False)\n", "# set plot attributes\n", - "IS2widgets.get_variables(dt)\n", + "IS2widgets.get_variables(ds)\n", "IS2widgets.set_atl15_defaults()\n", "wbox = IS2widgets.VBox([\n", - " IS2widgets.group,\n", " IS2widgets.variable,\n", " IS2widgets.timestep,\n", " IS2widgets.dynamic,\n", @@ -146,7 +148,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Add xarray datatree as an image service layer" + "### Add xarray dataset as an image service layer" ] }, { @@ -155,7 +157,7 @@ "metadata": {}, "outputs": [], "source": [ - "dt.leaflet.plot(m.map, lag=IS2widgets.lag,\n", + "ds.leaflet.plot(m.map, lag=IS2widgets.lag,\n", " vmin=IS2widgets.vmin, vmax=IS2widgets.vmax,\n", " group=IS2widgets.group.value,\n", " variable=IS2widgets.variable.value,\n", @@ -163,7 +165,7 @@ " opacity=0.75,\n", " enable_popups=False)\n", "# observe changes in widget parameters\n", - "dt.leaflet.set_observables(IS2widgets)" + "ds.leaflet.set_observables(IS2widgets)" ] }, { @@ -184,29 +186,11 @@ "outputs": [], "source": [ "for feature in m.geometries['features']:\n", - " dt.timeseries.plot(feature, cmap='viridis', legend=True,\n", - " group=IS2widgets.group.value,\n", + " ds.timeseries.plot(feature, cmap='viridis', legend=True,\n", " variable=IS2widgets.variable.value,\n", " )" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Remove the image service layer from the map" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dt.leaflet.reset()" - ] - }, { "cell_type": "markdown", "metadata": {}, From a3c6b6f23d54a686e1074e42809a0159dbca21e8 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 7 Jun 2024 20:21:24 +0000 Subject: [PATCH 3/8] fix: update release widget --- IS2view/tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IS2view/tools.py b/IS2view/tools.py index 82b0116..37af79a 100644 --- a/IS2view/tools.py +++ b/IS2view/tools.py @@ -78,7 +78,7 @@ def __init__(self, **kwargs): self.directory.layout.display = 'none' # dropdown menu for setting ATL14/15 release - release_list = ['001', '002', '003'] + release_list = ['001', '002', '003', '004'] self.release = ipywidgets.Dropdown( options=release_list, value='003', @@ -86,7 +86,8 @@ def __init__(self, **kwargs): description_tooltip=("Release: ATL14/15 data release\n\t" "001: Release-01\n\t" "002: Release-02\n\t" - "003: Release-03"), + "003: Release-03\n\t" + "004: Release-04"), disabled=False, style=self.style, ) From 337e88845744258a24924ccce0344a06d0caf75b Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 21 Jun 2024 10:41:23 -0700 Subject: [PATCH 4/8] refactor: import top level dependencies --- IS2view/api.py | 14 ++++++++------ IS2view/io.py | 4 ++-- IS2view/utilities.py | 5 +++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/IS2view/api.py b/IS2view/api.py index c5f5701..410cf4d 100644 --- a/IS2view/api.py +++ b/IS2view/api.py @@ -63,9 +63,11 @@ gpd = import_dependency('geopandas') ipywidgets = import_dependency('ipywidgets') ipyleaflet = import_dependency('ipyleaflet') -wms = import_dependency('owslib.wms') -riotransform = import_dependency('rasterio.transform') -riowarp = import_dependency('rasterio.warp') +owslib = import_dependency('owslib') +owslib.wms = import_dependency('owslib.wms') +rio = import_dependency('rasterio') +rio.transform = import_dependency('rasterio.transform') +rio.warp = import_dependency('rasterio.warp') xr = import_dependency('xarray') xyzservices = import_dependency('xyzservices') @@ -500,7 +502,7 @@ def plot_basemap(self, ax=None, **kwargs): ax: obj, default None Figure axis kwargs: dict, default {} - Additional keyword arguments for ``wms.getmap`` + Additional keyword arguments for ``owslib.wms.getmap`` """ # set default keyword arguments kwargs.setdefault('layers', ['BlueMarble_NextGeneration']) @@ -527,8 +529,8 @@ def plot_basemap(self, ax=None, **kwargs): # https://wiki.earthdata.nasa.gov/display/GIBS # https://worldview.earthdata.nasa.gov/ url = f'https://gibs.earthdata.nasa.gov/wms/{srs}/best/wms.cgi?' - mappingservice = wms.WebMapService(url=url, version='1.1.1') - basemap = mappingservice.getmap(**kwargs) + wms = owslib.wms.WebMapService(url=url, version='1.1.1') + basemap = wms.getmap(**kwargs) # read WMS layer and plot img = plt.imread(io.BytesIO(basemap.read())) ax.imshow(img, extent=[bbox[0],bbox[2],bbox[1],bbox[3]]) diff --git a/IS2view/io.py b/IS2view/io.py index e9abed3..0159518 100644 --- a/IS2view/io.py +++ b/IS2view/io.py @@ -32,7 +32,7 @@ # attempt imports rioxarray = import_dependency('rioxarray') -riomerge = import_dependency('rioxarray.merge') +rioxarray.merge = import_dependency('rioxarray.merge') dask = import_dependency('dask') xr = import_dependency('xarray') @@ -92,7 +92,7 @@ def open_dataset(granule, if parallel: datasets, closers = dask.compute(datasets, closers) # merge datasets - ds = riomerge.merge_datasets(datasets) + ds = rioxarray.merge.merge_datasets(datasets) else: # read a single granule ds = from_file(granule, diff --git a/IS2view/utilities.py b/IS2view/utilities.py index a36681e..01cd354 100644 --- a/IS2view/utilities.py +++ b/IS2view/utilities.py @@ -103,9 +103,10 @@ def import_dependency( # check if the module name is a string msg = f"Invalid module name: '{name}'; must be a string" assert isinstance(name, str), msg - # try to import the module + # default error if module cannot be imported err = f"Missing optional dependency '{name}'. {extra}" - module = None + module = type('module', (), {}) + # try to import the module try: module = importlib.import_module(name) except (ImportError, ModuleNotFoundError) as exc: From 27cb89cd224e14a5bf766f03934595cf71ba477e Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 21 Jun 2024 10:44:01 -0700 Subject: [PATCH 5/8] Update api.py --- IS2view/api.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/IS2view/api.py b/IS2view/api.py index 410cf4d..ce17e77 100644 --- a/IS2view/api.py +++ b/IS2view/api.py @@ -901,7 +901,7 @@ def get_bounds(self): """get the bounds of the leaflet map in geographical coordinates """ self.get_bbox() - lon, lat = riowarp.transform( + lon, lat = rio.warp.transform( self.crs['name'], 'EPSG:4326', [self.sw['x'], self.ne['x']], [self.sw['y'], self.ne['y']]) @@ -990,7 +990,7 @@ def clip_image(self, ds): # attempt to get the coordinate reference system of the dataset self.get_crs() # convert map bounds to coordinate reference system of image - minx, miny, maxx, maxy = riowarp.transform_bounds( + minx, miny, maxx, maxy = rio.warp.transform_bounds( self.crs['name'], self._ds.rio.crs, self.sw['x'], self.sw['y'], self.ne['x'], self.ne['y']) @@ -1013,14 +1013,14 @@ def clip_image(self, ds): # warp image to map bounds and resolution # input and output affine transformations src_transform = ds.rio.transform() - dst_transform = riotransform.from_origin(minx, maxy, + dst_transform = rio.transform.from_origin(minx, maxy, self.resolution, self.resolution) # allocate for output warped image dst_width = int((maxx - minx)//self.resolution) dst_height = int((maxy - miny)//self.resolution) dst_data = np.zeros((dst_height, dst_width), dtype=ds.dtype.type) # warp image to output resolution - riowarp.reproject(source=ds.values, destination=dst_data, + rio.warp.reproject(source=ds.values, destination=dst_data, src_transform=src_transform, src_crs=self._ds.rio.crs, src_nodata=np.nan, @@ -1260,7 +1260,7 @@ def handle_click(self, **kwargs): else: self._ds.rio.set_crs(crs) # get the clicked point in dataset coordinate reference system - x, y = riowarp.transform('EPSG:4326', crs, [lon], [lat]) + x, y = rio.warp.transform('EPSG:4326', crs, [lon], [lat]) # find nearest point in dataset self._data = self._ds_selected.sel(x=x, y=y, method='nearest').values[0] self._units = self._ds[self._variable].attrs['units'] @@ -1572,7 +1572,7 @@ def point(self, ax, **kwargs): """ # convert point to dataset coordinate reference system lon, lat = self.geometry['coordinates'] - x, y = riowarp.transform(self.crs, self._ds.rio.crs, [lon], [lat]) + x, y = rio.warp.transform(self.crs, self._ds.rio.crs, [lon], [lat]) # output time series for point self._data = np.zeros_like(self._ds.time) # reduce dataset to geometry @@ -1626,7 +1626,7 @@ def transect(self, ax, **kwargs): """ # convert linestring to dataset coordinate reference system lon, lat = np.transpose(self.geometry['coordinates']) - x, y = riowarp.transform(self.crs, self._ds.rio.crs, lon, lat) + x, y = rio.warp.transform(self.crs, self._ds.rio.crs, lon, lat) # get coordinates of each grid cell gridx, gridy = np.meshgrid(self._ds.x, self._ds.y) # clip ice area to geometry @@ -1994,7 +1994,7 @@ def transect(self, ax, **kwargs): """ # convert linestring to dataset coordinate reference system lon, lat = np.transpose(self.geometry['coordinates']) - x, y = riowarp.transform(self.crs, self._ds.rio.crs, lon, lat) + x, y = rio.warp.transform(self.crs, self._ds.rio.crs, lon, lat) # get coordinates of each grid cell gridx, gridy = np.meshgrid(self._ds.x, self._ds.y) # clip variable to geometry and create mask From 879b01fcd6977417f696a216a3f93f46c56931b1 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 21 Jun 2024 10:46:11 -0700 Subject: [PATCH 6/8] Update IS2-ATL15-Viewer.ipynb --- notebooks/IS2-ATL15-Viewer.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/IS2-ATL15-Viewer.ipynb b/notebooks/IS2-ATL15-Viewer.ipynb index 80ac2c3..f6713d3 100644 --- a/notebooks/IS2-ATL15-Viewer.ipynb +++ b/notebooks/IS2-ATL15-Viewer.ipynb @@ -186,7 +186,7 @@ "outputs": [], "source": [ "for feature in m.geometries['features']:\n", - " ds.timeseries.plot(feature, cmap='viridis', legend=True,\n", + " ds.timeseries.plot(feature,\n", " variable=IS2widgets.variable.value,\n", " )" ] From 43b006b99157ada3e0a7eb65081d9115eea97995 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Mon, 22 Jul 2024 14:55:20 -0700 Subject: [PATCH 7/8] docs: bump docutils docs: use `importlib` rather than `pkg_resources` --- doc/environment.yml | 2 +- doc/source/conf.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/environment.yml b/doc/environment.yml index 0a1b843..fe70dac 100644 --- a/doc/environment.yml +++ b/doc/environment.yml @@ -2,7 +2,7 @@ name: is2view-docs channels: - conda-forge dependencies: - - docutils<0.18 + - docutils - graphviz - ipywidgets - notebook diff --git a/doc/source/conf.py b/doc/source/conf.py index 88e1588..076a5de 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -14,19 +14,18 @@ # import sys import datetime # sys.path.insert(0, os.path.abspath('.')) -from pkg_resources import get_distribution +import importlib.metadata - -# -- Project information ----------------------------------------------------- - -project = 'IS2view' +# package metadata +metadata = importlib.metadata.metadata("IS2view") +project = metadata["Name"] year = datetime.date.today().year copyright = f"2022\u2013{year}, Tyler C. Sutterley" -author = 'Tyler Sutterley' +author = 'Tyler C. Sutterley' # The full version, including alpha/beta/rc tags # get semantic version from setuptools-scm -version = get_distribution("IS2view").version +version = metadata["version"] # append "v" before the version release = f"v{version}" From b94cf0379dcaa5e83c1f66a5ba84da1d07928324 Mon Sep 17 00:00:00 2001 From: tsutterley Date: Tue, 30 Jul 2024 16:58:34 -0700 Subject: [PATCH 8/8] chore: bump version and add release notes refactor: bump default version to 004 --- IS2view/tools.py | 2 +- IS2view/utilities.py | 4 ++-- doc/source/release_notes/release-v0.0.9.rst | 10 ++++++++++ version.txt | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 doc/source/release_notes/release-v0.0.9.rst diff --git a/IS2view/tools.py b/IS2view/tools.py index 37af79a..0379538 100644 --- a/IS2view/tools.py +++ b/IS2view/tools.py @@ -81,7 +81,7 @@ def __init__(self, **kwargs): release_list = ['001', '002', '003', '004'] self.release = ipywidgets.Dropdown( options=release_list, - value='003', + value='004', description='Release:', description_tooltip=("Release: ATL14/15 data release\n\t" "001: Release-01\n\t" diff --git a/IS2view/utilities.py b/IS2view/utilities.py index 01cd354..2bfd69f 100644 --- a/IS2view/utilities.py +++ b/IS2view/utilities.py @@ -1260,7 +1260,7 @@ def query_resources(**kwargs): - ``ATL14`` : land ice height - ``ATL15`` : land ice height change - release: str, default '003' + release: str, default '004' ICESat-2 data release version: str, default '01' ICESat-2 data version @@ -1298,7 +1298,7 @@ def query_resources(**kwargs): kwargs.setdefault('bucket', 'is2view') kwargs.setdefault('directory', None) kwargs.setdefault('product', 'ATL15') - kwargs.setdefault('release', '003') + kwargs.setdefault('release', '004') kwargs.setdefault('version', '01') kwargs.setdefault('cycles', None) kwargs.setdefault('region', 'AA') diff --git a/doc/source/release_notes/release-v0.0.9.rst b/doc/source/release_notes/release-v0.0.9.rst new file mode 100644 index 0000000..10516af --- /dev/null +++ b/doc/source/release_notes/release-v0.0.9.rst @@ -0,0 +1,10 @@ +################## +`Release v0.0.9`__ +################## + +* ``refactor``: simplify and generalize mapping between observables and functionals (`#42 `_) +* ``feat``: minor updates for R004 (`#43 `_) +* ``docs``: bump ``docutils`` (`#43 `_) +* ``docs``: use ``importlib`` rather than ``pkg_resources`` (`#43 `_) + +.. __: https://github.com/tsutterley/IS2view/releases/tag/0.0.9 diff --git a/version.txt b/version.txt index d169b2f..c5d54ec 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.8 +0.0.9