Merge in devel for python 3.6--3.10 and ApRES improvements

.github/workflows/python-package.yml

@@ -12,13 +12,15 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         qt5: [true, false]
         gdal: [true, false]
         seisunix: [true, false]
         exclude:
           - gdal: true
             python-version: "3.9"  # This is broken on pip at [email protected]
+          - gdal: true
+            python-version: "3.10"  # This is broken on pip at [email protected]
           - os: macos-latest
             qt5: true
           - os: macos-latest

impdar/bin/apdar.py

@@ -0,0 +1,104 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# Copyright © 2021 David Lilien <[email protected]>
+#
+# Distributed under terms of the GNU GPL3.0 license.
+
+"""
+Make an executable for single actions of ApRES handling.
+"""
+import sys
+import os.path
+import argparse
+
+from impdar.lib.ApresData import load_apres
+
+
+def _get_args():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help='Choose a processing step')
+
+    # Horizontal window filter
+    parser_load = _add_procparser(subparsers,
+                                  'load',
+                                  'load apres data',
+                                  lambda x: x,
+                                  defname='load')
+    _add_def_args(parser_load)
+    return parser
+
+
+def _add_simple_procparser(subparsers, name, helpstr, func, defname='proc'):
+    """Add a sub parser that can do a simple thing with no arguments."""
+    parser = _add_procparser(subparsers, name, helpstr, func, defname=defname)
+    _add_def_args(parser)
+    return parser
+
+
+def _add_procparser(subparsers, name, helpstr, func, defname='proc'):
+    """Wrap adding subparser because we mostly want the same args."""
+    parser = subparsers.add_parser(name, help=helpstr)
+    parser.set_defaults(func=func, name=defname)
+    return parser
+
+
+def _add_def_args(parser):
+    """Set some default arguments common to the different processing types."""
+    parser.add_argument('fns',
+                        type=str,
+                        nargs='+',
+                        help='The files to process')
+    parser.add_argument('-o',
+                        type=str,
+                        help='Output to this file (folder if multiple inputs)')
+
+
+def main():
+    """Get arguments, process data, handle saving."""
+    parser = _get_args()
+    args = parser.parse_args(sys.argv[1:])
+    if not hasattr(args, 'func'):
+        parser.parse_args(['-h'])
+
+    apres_data = [load_apres.load_apres_single_file(fn) for fn in args.fns]
+
+    if args.name == 'load':
+        pass
+    else:
+        for dat in apres_data:
+            args.func(dat, **vars(args))
+
+    if args.name == 'load':
+        name = 'raw'
+    else:
+        name = args.name
+
+    if args.o is not None:
+        if ((len(apres_data) > 1) or (args.o[-1] == '/')):
+            for d, f in zip(apres_data, args.fns):
+                bn = os.path.split(os.path.splitext(f)[0])[1]
+                if bn[-4:] == '_raw':
+                    bn = bn[:-4]
+                out_fn = os.path.join(args.o, bn + '_{:s}.h5'.format(name))
+                d.save(out_fn)
+        else:
+            out_fn = args.o
+            apres_data[0].save(out_fn)
+    else:
+        for d, f in zip(apres_data, args.fns):
+            bn = os.path.splitext(f)[0]
+            if bn[-4:] == '_raw':
+                bn = bn[:-4]
+            out_fn = bn + '_{:s}.h5'.format(name)
+            d.save(out_fn)
+
+
+def crop(dat, lim=0, top_or_bottom='top', dimension='snum', **kwargs):
+    """Crop in the vertical."""
+    dat.crop(lim, top_or_bottom=top_or_bottom, dimension=dimension)
+
+
+if __name__ == '__main__':
+    main()

impdar/bin/impdarexec.py

@@ -38,6 +38,10 @@ def _get_args():
     parser_load.add_argument('-o', type=str, help='Write to this filename')
     parser_load.add_argument('--nans', type=str, choices=['interp', 'delete'], default=None,
                              help='Interpolate or delete bad GPS. Only used by BSI.')
+    parser_load.add_argument('-dname',
+                             type=str,
+                             help='Name of data field',
+                             default='data')
 
     # Options for processing data
     parser_proc = subparsers.add_parser('proc', help='Process data')

impdar/bin/impproc.py

@@ -390,7 +390,7 @@ def main():
             args.func(dat, **vars(args))
 
     if args.o is not None:
-        if len(radar_data) > 1:
+        if ((len(radar_data) > 1) or (args.o[-1] == '/')):
             for d, f in zip(radar_data, args.fns):
                 bn = os.path.split(os.path.splitext(f)[0])[1]
                 if bn[-4:] == '_raw':

impdar/lib/ApresData/ApresFlags.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # vim:fenc=utf-8
 #
-# Copyright © 2019 David Lilien <[email protected]>
+# Copyright © 2019 Benjamin Hills <[email protected]>
 #
 # Distributed under terms of the GNU GPL3.0 license.
 
@@ -11,6 +11,7 @@
 """
 
 import numpy as np
+import h5py
 
 class ApresFlags():
     """Flags that indicate the processing that has been used on the data.
@@ -21,35 +22,49 @@ class ApresFlags():
     ----------
     batch: bool
         Legacy indication of whether we are batch processing. Always False.
-    agc: bool
-        Automatic gain control has been applied.
-    reverse: bool
-        Data have been reversed.
-    restack: bool
-        Data have been restacked.
-    rgain: bool
-        Data have a linear range gain applied.
-    bpass: 3x1 :class:`numpy.ndarray`
-        Elements: (1) 1 if bandpassed; (2) Low; and (3) High (MHz) bounds
-    hfilt: 2x1 :class:`numpy.ndarray`
-        Elements: (1) 1 if horizontally filtered; (2) Filter type
-    interp: 2x1 :class:`numpy.ndarray`
-        Elements: (1) 1 if constant distance spacing applied (2) The constant spacing (m)
-    mig: 2x1 :class: String
-        None if no migration done, mtype if migration done.
+    range: float
+        max range
+    stack: int
+        number of chirps stacked
      """
 
     def __init__(self):
         self.file_read_code = None
         self.range = 0
         self.stack = 1
-        self.attrs = ['file_read_code','phase2range','stack']
-        self.attr_dims = [None,None,None]
+        self.attrs = ['file_read_code', 'range', 'stack']
+        self.attr_dims = [None, None, None]
+
+    def write_h5(self, grp):
+        """Write to a subgroup in hdf5 file
+
+        Parameters
+        ----------
+        grp: h5py.Group
+            The group to which the ApresFlags subgroup is written
+        """
+        subgrp = grp.create_group('ApresFlags')
+        for attr in self.attrs:
+            val = getattr(self, attr)
+            if val is None:
+                subgrp[attr] = h5py.Empty('f')
+            else:
+                if hasattr(val, 'dtype'):
+                    val = val.astype('f')
+                subgrp.attrs[attr] = val
+
+    def read_h5(self, grp):
+        subgrp = grp['ApresFlags']
+        for attr in subgrp.attrs.keys():
+            val = subgrp.attrs[attr]
+            if isinstance(val, h5py.Empty):
+                val = None
+            setattr(self, attr, val)
 
     def to_matlab(self):
         """Convert all associated attributes into a dictionary formatted for use with :func:`scipy.io.savemat`
         """
-        outmat = {att: getattr(self, att) for att in self.attrs}
+        outmat = {att: (getattr(self, att) if getattr(self, att) is not None else np.NaN) for att in self.attrs}
         return outmat
 
     def from_matlab(self, matlab_struct):

impdar/lib/ApresData/ApresHeader.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # vim:fenc=utf-8
 #
-# Copyright © 2019 David Lilien <[email protected]>
+# Copyright © 2019 Benjamin Hills <[email protected]>
 #
 # Distributed under terms of the GNU GPL3 license.
 
@@ -21,13 +21,12 @@
 Earth and Space Sciences
 
 Sept 23 2019
-
 """
 
 import numpy as np
+import h5py
 import re
 
-# --------------------------------------------------------------------------------------------
 
 class ApresHeader():
     """
@@ -54,6 +53,28 @@ def __init__(self):
         self.chirp_grad = None
         self.nchirp_samples = None
         self.ramp_dir = None
+        self.f1 = None
+        self.bandwidth = None
+        self.fc = None
+        self.er = None
+        self.ci = None
+        self.lambdac = None
+        self.n_attenuators = None
+        self.attenuator1 = None
+        self.attenuator2 = None
+        self.tx_ant = None
+        self.rx_ant = None
+
+        self.attrs = ['fsysclk','fs','fn','header_string','file_format','noDwellHigh','noDwellLow',
+                    'f0','f_stop','ramp_up_step','ramp_down_step','tstep_up','tstep_down','snum','nsteps_DDS',
+                    'chirp_length','chirp_grad','nchirp_samples','ramp_dir','f1','bandwidth','fc','er','ci','lambdac',
+                    'n_attenuators','attenuator1','attenuator2','tx_ant','rx_ant']
+        self.attr_dims = ['none','none','none','none','none',
+                        'none','none','none','none','none',
+                        'none','none','none','none','none',
+                        'none','none','none','none','none',
+                        'none','none','none','none','none',
+                        'none','none','none','none','none']
 
     # --------------------------------------------------------------------------------------------
 
@@ -67,17 +88,12 @@ def read_header(self,fn_apres,max_header_len=2000):
             file name to update with
         max_header_len: int
             maximum length of header to read (can be too long)
-
-        Output
-        ---------
         """
         self.fn = fn_apres
         fid = open(fn_apres,'rb')
         self.header_string = str(fid.read(max_header_len))
         fid.close()
 
-    # --------------------------------------------------------------------------------------------
-
     def get_file_format(self):
         """
         Determine fmcw file format from burst header using keyword presence
@@ -104,7 +120,6 @@ def update_parameters(self,fn_apres=None):
         """
         Update the parameters with the apres file header
 
-
         ### Original Matlab Notes ###
         Extract from the hex codes the actual paramaters used by RMB2
         The contents of config.ini are copied into a data header.
@@ -150,11 +165,6 @@ def update_parameters(self,fn_apres=None):
                 self.noDwellHigh = int(val[18])
                 self.noDwellLow = int(val[17])
 
-            #elif case == 'Reg08':
-            #    # Phase offset word Register (POW) Address 0x08. 2 Bytes dTheta = 360*POW/2^16.
-            #    val = char(reg{1,2}(k));
-            #    H.phaseOffsetDeg = hex2dec(val(1:4))*360/2^16;
-
             elif case == 'Reg0B':
                 # Digital Ramp Limit Register Address 0x0B
                 # Digital ramp upper limit 32-bit digital ramp upper limit value.
@@ -191,10 +201,10 @@ def update_parameters(self,fn_apres=None):
                 output[i] = self.header_string[search_start+len(string):search_end+search_start]
 
         self.fs = output[0]
-        if self.fs == 1:        # if self.fs > 70e3:
-            self.fs = 8e4       #     self.fs = 80e3
-        else:                   # else
-            self.fs = 4e4       #     self.fs = 40e3
+        if self.fs == 1:
+            self.fs = 8e4
+        else:
+            self.fs = 4e4
 
         self.snum = int(output[1])
 
@@ -217,12 +227,37 @@ def update_parameters(self,fn_apres=None):
             self.ramp_dir = 'upDown'
             self.nchirpsPerPeriod = np.nan # self.nchirpSamples/(self.chirpLength)
 
-# --------------------------------------------------------------------------------------------
+    def write_h5(self, grp):
+        """
+        Write to a subgroup in hdf5 file
+
+        Parameters
+        ----------
+        grp: h5py.Group
+            The group to which the ApresHeader subgroup is written
+        """
+        subgrp = grp.create_group('ApresHeader')
+        for attr in vars(self):
+            val = getattr(self, attr)
+            if val is None:
+                subgrp.attrs[attr] = h5py.Empty("f")
+            else:
+                if hasattr(val, 'dtype'):
+                    val = val.astype('f')
+                subgrp.attrs[attr] = val
+
+    def read_h5(self, grp):
+        subgrp = grp['ApresHeader']
+        for attr in subgrp.attrs.keys():
+            val = subgrp.attrs[attr]
+            if isinstance(val, h5py.Empty):
+                val = None
+            setattr(self, attr, val)
 
     def to_matlab(self):
         """Convert all associated attributes into a dictionary formatted for use with :func:`scipy.io.savemat`
         """
-        outmat = {att: getattr(self, att) for att in vars(self)}
+        outmat = {att: (getattr(self, att) if getattr(self, att) is not None else np.NaN) for att in vars(self)}
         return outmat
 
     def from_matlab(self, matlab_struct):
@@ -232,7 +267,7 @@ def from_matlab(self, matlab_struct):
             setattr(self, attr, matlab_struct[attr][0][0][0])
             # Use this because matlab inputs may have zeros for flags that
             # were lazily appended to be arrays, but we preallocate
-            if attr_dim is not None and getattr(self, attr).shape[0] == 1:
+            if attr_dim != 'none' and getattr(self, attr).shape[0] == 1:
                 setattr(self, attr, np.zeros((attr_dim, )))
 
         for attr in self.bool_attrs: