Add scaffolding for JSON options in HDF5

ax3l · Jun 23, 2021 · 2e01d59 · 2e01d59
1 parent a9b8c68
commit 2e01d59
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 17 deletions.
diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp
@@ -22,6 +22,8 @@
 
 #include "openPMD/IO/AbstractIOHandler.hpp"
 
+#include <nlohmann/json.hpp>
+
 #include <future>
 #include <memory>
 #include <string>
@@ -34,7 +36,7 @@ class HDF5IOHandlerImpl;
 class HDF5IOHandler : public AbstractIOHandler
 {
 public:
-    HDF5IOHandler(std::string path, Access);
+    HDF5IOHandler(std::string path, Access, nlohmann::json config);
     ~HDF5IOHandler() override;
 
     std::string backendName() const override { return "HDF5"; }

diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp
@@ -24,6 +24,7 @@
 #if openPMD_HAVE_HDF5
 #   include "openPMD/IO/AbstractIOHandlerImpl.hpp"
 
+#   include "openPMD/auxiliary/JSON.hpp"
 #   include "openPMD/auxiliary/Option.hpp"
 
 #   include <hdf5.h>
@@ -38,7 +39,7 @@ namespace openPMD
     class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
     {
     public:
-        HDF5IOHandlerImpl(AbstractIOHandler*);
+        HDF5IOHandlerImpl(AbstractIOHandler*, nlohmann::json config);
         ~HDF5IOHandlerImpl() override;
 
         void createFile(Writable*, Parameter< Operation::CREATE_FILE > const&) override;
@@ -77,6 +78,7 @@ namespace openPMD
         hid_t m_H5T_CLONG_DOUBLE;
 
     private:
+        auxiliary::TracingJSON m_config;
         struct File
         {
             std::string name;

diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp
@@ -23,6 +23,8 @@
 #include "openPMD/config.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 
+#include <nlohmann/json.hpp>
+
 #include <future>
 #include <memory>
 #include <string>
@@ -36,9 +38,10 @@ namespace openPMD
     {
     public:
     #if openPMD_HAVE_MPI
-        ParallelHDF5IOHandler(std::string path, Access, MPI_Comm);
+        ParallelHDF5IOHandler(
+            std::string path, Access, MPI_Comm, nlohmann::json config);
     #else
-        ParallelHDF5IOHandler(std::string path, Access);
+        ParallelHDF5IOHandler(std::string path, Access, nlohmann::json config);
     #endif
         ~ParallelHDF5IOHandler() override;
 

diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp
@@ -27,6 +27,7 @@
 #   include <mpi.h>
 #   if openPMD_HAVE_HDF5
 #       include "openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp"
+#       include <nlohmann/json.hpp>
 #   endif
 #endif
 
@@ -37,7 +38,8 @@ namespace openPMD
     class ParallelHDF5IOHandlerImpl : public HDF5IOHandlerImpl
     {
     public:
-        ParallelHDF5IOHandlerImpl(AbstractIOHandler*, MPI_Comm);
+        ParallelHDF5IOHandlerImpl(
+            AbstractIOHandler*, MPI_Comm, nlohmann::json config);
         ~ParallelHDF5IOHandlerImpl() override;
 
         MPI_Comm m_mpiComm;

diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp
@@ -45,7 +45,8 @@ namespace openPMD
         switch( format )
         {
             case Format::HDF5:
-                return std::make_shared< ParallelHDF5IOHandler >( path, access, comm );
+                return std::make_shared< ParallelHDF5IOHandler >(
+                    path, access, comm, std::move( optionsJson ) );
             case Format::ADIOS1:
 #   if openPMD_HAVE_ADIOS1
                 return std::make_shared< ParallelADIOS1IOHandler >( path, access, comm );
@@ -80,7 +81,8 @@ namespace openPMD
         switch( format )
         {
             case Format::HDF5:
-                return std::make_shared< HDF5IOHandler >( path, access );
+                return std::make_shared< HDF5IOHandler >(
+                    path, access, std::move( optionsJson ) );
             case Format::ADIOS1:
 #if openPMD_HAVE_ADIOS1
                 return std::make_shared< ADIOS1IOHandler >( path, access );

diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp
@@ -51,7 +51,8 @@ namespace openPMD
 #       define VERIFY(CONDITION, TEXT) do{ (void)sizeof(CONDITION); } while( 0 )
 #   endif
 
-HDF5IOHandlerImpl::HDF5IOHandlerImpl(AbstractIOHandler* handler)
+HDF5IOHandlerImpl::HDF5IOHandlerImpl(
+    AbstractIOHandler* handler, nlohmann::json config)
         : AbstractIOHandlerImpl(handler),
           m_datasetTransferProperty{H5P_DEFAULT},
           m_fileAccessProperty{H5P_DEFAULT},
@@ -82,6 +83,21 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl(AbstractIOHandler* handler)
     H5Tinsert(m_H5T_CDOUBLE, "i", sizeof(double), H5T_NATIVE_DOUBLE);
     H5Tinsert(m_H5T_CLONG_DOUBLE, "r", 0, H5T_NATIVE_LDOUBLE);
     H5Tinsert(m_H5T_CLONG_DOUBLE, "i", sizeof(long double), H5T_NATIVE_LDOUBLE);
+
+    if( config.contains( "hdf5" ) )
+    {
+        m_config = std::move( config[ "hdf5" ] );
+        /*
+         * @todo Apply global configuration options from the JSON config here.
+         */
+        auto shadow = m_config.invertShadow();
+        if( shadow.size() > 0 )
+        {
+            std::cerr << "Warning: parts of the JSON configuration for "
+                         "HDF5 remain unused:\n"
+                      << shadow << std::endl;
+        }
+    }
 }
 
 HDF5IOHandlerImpl::~HDF5IOHandlerImpl()
@@ -239,6 +255,25 @@ HDF5IOHandlerImpl::createDataset(Writable* writable,
         if( auxiliary::ends_with(name, '/') )
             name = auxiliary::replace_last(name, "/", "");
 
+        auto config = nlohmann::json::parse( parameters.options );
+        if( config.contains( "hdf5" ) &&
+            config[ "hdf5" ].contains( "dataset" ) )
+        {
+            auxiliary::TracingJSON datasetConfig{
+                config[ "hdf5" ][ "dataset" ] };
+            /*
+             * @todo Read options from config here.
+             */
+            auto shadow = datasetConfig.invertShadow();
+            if( shadow.size() > 0 )
+            {
+                std::cerr << "Warning: parts of the JSON configuration for "
+                             "HDF5 dataset '"
+                          << name << "' remain unused:\n"
+                          << shadow << std::endl;
+            }
+        }
+
         /* Open H5Object to write into */
         auto res = getFile( writable );
         File file = res ? res.get() : getFile( writable->parent ).get();
@@ -1843,9 +1878,9 @@ HDF5IOHandlerImpl::getFile( Writable * writable )
 #endif
 
 #if openPMD_HAVE_HDF5
-HDF5IOHandler::HDF5IOHandler(std::string path, Access at)
+HDF5IOHandler::HDF5IOHandler(std::string path, Access at, nlohmann::json config)
         : AbstractIOHandler(std::move(path), at),
-          m_impl{new HDF5IOHandlerImpl(this)}
+          m_impl{new HDF5IOHandlerImpl(this, std::move(config))}
 { }
 
 HDF5IOHandler::~HDF5IOHandler() = default;

diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp
@@ -38,11 +38,10 @@ namespace openPMD
 #       define VERIFY(CONDITION, TEXT) do{ (void)sizeof(CONDITION); } while( 0 )
 #   endif
 
-ParallelHDF5IOHandler::ParallelHDF5IOHandler(std::string path,
-                                             Access at,
-                                             MPI_Comm comm)
+ParallelHDF5IOHandler::ParallelHDF5IOHandler(
+    std::string path, Access at, MPI_Comm comm, nlohmann::json config )
         : AbstractIOHandler(std::move(path), at, comm),
-          m_impl{new ParallelHDF5IOHandlerImpl(this, comm)}
+          m_impl{new ParallelHDF5IOHandlerImpl(this, comm, std::move(config))}
 { }
 
 ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default;
@@ -53,9 +52,9 @@ ParallelHDF5IOHandler::flush()
     return m_impl->flush();
 }
 
-ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(AbstractIOHandler* handler,
-                                                     MPI_Comm comm)
-        : HDF5IOHandlerImpl{handler},
+ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(
+    AbstractIOHandler* handler, MPI_Comm comm, nlohmann::json config )
+        : HDF5IOHandlerImpl{handler, std::move(config)},
           m_mpiComm{comm},
           m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */
 {