Changes up to Deliverable 5.1.5 // PPStee v0.3.0d

 - Added command line parameter for selection of partitioner
 - ARCHER support in Fabric and CMake

--HG--
branch : ppstee

CMakeLists.txt

@@ -79,7 +79,7 @@ ExternalProject_Add(
 	-DHEMELB_WAIT_ON_CONNECT=${HEMELB_WAIT_ON_CONNECT}
 	-DHEMELB_BUILD_MULTISCALE=${HEMELB_BUILD_MULTISCALE}
 	-DHEMELB_IMAGES_TO_NULL=${HEMELB_IMAGES_TO_NULL}
-	BUILD_COMMAND make -j${HEMELB_SUBPROJECT_MAKE_JOBS}
+    BUILD_COMMAND $(MAKE) -j${HEMELB_SUBPROJECT_MAKE_JOBS}
 )
 
 add_dependencies(hemelb cppunit zoltan ptscotch parmetis ppstee tinyxml boost ctemplate zlib mpwide)

Code/SimulationMaster.cc

@@ -62,6 +62,7 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options)
   neighbouringDataManager = NULL;
   imagesPerSimulation = options.NumberOfImages();
   steeringSessionId = options.GetSteeringSessionId();
+  requestedPartitioner = options.GetRequestedPartitioner();
 
   fileManager = new hemelb::io::PathManager(options, IsCurrentProcTheIOProc(), GetProcessorCount());
   if (fileManager->HasProblems())
@@ -158,7 +159,7 @@ void SimulationMaster::Initialise()
   hemelb::geometry::GeometryReader reader(hemelb::steering::SteeringComponent::RequiresSeparateSteeringCore(),
                                           latticeType::GetLatticeInfo(),
                                           timings);
-  hemelb::geometry::Geometry readGeometryData = reader.LoadAndDecompose(simConfig->GetDataFilePath());
+  hemelb::geometry::Geometry readGeometryData = reader.LoadAndDecompose(simConfig->GetDataFilePath(), requestedPartitioner);
 
   // Create a new lattice based on that info and return it.
   latticeData = new hemelb::geometry::LatticeData(latticeType::GetLatticeInfo(), readGeometryData);

Code/SimulationMaster.h

@@ -113,6 +113,7 @@ class SimulationMaster
 
     unsigned int imagesPerSimulation;
     int steeringSessionId;
+    int requestedPartitioner;
     unsigned int imagesPeriod;
     static const hemelb::LatticeTime FORCE_FLUSH_PERIOD=1000;
 };

Code/configuration/CommandLine.cc

@@ -15,7 +15,7 @@ namespace hemelb
   namespace configuration
   {
     CommandLine::CommandLine(int aargc, const char * const * const aargv) :
-        inputFile("input.xml"), outputDir(""), images(10), steeringSessionId(1), argc(aargc), argv(aargv), ok(false)
+        inputFile("input.xml"), outputDir(""), images(10), steeringSessionId(1), requestedPartitioner(0), argc(aargc), argv(aargv), ok(false)
     {
 
       // Initialise the network discovery. If this fails, abort.
@@ -63,6 +63,11 @@ namespace hemelb
           char *dummy;
           steeringSessionId = (unsigned int) (strtoul(paramValue, &dummy, 10));
         }
+        else if (std::strcmp(paramName, "-part") == 0)
+        {
+          char *dummy;
+          requestedPartitioner = (unsigned int) (strtoul(paramValue, &dummy, 10));
+        }
         else
         {
           PrintUsage();
@@ -82,6 +87,7 @@ namespace hemelb
       printf("-out \t Path to the output folder (default is based on input file, e.g. config_xml_results)\n");
       printf("-i \t Number of images to create (default is 10)\n");
       printf("-ss \t Steering session identifier (default is 1)\n");
+      printf("-part \t Partitioner selection for PPStee: 0=ParMETIS, 1=PTScotch, 2=Zoltan (default is 0)\n");
       printf("-!-!-!-!-!-!-!-!-!-!-!-!");
     }
   }

Code/configuration/CommandLine.h

@@ -70,6 +70,13 @@ namespace hemelb
         {
           return (steeringSessionId);
         }
+        /**
+         * @return An integer representing the requested partitioning library.
+         */
+        int GetRequestedPartitioner() const
+        {
+          return (requestedPartitioner);
+        }
         /**
          * @return  Total count of command line arguments.
          */
@@ -100,6 +107,7 @@ namespace hemelb
         std::string outputDir; //! local or full path to input file
         unsigned int images; //! images to produce
         int steeringSessionId; //! unique identifier for steering session
+        int requestedPartitioner; //! requested partitioner
         int argc; //! count of command line arguments, including program name
         const char * const * const argv; //! command line arguments
         bool ok; //! track if the construction went OK.

Code/geometry/GeometryReader.cc

@@ -80,7 +80,7 @@ namespace hemelb
       }
     }
 
-    Geometry GeometryReader::LoadAndDecompose(const std::string& dataFilePath)
+    Geometry GeometryReader::LoadAndDecompose(const std::string& dataFilePath, const int requestedPartitioner /* = 0 */)
     {
       log::Logger::Log<log::Debug, log::OnePerCore>("Starting file read timer");
       timings[hemelb::reporting::Timers::fileRead].Start();
@@ -189,7 +189,7 @@ namespace hemelb
       if (participateInTopology)
       {
         log::Logger::Log<log::Debug, log::OnePerCore>("Beginning domain decomposition optimisation");
-        OptimiseDomainDecomposition(geometry, principalProcForEachBlock);
+        OptimiseDomainDecomposition(geometry, principalProcForEachBlock, requestedPartitioner);
         log::Logger::Log<log::Debug, log::OnePerCore>("Ending domain decomposition optimisation");
 
         if (ShouldValidate())
@@ -812,14 +812,15 @@ namespace hemelb
       return shouldReadBlock;
     }
 
-    void GeometryReader::OptimiseDomainDecomposition(Geometry& geometry, const std::vector<proc_t>& procForEachBlock)
+    void GeometryReader::OptimiseDomainDecomposition(Geometry& geometry, const std::vector<proc_t>& procForEachBlock, const int requestedPartitioner /* = 0 */)
     {
       decomposition::OptimisedDecomposition optimiser(timings,
                                                       topologyComms,
                                                       geometry,
                                                       latticeInfo,
                                                       procForEachBlock,
-                                                      fluidSitesOnEachBlock);
+                                                      fluidSitesOnEachBlock,
+                                                      requestedPartitioner);
 
       timings[hemelb::reporting::Timers::reRead].Start();
       log::Logger::Log<log::Debug, log::OnePerCore>("Rereading blocks");

Code/geometry/GeometryReader.h

@@ -37,7 +37,7 @@ namespace hemelb
         GeometryReader(const bool reserveSteeringCore, const lb::lattices::LatticeInfo&, reporting::Timers &timings);
         ~GeometryReader();
 
-        Geometry LoadAndDecompose(const std::string& dataFilePath);
+        Geometry LoadAndDecompose(const std::string& dataFilePath, const int requestedPartitioner = 0);
 
       private:
         /**
@@ -126,7 +126,7 @@ namespace hemelb
          * @param geometry
          * @param procForEachBlock
          */
-        void OptimiseDomainDecomposition(Geometry& geometry, const std::vector<proc_t>& procForEachBlock);
+        void OptimiseDomainDecomposition(Geometry& geometry, const std::vector<proc_t>& procForEachBlock, const int requestedPartitioner = 0);
 
         void ValidateGeometry(const Geometry& geometry);
 

Code/geometry/decomposition/OptimisedDecomposition.cc

@@ -12,6 +12,10 @@
 #include "log/Logger.h"
 #include "net/net.h"
 
+#define HEMELB_PPSTEE_PARTITIONER_PARMETIS 0
+#define HEMELB_PPSTEE_PARTITIONER_PTSCOTCH 1
+#define HEMELB_PPSTEE_PARTITIONER_ZOLTAN 2
+
 namespace hemelb
 {
   namespace geometry
@@ -23,9 +27,10 @@ namespace hemelb
                                                      const Geometry& geometry,
                                                      const lb::lattices::LatticeInfo& latticeInfo,
                                                      const std::vector<proc_t>& procForEachBlock,
-                                                     const std::vector<site_t>& fluidSitesOnEachBlock) :
+                                                     const std::vector<site_t>& fluidSitesOnEachBlock,
+                                                     const int requestedPartitioner) :
         timers(timers), comms(comms), geometry(geometry), latticeInfo(latticeInfo), procForEachBlock(procForEachBlock),
-            fluidSitesPerBlock(fluidSitesOnEachBlock)
+            fluidSitesPerBlock(fluidSitesOnEachBlock), requestedPartitioner(requestedPartitioner)
       {
         timers[hemelb::reporting::Timers::InitialGeometryRead].Start(); //overall dbg timing
 
@@ -61,9 +66,9 @@ namespace hemelb
 
         // Call parmetis.
         timers[hemelb::reporting::Timers::parmetis].Start();
-        log::Logger::Log<log::Debug, log::OnePerCore>("Making the call to Parmetis");
+        log::Logger::Log<log::Debug, log::OnePerCore>("Making the call to Partitioner");
 
-        CallParmetis(localVertexCount);
+        CallPartitioner(localVertexCount);
 
         timers[hemelb::reporting::Timers::parmetis].Stop();
         log::Logger::Log<log::Debug, log::OnePerCore>("Parmetis has finished.");
@@ -101,8 +106,7 @@ namespace hemelb
         // comm* is a pointer to the MPI communicator of the processes involved
 
         // Initialise the partition vector.
-	// Not needed for PPStee. See below.
-        //partitionVector = std::vector<idx_t>(localVertexCount, comms.GetRank());
+        partitionVector = std::vector<idx_t>(localVertexCount, comms.GetRank());
 
         // Weight all vertices evenly.
         std::vector < idx_t > vertexWeight(localVertexCount, 1);
@@ -138,14 +142,13 @@ namespace hemelb
         // Reserve 1 on these vectors so that the reference to their first element
         // exists (even if it's unused).
         // Reserve on the vectors to be certain they're at least 1 in capacity (so &vector[0] works)
-        //partitionVector.reserve(1);
+        partitionVector.reserve(1);
         vtxDistribn.reserve(1);
         adjacenciesPerVertex.reserve(1);
         localAdjacencies.reserve(1);
         vertexWeight.reserve(1);
         MPI_Comm communicator = comms.GetCommunicator();
 
-        /* replaced by PPStee:
         ParMETIS_V3_PartKway(&vtxDistribn[0],
                              &adjacenciesPerVertex[0],
                              &localAdjacencies[0],
@@ -162,37 +165,56 @@ namespace hemelb
                              &partitionVector[0],
                              &communicator);
         log::Logger::Log<log::Debug, log::OnePerCore>("ParMetis returned.");
-        */ // replaced by PPStee: END
+      }
+
+      void OptimisedDecomposition::CallPartitioner(idx_t localVertexCount)
+      {
+        // Weight all vertices evenly.
+        std::vector < idx_t > vertexWeight(localVertexCount, 1);
+
+        // Set the weights of each partition to be even, and to sum to 1.
+        idx_t desiredPartitionSize = comms.GetSize();
+
+        std::vector < real_t > domainWeights(desiredPartitionSize, (real_t)(1.0) / ( (real_t)(desiredPartitionSize)));
+
+        log::Logger::Log<log::Debug, log::OnePerCore>("Calling PPStee");
+
+        // Reserve 1 on these vectors so that the reference to their first element
+        // exists (even if it's unused).
+        // Reserve on the vectors to be certain they're at least 1 in capacity (so &vector[0] works)
+        vtxDistribn.reserve(1);
+        adjacenciesPerVertex.reserve(1);
+        localAdjacencies.reserve(1);
+        vertexWeight.reserve(1);
+        MPI_Comm communicator = comms.GetCommunicator();
 
-        /* PPStee modifications */
         // graph
-        /**/
-        // use ParMETIS
-        PPSteeGraphParmetis pgraph = PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
-        /**/
-
-        /*
-        // use PTScotch
-        PPSteeGraphParmetis pgraph_prime = PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
-        PPSteeGraphiPtscotch pgraph = PPSteeGraphPtscotch(&pgraph_prime);
-        */
-
-        /*
-        // use Zoltan
-        PPSteeGraphParmetis pgraph_prime = PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
-        PPSteeGraphZoltan pgraph = PPSteeGraphZoltan(&pgraph_prime);
-        */
+        PPSteeGraph* pgraph;
+        if (requestedPartitioner == HEMELB_PPSTEE_PARTITIONER_PTSCOTCH) {
+          // use PTScotch
+          log::Logger::Log<log::Info, log::OnePerCore>("PPStee uses PTScotch.");
+          PPSteeGraphParmetis pgraph_prime = PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
+          pgraph = new PPSteeGraphPtscotch(&pgraph_prime);
+        } else if (requestedPartitioner == HEMELB_PPSTEE_PARTITIONER_ZOLTAN) {
+          // use Zoltan
+          log::Logger::Log<log::Info, log::OnePerCore>("PPStee uses Zoltan.");
+          PPSteeGraphParmetis pgraph_prime = PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
+          pgraph = new PPSteeGraphZoltan(&pgraph_prime);
+        } else {
+          // HEMELB_PPSTEE_PARTITIONER_PARMETIS:
+          log::Logger::Log<log::Info, log::OnePerCore>("PPStee uses ParMETIS.");
+          pgraph = new PPSteeGraphParmetis(communicator, &vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0]);
+        }
 
         // weights for computation
-        PPSteeWeights pweights(&pgraph);
-std::vector < int > adjwgt(pgraph.getEdgeloccnt(), 1);
-pweights.setWeightsData(&vertexWeight[0], &adjwgt[0]);
+        PPSteeWeights pweights(pgraph);
+        pweights.setOnlyVertexWeightsData(&vertexWeight[0]);
 
         // interface
         PPStee ppstee;
 
         // submit graph
-        ppstee.submitGraph(pgraph);
+        ppstee.submitGraph(*pgraph);
 
         // submit weights
         ppstee.submitNewStage(pweights, PPSTEE_STAGE_COMPUTATION);
@@ -202,34 +224,24 @@ pweights.setWeightsData(&vertexWeight[0], &adjwgt[0]);
         ppstee.getPartitioning(&ppart);
 
         // copy partitioning (needs a decent functionality to do so)
-	partitionVector.assign(ppart->getPartData(), ppart->getPartData()+ppart->getVertloccnt());
+        partitionVector.assign(ppart->getPartData(), ppart->getPartData()+ppart->getVertloccnt());
 
         // add debug log message: show number of vertices moving to threads #0 - #mpi_n-1
-        int offrange = 0;
-        std::vector<int> pcounter(12, 0);
-        for (int i=0; i<ppart->getVertloccnt(); ++i) {
-          if (ppart->getPartData()[i] >= 0 && ppart->getPartData()[i] <= 11) {
-            pcounter[ppart->getPartData()[i]]++;
-          } else {
-            offrange++;
+        int numStaying = 0;
+        int numLeaving = 0;
+        for (int i = 0; i < ppart->getVertloccnt(); ++i) {
+          if (ppart->getPartData()[i] == comms.GetRank()) {
+            ++numStaying;
+          } else if (ppart->getPartData()[i] >= 0 && ppart->getPartData()[i] <= comms.GetSize()) {
+            ++numLeaving;
           }
         }
-        log::Logger::Log<log::Debug, log::OnePerCore>("PPStee part counts: %i %i %i %i %i %i %i %i %i %i %i %i (offrange: %i).",
-          pcounter[0],
-          pcounter[1],
-          pcounter[2],
-          pcounter[3],
-          pcounter[4],
-          pcounter[5],
-          pcounter[6],
-          pcounter[7],
-          pcounter[8],
-          pcounter[9],
-          pcounter[10],
-          pcounter[11],
-          offrange
+        log::Logger::Log<log::Info, log::OnePerCore>("PPStee: parts staying/leaving/sum/all: %i %i %i %i.",
+          numStaying,
+          numLeaving,
+          numStaying + numLeaving,
+          ppart->getVertloccnt()
         );
-/*	*/ // PPStee modifications: END
       }
 
       void OptimisedDecomposition::PopulateSiteDistribution()

Code/geometry/decomposition/OptimisedDecomposition.h

@@ -34,7 +34,8 @@ namespace hemelb
                                  const Geometry& geometry,
                                  const lb::lattices::LatticeInfo& latticeInfo,
                                  const std::vector<proc_t>& procForEachBlock,
-                                 const std::vector<site_t>& fluidSitesPerBlock);
+                                 const std::vector<site_t>& fluidSitesPerBlock,
+                                 const int requestedPartitioner);
 
           /**
            * Returns a vector with the number of moves coming from each core
@@ -85,6 +86,15 @@ namespace hemelb
            */
           void CallParmetis(idx_t localVertexCount);
 
+          /**
+           * Perform the call to PPStee. Returns the result in the partition vector, other
+           * parameters are input only. These can't be made const because of the API to ParMetis
+           * (and other partitioning libraries) PPStee uses.
+           *
+           * @param localVertexCount [in] The number of local fluid sites
+           */
+          void CallPartitioner(idx_t localVertexCount);
+
           /**
            * Populate the list of moves from each proc that we need locally, using the
            * partition vector.
@@ -207,6 +217,7 @@ namespace hemelb
           const lb::lattices::LatticeInfo& latticeInfo; //! The lattice info to optimise for.
           const std::vector<proc_t>& procForEachBlock; //! The processor assigned to each block at the moment
           const std::vector<site_t>& fluidSitesPerBlock; //! The number of fluid sites on each block.
+          const int requestedPartitioner; //! The requested partitioner.
           std::vector<idx_t> vtxDistribn; //! The vertex distribution across participating cores.
           std::vector<idx_t> firstSiteIndexPerBlock; //! The global contiguous index of the first fluid site on each block.
           std::vector<idx_t> adjacenciesPerVertex; //! The number of adjacencies for each local fluid site