Skip to content

Commit

Permalink
Merge pull request #184 from AMReX-FHD/fix_do_2D_surfcov_restart
Browse files Browse the repository at this point in the history 
do_2D mode with surface coverage restart fixes.
  • Loading branch information
@ajnonaka
ajnonaka authored Feb 11, 2025
2 parents ecd7eb2 + 1212ec0 commit 771a1a3
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 14 deletions.
64 changes: 59 additions & 5 deletions src_compressible_stag/Checkpoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ void WriteCheckPoint2D(int step,
const std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
const std::array<MultiFab, AMREX_SPACEDIM>& velVars,
const amrex::MultiFab& coVars,
const amrex::MultiFab& surfcov,
const amrex::MultiFab& surfcovMeans,
const amrex::MultiFab& surfcovVars,
const amrex::MultiFab& surfcovcoVars,
const amrex::MultiFab& spatialCross, int /*ncross*/)
{
// timer for profiling
Expand Down Expand Up @@ -410,6 +414,18 @@ void WriteCheckPoint2D(int step,
VisMF::Write(coVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "coVars"));

if (n_ads_spec>0) {
// surfcov
VisMF::Write(surfcov,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcov"));
VisMF::Write(surfcovMeans,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovMeans"));
VisMF::Write(surfcovVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovVars"));
VisMF::Write(surfcovcoVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovcoVars"));
}

// spatialCross
VisMF::Write(spatialCross,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "spatialCross")); // (do later)
Expand Down Expand Up @@ -751,7 +767,7 @@ void ReadCheckPoint3D(int& step,
MPI_Comm_size(MPI_COMM_WORLD, &n_ranks);

// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
if (seed < 0) {
if (seed == -1) {

#ifdef AMREX_USE_CUDA
Abort("Restart with negative seed not supported on GPU");
Expand Down Expand Up @@ -902,6 +918,10 @@ void ReadCheckPoint2D(int& step,
std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
std::array<MultiFab, AMREX_SPACEDIM>& velVars,
amrex::MultiFab& coVars,
amrex::MultiFab& surfcov,
amrex::MultiFab& surfcovMeans,
amrex::MultiFab& surfcovVars,
amrex::MultiFab& surfcovcoVars,
amrex::MultiFab& spatialCross,
int ncross,
BoxArray& ba, DistributionMapping& dmap)
Expand Down Expand Up @@ -981,6 +1001,14 @@ void ReadCheckPoint2D(int& step,

// coVars
coVars.define(ba,dmap,26,0);

if (n_ads_spec>0) {
// surfcov
surfcov.define(ba,dmap,n_ads_spec,0);
surfcovMeans.define(ba,dmap,n_ads_spec,0);
surfcovVars.define(ba,dmap,n_ads_spec,0);
surfcovcoVars.define(ba,dmap,n_ads_spec*6,0);
}

// spatialCross
spatialCross.define(ba,dmap,ncross,0); // (do later)
Expand All @@ -995,10 +1023,16 @@ void ReadCheckPoint2D(int& step,
int n_ranks;
MPI_Comm_size(MPI_COMM_WORLD, &n_ranks);

// don't read in all the rng states at once (overload filesystem)
// one at a time write out the rng states to different files, one for each MPI rank
// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
if (seed == -1) {

#ifdef AMREX_USE_CUDA
Abort("Restart with negative seed not supported on GPU");
#endif

// don't read in all the rng states at once (overload filesystem)
// one at a time write out the rng states to different files, one for each MPI rank
// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
for (int rank=0; rank<n_ranks; ++rank) {

if (comm_rank == rank) {
Expand Down Expand Up @@ -1053,6 +1087,10 @@ void ReadCheckPoint2D(int& step,
Read_Copy_MF_Checkpoint(cumom[1],"cumomy",checkpointname,ba_old,dmap_old,1,1,1);
Read_Copy_MF_Checkpoint(cumom[2],"cumomz",checkpointname,ba_old,dmap_old,1,1,2);

if (n_ads_spec>0) {
Read_Copy_MF_Checkpoint(surfcov,"surfcov",checkpointname,ba_old,dmap_old,n_ads_spec,0);
}

// Set all stats to zero if reset stats, else read
if (reset_stats == 1) {
cuMeans.setVal(0.0);
Expand All @@ -1066,6 +1104,13 @@ void ReadCheckPoint2D(int& step,
cumomVars[d].setVal(0.);
}
coVars.setVal(0.0);
if (n_ads_spec>0) {
for (int m=0;m<n_ads_spec;m++) {
surfcovMeans.setVal(0.0);
surfcovVars.setVal(0.0);
surfcovcoVars.setVal(0.0);
}
}
spatialCross.setVal(0.0);
}
else {
Expand All @@ -1092,6 +1137,12 @@ void ReadCheckPoint2D(int& step,
Read_Copy_MF_Checkpoint(cumomVars[0],"cumomvarx",checkpointname,ba_old,dmap_old,1,0,0);
Read_Copy_MF_Checkpoint(cumomVars[1],"cumomvary",checkpointname,ba_old,dmap_old,1,0,1);
Read_Copy_MF_Checkpoint(cumomVars[2],"cumomvarz",checkpointname,ba_old,dmap_old,1,0,2);

if (n_ads_spec>0) {
Read_Copy_MF_Checkpoint(surfcovMeans,"surfcovMeans",checkpointname,ba_old,dmap_old,n_ads_spec,0);
Read_Copy_MF_Checkpoint(surfcovVars,"surfcovVars",checkpointname,ba_old,dmap_old,n_ads_spec,0);
Read_Copy_MF_Checkpoint(surfcovcoVars,"surfcovcoVars",checkpointname,ba_old,dmap_old,n_ads_spec*6,0);
}
}

// FillBoundaries
Expand Down Expand Up @@ -1209,8 +1260,11 @@ void ReadCheckPoint1D(int& step,
coVars.define(ba,dmap,26,0);

// spatialCross
if (all_correl) spatialCross.define(ba,dmap,ncross*5,0); // for five x*: [0, fl(n_cells[0]/4), fl(n_cells[0]/2), fl(n_cells[0]*3/4), n_cells[0]-1]
else spatialCross.define(ba,dmap,ncross,0);
if (all_correl) {
spatialCross.define(ba,dmap,ncross*5,0); // for five x*: [0, fl(n_cells[0]/4), fl(n_cells[0]/2), fl(n_cells[0]*3/4), n_cells[0]-1]
} else {
spatialCross.define(ba,dmap,ncross,0);
}

}

Expand Down
8 changes: 8 additions & 0 deletions src_compressible_stag/compressible_functions_stag.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ void WriteCheckPoint2D(int step,
const std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
const std::array<MultiFab, AMREX_SPACEDIM>& velVars,
const amrex::MultiFab& coVars,
const amrex::MultiFab& surfcov,
const amrex::MultiFab& surfcovMeans,
const amrex::MultiFab& surfcovVars,
const amrex::MultiFab& surfcovcoVars,
const amrex::MultiFab& spatialCross, int ncross);

void ReadCheckPoint2D(int& step,
Expand All @@ -270,6 +274,10 @@ void ReadCheckPoint2D(int& step,
std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
std::array<MultiFab, AMREX_SPACEDIM>& velVars,
amrex::MultiFab& coVars,
amrex::MultiFab& surfcov,
amrex::MultiFab& surfcovMeans,
amrex::MultiFab& surfcovVars,
amrex::MultiFab& surfcovcoVars,
amrex::MultiFab& spatialCross, int ncross,
BoxArray& ba, DistributionMapping& dmap);

Expand Down
29 changes: 20 additions & 9 deletions src_compressible_stag/main_driver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,17 +508,22 @@ void main_driver(const char* argv)
if (do_1D) {
ReadCheckPoint1D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross1D, ncross, ba, dmap);
vel, velMeans, velVars, coVars,
spatialCross1D, ncross, ba, dmap);
}
else if (do_2D) {
ReadCheckPoint2D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross2D, ncross, ba, dmap);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross2D, ncross, ba, dmap);
}
else {
ReadCheckPoint3D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, surfcov, surfcovMeans, surfcovVars, surfcovcoVars, spatialCross3D, ncross, turbforce, ba, dmap);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross3D, ncross, turbforce, ba, dmap);
}

if (reset_stats == 1) statsCount = 1;
Expand Down Expand Up @@ -1025,16 +1030,17 @@ void main_driver(const char* argv)
} else {
Abort("splitting_MFsurfchem can be 0 or 1");
}

VisMF::Write(surfcov,"a_surfcov");
}

// FHD
if (turbRestartRun) {
RK3stepStag(cu, cumom, prim, vel, source, eta, zeta, kappa, chi, D,
faceflux, edgeflux_x, edgeflux_y, edgeflux_z, cenflux, ranchem, geom, dt, step, turbforce);
} else {
calculateTransportCoeffs(prim, eta, zeta, kappa, chi, D);
}
else {
calculateTransportCoeffs(prim, eta, zeta, kappa, chi, D);
}

if (n_ads_spec>0 && splitting_MFsurfchem == 1) {
sample_MFsurfchem(cu, prim, surfcov, dNadsdes, dNads, dNdes, geom, dt/2.0);
Expand Down Expand Up @@ -1576,17 +1582,22 @@ void main_driver(const char* argv)
if (do_1D) {
WriteCheckPoint1D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross1D, ncross);
vel, velMeans, velVars, coVars,
spatialCross1D, ncross);
}
else if (do_2D) {
WriteCheckPoint2D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross2D, ncross);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross2D, ncross);
}
else {
WriteCheckPoint3D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, surfcov, surfcovMeans, surfcovVars, surfcovcoVars, spatialCross3D, ncross, turbforce);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross3D, ncross, turbforce);
}
}

Expand Down

0 comments on commit 771a1a3

Please sign in to comment.