diff --git a/.clang-tidy b/.clang-tidy index 9fada1bbdfe..21f0343519c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -14,7 +14,6 @@ Checks: ' -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, - -cppcoreguidelines-no-malloc, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, -cppcoreguidelines-owning-memory, @@ -31,23 +30,18 @@ Checks: ' -modernize-use-trailing-return-type, mpi-*, performance-*, - -performance-noexcept-move-constructor, -performance-unnecessary-copy-initialization, -performance-unnecessary-value-param, portability-*, readability-*, - -readability-braces-around-statements, -readability-convert-member-functions-to-static, -readability-else-after-return, -readability-function-cognitive-complexity, -readability-identifier-length, -readability-implicit-bool-conversion, - -readability-inconsistent-declaration-parameter-name, -readability-isolate-declaration, -readability-magic-numbers, - -readability-make-member-function-const, -readability-named-parameter, - -readability-qualified-auto, -readability-uppercase-literal-suffix ' diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ff50edc428e..b50989f2f5c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - + cd ../amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 9d0206efb76..2225e670bb0 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -41,7 +41,8 @@ sudo apt-get install -y --no-install-recommends \ rocm-dev \ rocfft-dev \ rocprim-dev \ - rocrand-dev + rocrand-dev \ + hiprand-dev # ccache $(dirname "$0")/ccache.sh diff --git a/.gitignore b/.gitignore index d614c1ae2d9..4fa65351331 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ Docs/warpx-doxygen-web.tag.xml Docs/openpmd-api-doxygen-web.tag.xml Docs/doxyhtml/ Docs/doxyxml/ -Docs/source/_static/ +Docs/source/_static/doxyhtml #################### # Package Managers # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 21577b0604a..35c38ac1ad6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.13.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) diff --git a/CMakeLists.txt b/CMakeLists.txt index e6aa3c8174f..7a0b28c9f86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 23.12) +project(WarpX VERSION 24.01) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index b43cc1329fb..b16bf161144 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -9,6 +9,8 @@ breathe docutils>=0.17.1 +openpmd-viewer # for checksumAPI + # PICMI API docs # note: keep in sync with version in ../requirements.txt picmistandard==0.28.0 @@ -26,3 +28,4 @@ sphinx-design sphinx_rtd_theme>=1.1.1 sphinxcontrib-bibtex sphinxcontrib-napoleon +yt # for checksumAPI diff --git a/Docs/source/_static/custom.css b/Docs/source/_static/custom.css new file mode 100644 index 00000000000..b31c9188d76 --- /dev/null +++ b/Docs/source/_static/custom.css @@ -0,0 +1,6 @@ +.eqno { + /* As of 2023 Dec, sphinx_rtd_theme has a bug where equation numbers appear above the equations instead of on the right */ + /* The following is a make-shift fix, but comes at the cost of "making the header link invisible," though I'm not sure what that means */ + /* Copied from https://github.com/readthedocs/sphinx_rtd_theme/issues/301#issuecomment-1205755904 */ + float: right; +} diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 3735459bf76..48a02c5d216 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -34,7 +34,7 @@ from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme -sys.path.insert(0, os.path.join( os.path.abspath(__file__), '../Python') ) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) # -- General configuration ------------------------------------------------ @@ -70,12 +70,12 @@ # BaseStyle class in pybtex/style/formatting/__init__.py # UnsrtStyle class in pybtex/style/formating/unsrt.py class WarpXBibStyle(UnsrtStyle): - # We want the family name, i.e, "last" name, of an author to appear first. - default_name_style = 'lastfirst' + # This option makes the family name, i.e, "last" name, of an author to appear first. + # default_name_style = 'lastfirst' def __init__(self, *args, **kwargs): - # We want the given names of an author to be abbreviated to just initials. - # Example: "Jean-Luc Vay" becomes "Vay, J.-L." + # This option makes the given names of an author abbreviated to just initials. + # Example: "Jean-Luc" becomes "J.-L." # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer kwargs['abbreviate_names'] = True super().__init__(*args, **kwargs) @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'23.12' +version = u'24.01' # The full version, including alpha/beta/rc tags. -release = u'23.12' +release = u'24.01' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -134,6 +134,11 @@ def __init__(self, *args, **kwargs): html_theme = 'sphinx_rtd_theme' numfig = True +math_eqref_format = "{number}" +numfig_format = {'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section %s'} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -146,6 +151,9 @@ def __init__(self, *args, **kwargs): # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'custom.css', +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/Docs/source/dataanalysis/sensei.rst b/Docs/source/dataanalysis/sensei.rst index 16826c3082c..5e4bfff10fd 100644 --- a/Docs/source/dataanalysis/sensei.rst +++ b/Docs/source/dataanalysis/sensei.rst @@ -30,7 +30,7 @@ and bridge code making it easy to use in AMReX based simulation codes. SENSEI provides a *configurable analysis adaptor* which uses an XML file to select and configure one or more back ends at run time. Run time selection of the back end via XML means one user can access Catalyst, another Libsim, yet -another Python with no changes to the code. This is depicted in figure +another Python with no changes to the code. This is depicted in :numref:`sensei_arch`. On the left side of the figure AMReX produces data, the bridge code pushes the data through the configurable analysis adaptor to the back end that was selected at run time. @@ -99,7 +99,7 @@ The back end is selected and configured at run time using the SENSEI XML file. The XML sets parameters specific to SENSEI and to the chosen back end. Many of the back ends have sophisticated configuration mechanisms which SENSEI makes use of. For example the following XML configuration was used on NERSC's Cori -with WarpX to render 10 iso surfaces, shown in figure :numref:`lpa_visit`, using +with WarpX to render 10 iso surfaces, shown in :numref:`lpa_visit`, using VisIt Libsim. .. code-block:: xml @@ -123,7 +123,7 @@ run of the desired simulation. quadrant has been clipped away to reveal innner structure. The same run and visualization was repeated using ParaView Catalyst, shown in -figure :numref:`lpa_pv`, by providing the following XML configuration. +:numref:`lpa_pv`, by providing the following XML configuration. .. code-block:: xml diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 4023bc06d50..4412c3e3cc4 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -14,9 +14,9 @@ From a user point of view, you should only need to use ``checksumAPI.py``. It co Include a checksum regression test in an analysis Python script --------------------------------------------------------------- -This relies on function ``evaluate_checksum``: +This relies on the function ``evaluate_checksum``: -.. doxygenfunction:: evaluate_checksum +.. autofunction:: checksumAPI.evaluate_checksum For an example, see @@ -32,7 +32,7 @@ You can execute ``checksumAPI.py`` as a Python script for that, and pass the plo .. code-block:: bash - ./checksumAPI.py --evaluate --plotfile --test-name + ./checksumAPI.py --evaluate --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options @@ -41,17 +41,17 @@ See additional options * ``--rtol`` relative tolerance for the comparison * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``) -Reset a benchmark with new values that you know are correct ------------------------------------------------------------ +Create/Reset a benchmark with new values that you know are correct +------------------------------------------------------------------ -Reset a benchmark from a plotfile generated locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Create/Reset a benchmark from a plotfile generated locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This is using ``checksumAPI.py`` as a Python script. .. code-block:: bash - ./checksumAPI.py --reset-benchmark --plotfile --test-name + ./checksumAPI.py --reset-benchmark --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst index a1c610eed4a..5acea786920 100644 --- a/Docs/source/developers/profiling.rst +++ b/Docs/source/developers/profiling.rst @@ -121,7 +121,7 @@ Perlmutter Example """""""""""""""""" Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). -You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. +You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. .. code-block:: bash diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 2bd74e862f6..2b3fe989f2f 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -18,7 +18,7 @@ We slightly modify this file in ``Regression/prepare_file_ci.py``. For example, if you like to change the compiler to compilation to build on Nvidia GPUs, modify this block to add ``-DWarpX_COMPUTE=CUDA``: -.. code-block:: toml +.. code-block:: ini [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 420bb376d67..98b0d85391e 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -149,7 +149,12 @@ Please see :ref:`this section `. Nuclear Fusion - Magnetically Confined Plasmas ********************************************** -#. Nicks, B. S., Putvinski, S. and Tajima, T. +#. Nicks B. S., Putvinski S. and Tajima T. **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**. Physics of Plasmas **30**, 102108, 2023. `DOI:10.1063/5.0163889 `__ + +#. Groenewald R. E., Veksler A., Ceccherini F., Necas A., Nicks B. S., Barnes D. C., Tajima T. and Dettrick S. A. + **Accelerated kinetic model for global macro stability studies of high-beta fusion reactors**. + Physics of Plasmas **30**, 122508, 2023. + `DOI:10.1063/5.0178288 `__ diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 4c5f791ca6b..9f2d2ff038e 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -75,9 +75,9 @@ Usage :hidden: usage/how_to_run + usage/examples usage/python usage/parameters - usage/examples usage/workflows usage/faq @@ -107,9 +107,9 @@ Theory :hidden: theory/intro - theory/picsar_theory + theory/pic theory/amr - theory/PML + theory/boundary_conditions theory/boosted_frame theory/input_output theory/collisions diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index a76e8c212a2..e299a60c75b 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -130,7 +130,7 @@ CMake Option Default & Values Des ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from ``WarpX_picsar_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_picsar_repo`` ``WarpX_picsar_internal`` **ON**/OFF Needs a pre-installed PICSAR library if set to ``OFF`` -``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) +``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_pyamrex_repo`` ``https://github.com/AMReX-Codes/pyamrex.git`` Repository URI to pull and build pyAMReX from ``WarpX_pyamrex_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_pyamrex_repo`` ``WarpX_pyamrex_internal`` **ON**/OFF Needs a pre-installed pyAMReX library if set to ``OFF`` diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 7bd6c7dc8d5..a939db840da 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -18,7 +18,7 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.0+ (11.3+ recommended) `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support - `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 7a8bf561986..fffb917df1c 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -12,83 +12,60 @@ Introduction If you are new to this system, **please see the following resources**: * `IT4I user guide `__ -* Batch system: `PBS `__ +* Batch system: `SLURM `__ * Jupyter service: not provided/documented (yet) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scratch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (10TB default) + * ``/mnt/proj/``: per-project work directory, used for long term data storage (20TB default) .. _building-karolina-preparation: -Preparation ------------ - -Use the following commands to download the WarpX source code: +Installation +------------ -.. code-block:: bash +We show how to install from scratch all the dependencies using `Spack `__. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx +For size reasons it is not advisable to install WarpX in the ``$HOME`` directory, it should be installed in the "work directory". For this purpose we set an environment variable ``$WORK`` with the path to the "work directory". On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. -.. tab-set:: - - .. tab-item:: A100 GPUs - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash +Profile file +^^^^^^^^^^^^ - export proj="DD-23-83" +One can use the pre-prepared ``karolina_warpx.profile`` script below, +which you can copy to ``${HOME}/karolina_warpx.profile``, edit as required and then ``source``. - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. important:: + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_warpx.profile.example + :language: bash + :caption: Copy the contents of this file to ``${HOME}/karolina_warpx.profile``. - Now, and as the first step on future logins to Karolina, activate these environment settings: +To have the environment activated on every login, add the following line to ``${HOME}/.bashrc``: - .. code-block:: bash - - source $HOME/karolina_gpu_warpx.profile - - Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once: - - .. code-block:: bash +.. code-block:: bash - bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh - source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate + source $HOME/karolina_warpx.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +To install the ``spack`` environment and Python packages: - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh - :language: bash +.. code-block:: bash + bash $WORK/src/warpx/Tools/machines/karolina-it4i/install_dependencies.sh - .. tab-item:: CPU Nodes +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - CPU usage is documentation is TODO. + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_dependencies.sh + :language: bash .. _building-karolina-compilation: @@ -98,117 +75,53 @@ Compilation Use the following :ref:`cmake commands ` to compile the application executable: -.. tab-set:: - - .. tab-item:: A100 GPUs - - .. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_gpu - - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu -j 12 - - The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: - - .. code-block:: bash - - rm -rf build_gpu_py - - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu_py -j 12 --target pip_install - - .. tab-item:: CPU Nodes - - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu + cd $WORK/src/warpx + rm -rf build_gpu - cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu -j 12 + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 48 - The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: +The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu_py + cd $WORK/src/warpx + rm -rf build_gpu_py - cmake -S . -B build_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu_py -j 12 --target pip_install + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. -.. _building-karolina-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. - - .. _running-cpp-karolina: Running ------- -.. tab-set:: +The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#SBATCH --nodes=`` accordingly) on the supercomputer Karolina at IT4I. +This partition has up to `72 nodes `__. +Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. - .. tab-item:: A100 (40GB) GPUs +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. +Note that we run one MPI rank per GPU. - The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I. - This partition as up to `72 nodes `__. - Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. +.. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.sbatch + :language: bash + :caption: You can copy this file from ``$WORK/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.sbatch``. - Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. - Note that we run one MPI rank per GPU. - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub - :language: bash - :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``. - - To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run - - .. code-block:: bash - - qsub karolina_gpu.qsub - - to submit the job. +To run a simulation, copy the lines above to a file ``karolina_gpu.sbatch`` and run +.. code-block:: bash - .. tab-item:: CPU Nodes + sbatch karolina_gpu.sbatch - CPU usage is documentation is TODO. +to submit the job. .. _post-processing-karolina: diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 3de7f4755cb..1f3c1e8fbfa 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -103,9 +103,9 @@ Additionally, the following commands will install WarpX as a Python module: cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install -Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Leonardo jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. .. _building-leonardo-update: diff --git a/Docs/source/latex_theory/AMR/AMR.tex b/Docs/source/latex_theory/AMR/AMR.tex index 0e9eace8390..3ca94d95436 100644 --- a/Docs/source/latex_theory/AMR/AMR.tex +++ b/Docs/source/latex_theory/AMR/AMR.tex @@ -22,7 +22,7 @@ \section{Mesh refinement} In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss' Law around the refined patch, leading to long range errors \cite{Vaylpb2002,Colellajcp2010}. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. \subsection{Electrostatic} -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. +A cornerstone of the Particle-In-Cell method is that, given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. A sketch of the implementation of mesh refinement in WarpX is given in Figure~\ref{fig:ESAMR} (left). Given the solution of the electric potential at a refinement level $L_n$, it is interpolated onto the boundaries of the grid patch(es) at the next refined level $L_{n+1}$. The electric potential is then computed at level $L_{n+1}$ by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. diff --git a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex index 471863ac3ee..fe8365d15dd 100644 --- a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex +++ b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex @@ -175,7 +175,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} the simulation, and the artificial ``bump'' is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in \cite{KirchenARXIV2016,LeheARXIV2016}, which +A new scheme was recently proposed, in \cite{KirchenPOP2016,LehePRE2016}, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity -- with no arbitrary correction -- by simply integrating the PIC equations in \emph{Galilean coordinates} (also known as @@ -217,7 +217,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} \emph{themselves} are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current $\vec{J}$ within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme \cite{LeheARXIV2016}. +window and in a Galilean PSATD scheme \cite{LehePRE2016}. In the Galilean coordinates $\vec{x}'$, the equations of particle motion and the Maxwell equations take the form @@ -235,7 +235,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Integrating these equations from $t=n\Delta t$ to $t=(n+1)\Delta t$ results in the following update equations (see -\cite{LeheARXIV2016} for the details of the derivation): +\cite{LehePRE2016} for the details of the derivation): % \begin{subequations} \begin{align} @@ -271,5 +271,5 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Note that, in the limit $\vgal=\vec{0}$, (\ref{eq:disc-maxwell1}) and (\ref{eq:disc-maxwell2}) reduce to the standard PSATD equations \cite{Habericnsp73}, as expected. -As shown in \cite{KirchenARXIV2016,LeheARXIV2016}, +As shown in \cite{KirchenPOP2016,LehePRE2016}, the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. diff --git a/Docs/source/latex_theory/Makefile b/Docs/source/latex_theory/Makefile index 916bb2399d7..9b2f598f51d 100644 --- a/Docs/source/latex_theory/Makefile +++ b/Docs/source/latex_theory/Makefile @@ -7,14 +7,14 @@ all: $(SRC_FILES) clean pandoc Boosted_frame/Boosted_frame.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o boosted_frame.rst pandoc input_output/input_output.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o input_output.rst mv *.rst ../theory - cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o picsar_theory.rst - mv ../../../../picsar/Doxygen/pages/latex_theory/picsar_theory.rst ../theory + cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o pic.rst + mv ../../../../picsar/Doxygen/pages/latex_theory/pic.rst ../theory cp ../../../../picsar/Doxygen/images/PIC.png ../theory cp ../../../../picsar/Doxygen/images/Yee_grid.png ../theory clean: rm -f ../theory/intro.rst rm -f ../theory/warpx_theory.rst - rm -f ../theory/picsar_theory.rst + rm -f ../theory/pic.rst rm -f ../theory/PIC.png rm -f ../theory/Yee_grid.png diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index 32ede278416..b3475c5a81b 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -4,11 +4,11 @@ BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop @article{QuickpicParallel, -author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W.B. and Muggli, P. and Katsouleas, T.}, +author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W. B. and Muggli, P. and Katsouleas, T.}, doi = {10.1016/j.jcp.2009.04.019}, issn = {00219991}, journal = {Journal of Computational Physics}, -month = {aug}, +month = {Aug}, number = {15}, pages = {5340--5348}, title = {{Enhancing parallel quasi-static particle-in-cell simulations with a pipelining algorithm}}, @@ -26,11 +26,11 @@ @book{HockneyEastwoodBook year = {1988} } @article{Parkerjcp1991, -author = {Parker, Se and Birdsall, Ck}, +author = {Parker, S E and Birdsall, C K}, doi = {10.1016/0021-9991(91)90040-R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {91--102}, title = {{Numerical Error In Electron Orbits With Large Omega-Ce Delta-T}}, @@ -45,7 +45,7 @@ @inproceedings{Fawleyipac10 year = {2010} } @article{Godfreyjcp74, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {4}, @@ -59,7 +59,7 @@ @article{Quickpic doi = {10.1016/J.Jcp.2006.01.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {658--679}, title = {{Quickpic: A Highly Efficient Particle-In-Cell Code For Modeling Wakefield Acceleration In Plasmas}}, @@ -71,7 +71,7 @@ @article{Lundprstab2009 doi = {10.1103/Physrevstab.12.114801}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {nov}, +month = {Nov}, number = {11}, title = {{Generation Of Initial Kinetic Distributions For Simulation Of Long-Pulse Charged Particle Beams With High Space-Charge Intensity}}, volume = {12}, @@ -82,7 +82,7 @@ @article{CowanPRSTAB13 doi = {10.1103/PhysRevSTAB.16.041303}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations}}, volume = {16}, @@ -103,7 +103,7 @@ @article{YuPRL2014 author = {Yu, L.-L. and Esarey, E and Schroeder, C B and Vay, J.-L. and Benedetti, C and Geddes, C G R and Chen, M and Leemans, W P}, doi = {10.1103/PhysRevLett.112.125001}, journal = {Phys. Rev. Lett.}, -month = {mar}, +month = {Mar}, number = {12}, pages = {125001}, publisher = {American Physical Society}, @@ -119,7 +119,7 @@ @article{Vaynim2007 institution = {Univ Paris Sud Xi}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {65--69}, title = {{Self-Consistent Simulations Of Heavy-Ion Beams Interacting With Electron-Clouds}}, @@ -142,7 +142,7 @@ @article{Vayjcp2011 author = {Vay, J-L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, doi = {10.1016/J.Jcp.2011.04.003}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {15}, pages = {5908--5929}, title = {{Numerical Methods For Instability Mitigation In The Modeling Of Laser Wakefield Accelerators In A Lorentz-Boosted Frame}}, @@ -153,21 +153,15 @@ @article{Krallpre1993 author = {Krall, J and Ting, A and Esarey, E and Sprangle, P}, doi = {10.1103/Physreve.48.2157}, journal = {Physical Review E}, -month = {sep}, +month = {Sep}, number = {3}, pages = {2157--2161}, title = {{Enhanced Acceleration In A Self-Modulated-Laser Wake-Field Accelerator}}, volume = {48}, year = {1993} } -@article{Vayarxiv10_2, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1011.0917V2}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Dennisw1997585, -author = {W., Dennis and Hewett}, +author = {Dennis W. Hewett}, doi = {10.1006/Jcph.1997.5835}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -179,12 +173,11 @@ @article{Dennisw1997585 year = {1997} } @article{Habib2016, - archivePrefix = {arXiv}, arxivId = {1603.09303}, -author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C. -K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J. -L. and Viren, B. and Wurthwein, F. and Xiao, L.}, +author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C.-K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J.-L. and Viren, B. and Wurthwein, F. and Xiao, L.}, eprint = {1603.09303}, -month = {mar}, +month = {Mar}, title = {{ASCR/HEP Exascale Requirements Review Report}}, url = {http://arxiv.org/abs/1603.09303}, year = {2016} @@ -193,8 +186,8 @@ @article{Shadwickpop09 author = {Shadwick, B A and Schroeder, C B and Esarey, E}, doi = {10.1063/1.3124185}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56704}, title = {{Nonlinear Laser Energy Depletion In Laser-Plasma Accelerators}}, @@ -207,7 +200,7 @@ @article{Leemansaac2010 journal = {Aip Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, pages = {3--11}, -title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 Gev Laser Plasma Accelerator}}, +title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 GeV Laser Plasma Accelerator}}, volume = {1299}, year = {2010} } @@ -215,7 +208,7 @@ @article{Marderjcp87 author = {Marder, B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jan}, +month = {Jan}, number = {1}, pages = {48--55}, title = {{A Method For Incorporating Gauss Law Into Electromagnetic Pic Codes}}, @@ -232,7 +225,7 @@ @article{Ohmurapiers2010 year = {2010} } @article{Adamjcp1982, -author = {Adam, Jc and Serveniere, Ag and Langdon, Ab}, +author = {Adam, J C and Serveniere, Ag and Langdon, A B}, doi = {10.1016/0021-9991(82)90076-6}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -243,7 +236,7 @@ @article{Adamjcp1982 year = {1982} } @article{Tajimaprl79, -author = {Tajima, T and Dawson, Jm}, +author = {Tajima, T and Dawson, J M}, issn = {0031-9007}, journal = {Physical Review Letters}, number = {4}, @@ -252,13 +245,19 @@ @article{Tajimaprl79 volume = {43}, year = {1979} } -@article{Benedetti08, -author = {{Benedetti et al.}, C}, -journal = {Nuclear Inst. And Methods In Physics Research A}, -pages = {94--98}, -title = {{No Title}}, +@article{Benedetti09, +author = {C. Benedetti and P. Londrillo and V. Petrillo and L. Serafini and A. Sgattoni and P. Tomassini and G. Turchetti}, +doi = {https://doi.org/10.1016/j.nima.2009.05.064}, +issn = {0168-9002}, +journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, +keywords = {PIC simulation, Laser–plasma interaction, FEL}, +note = {Compton sources for X/γ rays: Physics and applications}, +number = {1, Supplement }, +pages = {S94-S98}, +title = {PIC simulations of the production of high-quality electron beams via laser–plasma interaction}, +url = {https://www.sciencedirect.com/science/article/pii/S0168900209009784}, volume = {608}, -year = {2009} +year = {2009}, } @inproceedings{Godfreyicnsp80, author = {Godfrey, B B}, @@ -271,7 +270,7 @@ @article{Blumenfeld2007 author = {Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and Lu, Wei and Marsh, Kenneth A and Mori, Warren B and Muggli, Patric and Oz, Erdem and Siemann, Robert H and Walz, Dieter and Zhou, Miaomiao}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7129}, pages = {741--744}, title = {{Energy doubling of 42[thinsp]GeV electrons in a metre-scale plasma wakefield accelerator}}, @@ -281,7 +280,7 @@ @article{Blumenfeld2007 } @inproceedings{Vayicap2002, annote = {7Th International Conference On Computational Accelerator Physics, Michigan State Univ, E Lansing, Mi, Oct 15-18, 2002}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, booktitle = {Computational Accelerator Physics 2002}, editor = {Berz, M and Makino, K}, isbn = {0-7503-0939-3}, @@ -340,7 +339,7 @@ @inproceedings{Qiang @inproceedings{Geddespac09, address = {Vancouver, Canada}, annote = {We6Rfp075}, -author = {{Geddes et al.}, C G R}, +author = {C. G. R. Geddes and E. Cormier-Michel and E. Esarey and C. B. Schroeder and W. P. Leemans}, booktitle = {Proc. Particle Accelerator Conference}, title = {{Scaled Simulation Design Of High Quality Laser Wakefield Accelerator Stages}}, year = {2009} @@ -348,10 +347,9 @@ @inproceedings{Geddespac09 @article{LotovPRSTAB2003, author = {Lotov, K. V.}, doi = {10.1103/PhysRevSTAB.6.061301}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lotov - 2003 - Fine wakefield structure in the blowout regime of plasma wakefield accelerators.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {jun}, +month = {Jun}, number = {6}, pages = {061301}, publisher = {American Physical Society}, @@ -361,7 +359,7 @@ @article{LotovPRSTAB2003 year = {2003} } @article{Godfreyjcp75, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -374,7 +372,7 @@ @article{Kishekprl2012 author = {Kishek, R A}, doi = {10.1103/Physrevlett.108.035003}, journal = {Phys. Rev. Lett.}, -month = {jan}, +month = {Jan}, number = {3}, pages = {35003}, publisher = {American Physical Society}, @@ -392,12 +390,12 @@ @article{Zhang2016 } @article{Cohennim2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -405,10 +403,11 @@ @article{Cohennim2009 year = {2009} } @article{Osiris, -author = {{Fonseca et al.}, R A}, -journal = {Lec. Notes In Comp. Sci.}, -pages = {342}, -title = {{No Title}}, +author = {Fonseca, R. A. and Silva, L. O. and Tsung, F. S. and Decyk, V. K. and Lu, W. and Ren, C. and Mori, W. B. and Deng, S. and Lee, S. and Katsouleas, T. and Adam, J. C.}, +booktitle = {Computational Science --- ICCS 2002}, +pages = {342--351}, +publisher = {Springer Berlin Heidelberg}, +title = {{OSIRIS: A Three-Dimensional, Fully Relativistic Particle in Cell Code for Modeling Plasma Based Accelerators}}, volume = {2329}, year = {2002} } @@ -439,7 +438,7 @@ @article{GeddesPRL2008 author = {Geddes, C G R and Nakamura, K and Plateau, G R and Toth, Cs. and Cormier-Michel, E and Esarey, E and Schroeder, C B and Cary, J R and Leemans, W P}, doi = {10.1103/PhysRevLett.100.215004}, journal = {Phys. Rev. Lett.}, -month = {may}, +month = {May}, number = {21}, pages = {215004}, publisher = {American Physical Society}, @@ -465,12 +464,6 @@ @misc{Huebl2015 url = {http://dx.doi.org/10.5281/zenodo.591699}, year = {2015} } -@article{Vayarxiv10_1, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1009.2727V2}, -title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Huangscidac09, author = {Huang, C and An, W and Decyk, V K and Lu, W and Mori, W B and Tsung, F S and Tzoufras, M and Morshed, S and Antonsen, T and Feng, B and Katsouleas, T and Fonseca, R A and Martins, S F and Vieira, J and Silva, L O and Esarey, E and Geddes, C G R and Leemans, W P and Cormier-Michel, E and Vay, J.-L. and Bruhwiler, D L and Cowan, B and Cary, J R and Paul, K}, issn = {1742-6596}, @@ -494,7 +487,7 @@ @article{Cormierprstab10 title = {{Propagation Of Higher Order Modes In Plasma Channels And Shaping Of The Transverse Field In Laser Plasma Accelerators}} } @article{Yee, -author = {Yee, Ks}, +author = {Yee, K S}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, number = {3}, @@ -521,7 +514,7 @@ @inproceedings{Cormieraac08 booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, pages = {297--302}, -title = {{Scaled Simulations Of A 10 Gev Accelerator}}, +title = {{Scaled Simulations Of A 10 GeV Accelerator}}, volume = {1086}, year = {2009} } @@ -536,11 +529,11 @@ @inproceedings{Bruhwileraac08 } @article{Vaynim2005, annote = {15Th International Symposium On Heavy Ion Inertial Fusion, Princeton, Nj, Jun 07-11, 2004}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, doi = {10.1016/J.Nima.2005.01.232}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {may}, +month = {May}, number = {1-2}, pages = {347--352}, title = {{Application Of Adaptive Mesh Refinement To Pic Simulations In Heavy Ion Fusion}}, @@ -560,11 +553,11 @@ @article{BulanovSV2014 year = {2014} } @article{Furmanprstab2002, -author = {Furman, Ma and Pivi, Mtf}, +author = {Furman, M A and Pivi, M T F}, doi = {10.1103/Physrevstab.5.124404}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {dec}, +month = {Dec}, number = {12}, title = {{Probabilistic Model For The Simulation Of Secondary Electron Emission}}, volume = {5}, @@ -573,10 +566,9 @@ @article{Furmanprstab2002 @article{Qiang2014, author = {Qiang, J. and Corlett, J. and Mitchell, C. E. and Papadopoulos, C. F. and Penn, G. and Placidi, M. and Reinsch, M. and Ryne, R. D. and Sannibale, F. and Sun, C. and Venturini, M. and Emma, P. and Reiche, S.}, doi = {10.1103/PhysRevSTAB.17.030701}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Qiang et al. - 2014 - Start-to-end simulation of x-ray radiation of a next generation light source using the real number of electrons.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {030701}, publisher = {American Physical Society}, @@ -601,13 +593,12 @@ @misc{Bruhwilerpc08 year = {2008} } @article{Yu2014, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and An, Weiming and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2014.02.016}, issn = {00219991}, journal = {Journal of Computational Physics}, keywords = {Boosted frame simulation,IN-CELL CODE,INSTABILITIES,Laser wakefield accelerator,Numerical Cerenkov instability,PARTICLE SIMULATION,PLASMAS,Particle-in-cell,Plasma simulation,Spectral solver}, -month = {jun}, +month = {Jun}, pages = {124--138}, publisher = {ACADEMIC PRESS INC ELSEVIER SCIENCE, 525 B ST, STE 1900, SAN DIEGO, CA 92101-4495 USA}, title = {{Modeling of laser wakefield acceleration in Lorentz boosted frame using EM-PIC code with spectral solver}}, @@ -619,10 +610,10 @@ @article{Vaypop2011 author = {Vay, J.-L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, doi = {10.1063/1.3663841}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {dec}, +journal = {Physics of Plasmas}, +month = {Dec}, number = {12}, -title = {{Modeling Of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, +title = {{Modeling Of 10 GeV-1 TeV Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, volume = {18}, year = {2011} } @@ -631,7 +622,7 @@ @article{LehePRSTAB13 doi = {10.1103/PhysRevSTAB.16.021301}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{Numerical growth of emittance in simulations of laser-wakefield acceleration}}, volume = {16}, @@ -641,7 +632,7 @@ @article{Langdoncpc92 author = {Langdon, A B}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {jul}, +month = {Jul}, number = {3}, pages = {447--450}, title = {{On Enforcing Gauss Law In Electromagnetic Particle-In-Cell Codes}}, @@ -653,31 +644,29 @@ @article{Colellajcp2010 doi = {10.1016/J.Jcp.2009.07.004}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {4}, pages = {947--957}, title = {{Controlling Self-Force Errors At Refinement Boundaries For Amr-Pic}}, volume = {229}, year = {2010} } - @article{Coleieee1997, author = {Cole, J. B.}, issn = {0018-9480}, journal = {Ieee Transactions On Microwave Theory And Techniques}, -month = {jun}, +month = {Jun}, number = {6}, pages = {991--996}, title = {{A High-Accuracy Realization Of The Yee Algorithm Using Non-Standard Finite Differences}}, volume = {45}, year = {1997} } - @article{Coleieee2002, author = {Cole, J. B.}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, -month = {sep}, +month = {Sep}, number = {9}, pages = {1185--1191}, title = {{High-Accuracy Yee Algorithm Based On Nonstandard Finite Differences: New Developments And Verifications}}, @@ -685,7 +674,6 @@ @article{Coleieee2002 year = {2002}, doi = {10.1109/Tap.2002.801268}, } - @article{HajimaNIM09, annote = {Workshop on Compton Sources for X-gamma Rays, Porto Conte, ITALY, SEP 07-12, 2008}, author = {Hajima, R and Kikuzawa, N and Nishimori, N and Hayakawa, T and Shizuma, T and Kawase, K and Kando, M and Minehara, E and Toyokawa, H and Ohgaki, H}, @@ -693,7 +681,7 @@ @article{HajimaNIM09 institution = {Ist Nazl Fis Nucl; ICFA}, issn = {0168-9002}, journal = {NUCLEAR INSTRUMENTS {\&} METHODS IN PHYSICS RESEARCH SECTION A-ACCELERATORS SPECTROMETERS DETECTORS AND ASSOCIATED EQUIPMENT}, -month = {sep}, +month = {Sep}, number = {1}, pages = {S57--S61}, title = {{Detection of radioactive isotopes by using laser Compton scattered gamma-ray beams}}, @@ -705,7 +693,7 @@ @article{MatlisJOSA11 doi = {10.1364/JOSAB.28.000023}, issn = {0740-3224}, journal = {JOURNAL OF THE OPTICAL SOCIETY OF AMERICA B-OPTICAL PHYSICS}, -month = {jan}, +month = {Jan}, number = {1}, pages = {23--27}, title = {{Single-shot spatiotemporal measurements of ultrashort THz waveforms using temporal electric-field cross correlation}}, @@ -713,7 +701,6 @@ @article{MatlisJOSA11 year = {2011} } @article{Quickpic2, - author = {An, Weiming and Decyk, Viktor K. and Mori, Warren B. and Antonsen, Thomas M.}, doi = {10.1016/j.jcp.2013.05.020}, issn = {00219991}, @@ -730,29 +717,32 @@ @misc{Cowanpriv2010 year = {2010} } @inproceedings{Geddesscidac09, -author = {{Geddes et al.}, Cgr}, +author = {Geddes, Cameron G.R. and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and Hamill, Paul and Messmer, Peter and Mullowney, Paul and Nieter, Chet and Paul, Kevin and Shasharina, Svetlana and Veitzer, Seth and Weber, Gunther and Rubel, Oliver and Ushizima, Daniela and Bethel, Wes and Wu, John}, booktitle = {Scidac Review 13}, -pages = {13}, +journal = {SciDAC Review}, +number = {13}, +pages = {13--21}, title = {{Laser Plasma Particle Accelerators: Large Fields For Smaller Facility Sources}}, +url = {https://www.osti.gov/biblio/971264}, year = {2009} } @article{Gomberoffpop2007, author = {Gomberoff, K and Fajans, J and Friedman, A and Grote, D and Vay, J.-L. and Wurtele, J S}, doi = {10.1063/1.2778420}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {oct}, +journal = {Physics of Plasmas}, +month = {Oct}, number = {10}, title = {{Simulations Of Plasma Confinement In An Antihydrogen Trap}}, volume = {14}, year = {2007} } @article{Morapop1997, -author = {Mora, P and Antonsen, Tm}, +author = {Mora, P and Antonsen, T M}, doi = {10.1063/1.872134}, issn = {1070-664X}, journal = {Phys. Plasmas}, -month = {jan}, +month = {Jan}, number = {1}, pages = {217--229}, title = {{Kinetic Modeling Of Intense, Short Laser Pulses Propagating In Tenuous Plasmas}}, @@ -770,12 +760,12 @@ @inproceedings{Cowanaac08 } @article{Cohenprstab2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -783,7 +773,6 @@ @article{Cohenprstab2009 year = {2009} } @article{Kaganovich2012, - author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean-Luc and Friedman, Alex}, journal = {Nuclear Instruments and Methods in Physics Research, Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, keywords = {Beam dynamics,Longitudinal compression,Voltage errors}, @@ -792,20 +781,22 @@ @article{Kaganovich2012 volume = {678}, year = {2012} } -@article{Vincenti2016, -author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J.-L.}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Vincenti et al. - 2016 - An efficient and portable SIMD algorithm for chargecurrent deposition in Particle-In-Cell codes.pdf:pdf}, -journal = {Computer Programs in Physics}, -pages = {To appear}, -title = {{An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}}, -url = {https://arxiv.org/abs/1601.02056}, -year = {2016} +@article{VincentiCPC2017, +author = {H. Vincenti and M. Lobet and R. Lehe and R. Sasanka and J.-L. Vay}, +doi = {https://doi.org/10.1016/j.cpc.2016.08.023}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +pages = {145-154}, +title = {An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465516302764}, +volume = {210}, +year = {2017}, } @article{Vaypop2008, author = {Vay, J-L}, doi = {10.1063/1.2837054}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56701}, title = {{Simulation Of Beams Or Plasmas Crossing At Relativistic Velocity}}, @@ -817,7 +808,7 @@ @article{Wieland2016 doi = {10.3847/0004-637X/820/1/62}, issn = {1538-4357}, journal = {The Astrophysical Journal}, -month = {mar}, +month = {Mar}, number = {1}, pages = {62}, title = {{NONRELATIVISTIC PERPENDICULAR SHOCKS MODELING YOUNG SUPERNOVA REMNANTS: NONSTATIONARY DYNAMICS AND PARTICLE ACCELERATION AT FORWARD AND REVERSE SHOCKS}}, @@ -831,11 +822,11 @@ @misc{Vay url = {http://www.nersc.gov/assets/Uploads/DOEExascaleReviewVay.pdf} } @article{Faurenature04, -author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, Jp and Burgy, F and Malka, V}, +author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, J P and Burgy, F and Malka, V}, doi = {10.1038/Nature02963}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {541--544}, title = {{A Laser-Plasma Accelerator Producing Monoenergetic Electron Beams}}, @@ -843,11 +834,11 @@ @article{Faurenature04 year = {2004} } @article{Greenwoodjcp04, -author = {Greenwood, Ad and Cartwright, Kl and Luginsland, Jw and Baca, Ea}, +author = {Greenwood, A D and Cartwright, K L and Luginsland, J W and Baca, E A}, doi = {10.1016/J.Jcp.2004.06.021}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {665--684}, title = {{On The Elimination Of Numerical Cerenkov Radiation In Pic Simulations}}, @@ -855,7 +846,6 @@ @article{Greenwoodjcp04 year = {2004} } @article{GodfreyIEEE2014, - author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {IEEE Transactions on Plasma Science}, keywords = {Accelerators,numerical stability,particle beams,particle-in-cell (PIC),relativistic effects,simulation,spectral methods}, @@ -874,23 +864,28 @@ @inproceedings{Cowanicap09 year = {2009} } @article{Wuprstab2011, -author = {Wu, H -C. and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, +author = {Wu, H-C and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, doi = {10.1103/Physrevstab.14.070702}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {jul}, +month = {Jul}, number = {7}, title = {{Nonlinear Coherent Thomson Scattering From Relativistic Electron Sheets As A Means To Produce Isolated Ultrabright Attosecond X-Ray Pulses}}, volume = {14}, year = {2011} } @article{VayAAC2010, -author = {Vay, J-L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, +author = {Vay, J.‐L. and Geddes, C. G. R. and Benedetti, C. and Bruhwiler, D. L. and Cormier‐Michel, E. and Cowan, B. M. and Cary, J. R. and Grote, D. P.}, doi = {10.1063/1.3520322}, -journal = {Aip Conference Proceedings}, +eprint = {https://pubs.aip.org/aip/acp/article-pdf/1299/1/244/11928106/244\_1\_online.pdf}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, +month = {Nov}, +number = {1}, pages = {244--249}, title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, +url = {https://doi.org/10.1063/1.3520322}, volume = {1299}, year = {2010} } @@ -905,21 +900,23 @@ @article{Vayprl07 year = {2007} } @article{VayJCP2013, - author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B.}, +doi = {10.1016/j.jcp.2013.03.010}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Domain decomposition,Electromagnetic,FFT,Fast fourier transform,Parallel,Particle-In-Cell,Spectral}, +month = {Jun}, pages = {260--268}, title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, volume = {243}, year = {2013} } @article{Kaganovichpop2004, -author = {Kaganovich, Id and Startsev, Ea and Davidson, Rc}, +author = {Kaganovich, I D and Startsev, E A and Davidson, R C}, doi = {10.1063/1.1758945}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {jul}, +journal = {Physics of Plasmas}, +month = {Jul}, number = {7}, pages = {3546--3552}, title = {{Nonlinear Plasma Waves Excitation By Intense Ion Beams In Background Plasma}}, @@ -928,12 +925,12 @@ @article{Kaganovichpop2004 } @article{Cohenpop2005, annote = {46Th Annual Meeting Of The Division Of Plasma Physics Of The American-Physical-Society, Savannah, Ga, Nov 15-19, 2004}, -author = {Cohen, Rh and Friedman, A and Covo, Mk and Lund, Sm and Molvik, Aw and Bieniosek, Fm and Seidl, Pa and Vay, Jl and Stoltz, P and Veitzer, S}, +author = {Cohen, R H and Friedman, A and Covo, M K and Lund, S M and Molvik, A W and Bieniosek, F M and Seidl, P A and Vay, J-L and Stoltz, P and Veitzer, S}, doi = {10.1063/1.1882292}, institution = {Amer Phys Soc}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Simulating Electron Clouds In Heavy-Ion Accelerators}}, volume = {12}, @@ -941,11 +938,11 @@ @article{Cohenpop2005 } @article{Vayfed1996, annote = {7Th International Symposium On Heavy Ion Inertial Fusion, Princeton Plasma Phys Lab, Princeton, Nj, Sep 06-09, 1995}, -author = {Vay, Jl and Deutsch, C}, +author = {Vay, J-L and Deutsch, C}, doi = {10.1016/S0920-3796(96)00502-9}, issn = {0920-3796}, journal = {Fusion Engineering And Design}, -month = {nov}, +month = {Nov}, pages = {467--476}, title = {{A Three-Dimensional Electromagnetic Particle-In-Cell Code To Simulate Heavy Ion Beam Propagation In The Reaction Chamber}}, volume = {32-33}, @@ -962,10 +959,8 @@ @article{Fengjcp09 year = {2009} } @article{BESAC2013, - author = {BESAC}, doi = {10.1037/a0034573}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/582d1799e353fccb9458ff0ccb827db4b01d7308.pdf:pdf}, issn = {1935-990X}, journal = {The American psychologist}, number = {7}, @@ -977,11 +972,11 @@ @article{BESAC2013 year = {2013} } @article{Prostprstab2005, -author = {Prost, Lr and Seidl, Pa and Bieniosek, Fm and Celata, Cm and Faltens, A and Baca, D and Henestroza, E and Kwan, Jw and Leitner, M and Waldron, Wl and Cohen, R and Friedman, A and Grote, D and Lund, Sm and Molvik, Aw and Morse, E}, +author = {Prost, L R and Seidl, P A and Bieniosek, F M and Celata, C M and Faltens, A and Baca, D and Henestroza, E and Kwan, J W and Leitner, M and Waldron, W L and Cohen, R and Friedman, A and Grote, D and Lund, S M and Molvik, A W and Morse, E}, doi = {10.1103/Physrevstab.8.020101}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{High Current Transport Experiment For Heavy Ion Inertial Fusion}}, volume = {8}, @@ -1000,13 +995,12 @@ @inproceedings{Godfrey2013PPPS title = {{Numerical Stability of the Pseudo-Spectral EM PIC Algorithm}}, year = {2013} } - @article{Vaypop04, author = {Vay, J.-L. and Colella, P and Kwan, J W and Mccorquodale, P and Serafini, D B and Friedman, A and Grote, D P and Westenskow, G and Adam, J.-C. and Heron, A and Haber, I}, doi = {10.1063/1.1689669}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {2928--2934}, title = {{Application Of Adaptive Mesh Refinement To Particle-In-Cell Simulations Of Plasmas And Beams}}, @@ -1017,7 +1011,7 @@ @article{Friedmanjcp90 author = {Friedman, A}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {292--312}, title = {{A 2Nd-Order Implicit Particle Mover With Adjustable Damping}}, @@ -1027,8 +1021,8 @@ @article{Friedmanjcp90 @article{Rittershoferpop2010, author = {Rittershofer, W and Schroeder, C B and Esarey, E and Gruner, F J and Leemans, W P}, doi = {10.1063/1.3430638}, -journal = {Physics Of Plasmas}, -month = {jun}, +journal = {Physics of Plasmas}, +month = {Jun}, number = {6}, pages = {63104}, title = {{Tapered Plasma Channels To Phase-Lock Accelerating And Focusing Forces In Laser-Plasma Accelerators}}, @@ -1043,16 +1037,17 @@ @article{Vorpal volume = {196}, year = {2004} } -@article{LiARXIV2016, - -archivePrefix = {arXiv}, -arxivId = {1605.01496}, -author = {Li, Fei and Yu, Peicheng and Xu, Xinlu and Fiuza, Frederico and Decyk, Viktor K. and Dalichaouch, Thamine and Davidson, Asher and Tableman, Adam and An, Weiming and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Mori, Warren B.}, -eprint = {1605.01496}, -month = {may}, -title = {{Controlling the Numerical Cerenkov Instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, -url = {http://arxiv.org/abs/1605.01496}, -year = {2016} +@article{LiCPC2017, +author = {Fei Li and Peicheng Yu and Xinlu Xu and Frederico Fiuza and Viktor K. Decyk and Thamine Dalichaouch and Asher Davidson and Adam Tableman and Weiming An and Frank S. Tsung and Ricardo A. Fonseca and Wei Lu and Warren B. Mori}, +doi = {https://doi.org/10.1016/j.cpc.2017.01.001}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {PIC simulation, Hybrid Maxwell solver, Relativistic plasma drift, Numerical Cerenkov instability, Lorentz boosted frame}, +pages = {6-17}, +title = {{Controlling the numerical Cerenkov instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465517300012}, +volume = {214}, +year = {2017}, } @article{XuJCP2013, author = {Xu, Xinlu and Yu, Peicheng and Martins, Samual F and Tsung, Frank S and Decyk, Viktor K and Vieira, Jorge and Fonseca, Ricardo A and Lu, Wei and Silva, Luis O and Mori, Warren B}, @@ -1072,17 +1067,17 @@ @article{Cormierpre08 doi = {10.1103/Physreve.78.016404}, issn = {1539-3755}, journal = {Physical Review E}, -month = {jul}, +month = {Jul}, number = {1, Part 2}, title = {{Unphysical Kinetic Effects In Particle-In-Cell Modeling Of Laser Wakefield Accelerators}}, volume = {78}, year = {2008} } @article{Berengerjcp96, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {363--379}, title = {{Three-Dimensional Perfectly Matched Layer For The Absorption Of Electromagnetic Waves}}, @@ -1090,15 +1085,20 @@ @article{Berengerjcp96 year = {1996} } @article{Kalmykovprl09, -author = {{Kalmykov et al.}, S}, +author = {Kalmykov, S. and Yi, S. A. and Khudik, V. and Shvets, G.}, +doi = {10.1103/PhysRevLett.103.135004}, +issue = {13}, journal = {Phys. Rev. Lett.}, +month = {Sep}, +numpages = {4}, pages = {135004}, -title = {{No Title}}, +publisher = {American Physical Society}, +title = {{Electron Self-Injection and Trapping into an Evolving Plasma Bubble}}, +url = {https://link.aps.org/doi/10.1103/PhysRevLett.103.135004}, volume = {103}, year = {2009} } @article{Geddes2015, - author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean-Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, journal = {Nuclear Instruments and Methods in Physics Research, Section B: Beam Interactions with Materials and Atoms}, keywords = {Active interrogation,Homeland security,Laser plasma accelerator,Monoenergetic photon source,Nonproliferation}, @@ -1112,7 +1112,7 @@ @article{Antonsenprl1992 author = {Antonsen, T M and Mora, P}, doi = {10.1103/Physrevlett.69.2204}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {15}, pages = {2204--2207}, title = {{Self-Focusing And Raman-Scattering Of Laser-Pulses In Tenuous Plasmas}}, @@ -1120,7 +1120,6 @@ @article{Antonsenprl1992 year = {1992} } @article{GodfreyCPC2015, - author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Computer Physics Communications}, keywords = {Numerical stability,Particle-in-cell,Pseudo-Spectral Time-Domain,Relativistic beam}, @@ -1142,10 +1141,10 @@ @article{VayCSD12 year = {2012} } @article{Esirkepovcpc01, -author = {Esirkepov, Tz}, +author = {Esirkepov, T Z}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {apr}, +month = {Apr}, number = {2}, pages = {144--153}, title = {{Exact Charge Conservation Scheme For Particle-In-Cell Simulation With An Arbitrary Form-Factor}}, @@ -1156,8 +1155,8 @@ @article{Martinspop10 author = {Martins, S F and Fonseca, R A and Vieira, J and Silva, L O and Lu, W and Mori, W B}, doi = {10.1063/1.3358139}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56705}, title = {{Modeling Laser Wakefield Accelerator Experiments With Ultrafast Particle-In-Cell Simulations In Boosted Frames}}, @@ -1166,7 +1165,7 @@ @article{Martinspop10 } @inproceedings{Habericnsp73, address = {Berkeley, Ca}, -author = {Haber, I and Lee, R and Klein, Hh and Boris, Jp}, +author = {Haber, I and Lee, R and Klein, H H and Boris, J P}, booktitle = {Proc. Sixth Conf. Num. Sim. Plasmas}, pages = {46--48}, title = {{Advances In Electromagnetic Simulation Techniques}}, @@ -1177,7 +1176,7 @@ @article{Martinscpc10 doi = {10.1016/J.Cpc.2009.12.023}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {may}, +month = {May}, number = {5}, pages = {869--875}, title = {{Numerical Simulations Of Laser Wakefield Accelerators In Optimal Lorentz Frames}}, @@ -1189,7 +1188,7 @@ @article{Vaycpc04 doi = {10.1016/J.Cpc.2004.06.026}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {dec}, +month = {Dec}, number = {1-3}, pages = {171--177}, title = {{Asymmetric Pml For The Absorption Of Waves. Application To Mesh Refinement In Electromagnetic Particle-In-Cell Plasma Simulations}}, @@ -1197,7 +1196,6 @@ @article{Vaycpc04 year = {2004} } @article{GodfreyJCP2014_PSATD, - author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {Journal of Computational Physics}, keywords = {Numerical stability,Particle-in-cell,Pseudo-spectral,Relativistic beam}, @@ -1211,20 +1209,19 @@ @article{PruetJAP06 doi = {10.1063/1.2202005}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {jun}, +month = {Jun}, number = {12}, title = {{Detecting clandestine material with nuclear resonance fluorescence}}, volume = {99}, year = {2006} } @article{YuCPC2015-Circ, - author = {Yu, Peicheng and Xu, Xinlu and Tableman, Adam and Decyk, Viktor K. and Tsung, Frank S. and Fiuza, Frederico and Davidson, Asher and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.08.026}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,CODES,Hybrid Maxwell solver,IN-CELL SIMULATION,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,OSIRIS,PARTICLE SIMULATION,PIC SIMULATIONS,PIC simulation,PLASMAS,Quasi-3D algorithm,Relativistic plasma drift,STABILITY}, -month = {dec}, +month = {Dec}, pages = {144--152}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit}}, @@ -1233,10 +1230,10 @@ @article{YuCPC2015-Circ year = {2015} } @article{Berengerjcp94, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {185--200}, title = {{A Perfectly Matched Layer For The Absorption Of Electromagnetic-Waves}}, @@ -1253,11 +1250,11 @@ @article{Rubel2016 year = {2016} } @article{Geddesnature04, -author = {Geddes, Cgr and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, Cb and Bruhwiler, D and Nieter, C and Cary, J and Leemans, Wp}, +author = {Geddes, C G R and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, C B and Bruhwiler, D and Nieter, C and Cary, J and Leemans, W P}, doi = {10.1038/Nature02900}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {538--541}, title = {{High-Quality Electron Beams From A Laser Wakefield Accelerator Using Plasma-Channel Guiding}}, @@ -1265,11 +1262,11 @@ @article{Geddesnature04 year = {2004} } @article{Munzjcp2000, -author = {Munz, Cd and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, +author = {Munz, C D and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, doi = {10.1006/Jcph.2000.6507}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {2}, pages = {484--511}, title = {{Divergence Correction Techniques For Maxwell Solvers Based On A Hyperbolic Model}}, @@ -1277,8 +1274,7 @@ @article{Munzjcp2000 year = {2000} } @article{DavidsonJCP2015, - -author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F.S. and Lu, W. and Vieira, J. and Fonseca, R.A. and Silva, L.O. and Mori, W.B.}, +author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F. S. and Lu, W. and Vieira, J. and Fonseca, R. A. and Silva, L. O. and Mori, W. B.}, doi = {10.1016/j.jcp.2014.10.064}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1301,7 +1297,6 @@ @article{GodfreyJCP2014 year = {2014} } @article{Godfrey2013, - author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Esirkepov algorithm,Numerical stability,Particle-in-cell,Relativistic beam}, @@ -1316,8 +1311,8 @@ @article{Gilsonpop2010 doi = {10.1063/1.3354109}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Studies Of Emittance Growth And Halo Particle Production In Intense Charged Particle Beams Using The Paul Trap Simulator Experiment}}, volume = {17}, @@ -1348,7 +1343,7 @@ @article{Steinke2016 author = {Steinke, S and van Tilborg, J and Benedetti, C and Geddes, C G R and Schroeder, C B and Daniels, J and Swanson, K K and Gonsalves, A J and Nakamura, K and Matlis, N H and Shaw, B H and Esarey, E and Leemans, W P}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7589}, pages = {190--193}, publisher = {Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.}, @@ -1376,7 +1371,7 @@ @article{Genonioppj2010 year = {2010} } @article{LewisJCP1972, -author = {Lewis, H.Ralph}, +author = {Lewis, H. Ralph}, doi = {http://dx.doi.org/10.1016/0021-9991(72)90044-7}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1400,11 +1395,11 @@ @article{Vay2002 year = {2002} } @article{Manglesnature04, -author = {Mangles, Spd and Murphy, Cd and Najmudin, Z and Thomas, Agr and Collier, Jl and Dangor, Ae and Divall, Ej and Foster, Ps and Gallacher, Jg and Hooker, Cj and Jaroszynski, Da and Langley, Aj and Mori, Wb and Norreys, Pa and Tsung, Fs and Viskup, R and Walton, Br and Krushelnick, K}, +author = {Mangles, S P D and Murphy, C D and Najmudin, Z and Thomas, A G R and Collier, J L and Dangor, A E and Divall, E J and Foster, P S and Gallacher, J G and Hooker, C J and Jaroszynski, D A and Langley, A J and Mori, W B and Norreys, P A and Tsung, F S and Viskup, R and Walton, B R and Krushelnick, K}, doi = {10.1038/Nature02939}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {535--538}, title = {{Monoenergetic Beams Of Relativistic Electrons From Intense Laser-Plasma Interactions}}, @@ -1423,8 +1418,8 @@ @inproceedings{Schroederaac08 @article{Vaypop98, author = {Vay, J.-L. and Deutsch, C}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {apr}, +journal = {Physics of Plasmas}, +month = {Apr}, number = {4}, pages = {1190--1197}, title = {{Charge Compensated Ion Beam Propagation In A Reactor Sized Chamber}}, @@ -1432,11 +1427,11 @@ @article{Vaypop98 year = {1998} } @article{Friedmanjcp1991, -author = {Friedman, A and Parker, Se and Ray, Sl and Birdsall, Ck}, +author = {Friedman, A and Parker, S E and Ray, S L and Birdsall, C K}, doi = {10.1016/0021-9991(91)90265-M}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {1}, pages = {54--70}, title = {{Multiscale Particle-In-Cell Plasma Simulation}}, @@ -1444,11 +1439,11 @@ @article{Friedmanjcp1991 year = {1991} } @article{Liumotl1997, -author = {Liu, Qh}, +author = {Liu, Q H}, doi = {10.1002/(Sici)1098-2760(19970620)15:3<158::Aid-Mop11>3.3.Co;2-T}, issn = {0895-2477}, journal = {Microwave And Optical Technology Letters}, -month = {jun}, +month = {Jun}, number = {3}, pages = {158--165}, title = {{The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength}}, @@ -1475,45 +1470,19 @@ @article{Leemansphysicstoday10 author = {Leemans, Wim and Esarey, Eric}, issn = {0031-9228}, journal = {Physics Today}, -month = {mar}, +month = {Mar}, number = {3}, pages = {44--49}, title = {{Laser-Driven Plasma-Wave Electron Accelerators}}, volume = {62}, year = {2009} } -@article{Lehe2015, - -archivePrefix = {arXiv}, -arxivId = {1507.04790}, -author = {Lehe, Remi and Kirchen, Manuel and Andriyash, Igor a. and Godfrey, Brendan B. and Vay, Jean-Luc}, -eprint = {1507.04790}, -isbn = {5104866785}, -journal = {arXiv.org}, -keywords = {cylindrical geometry,hankel transform,particle-in-cell,pseudo-spectral}, -pages = {1507.04790v1}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -url = {http://arxiv.org/abs/1507.04790}, -volume = {physics.pl}, -year = {2015} -} -@article{VayJCP13, -author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B}, -doi = {10.1016/j.jcp.2013.03.010}, -issn = {0021-9991}, -journal = {Journal of Computational Physics}, -month = {jun}, -pages = {260--268}, -title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, -volume = {243}, -year = {2013} -} @inproceedings{Martinspac09, address = {Vancouver, Canada}, annote = {Th4Gbc05}, -author = {{Martins et al.}, S F}, +author = {Martins, S F and Fonseca, R A and Silva, L O and Mori, W B}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Boosted Frame Pic Simulations Of Lwfa: Towards The Energy Frontier}}, +title = {{Boosted Frame PIC Simulations of LWFA: Towards the Energy Frontier}}, year = {2009} } @article{Vayscidac09, @@ -1535,7 +1504,7 @@ @article{Sprangleprl90 author = {Sprangle, P and Esarey, E and Ting, A}, issn = {0031-9007}, journal = {Physical Review Letters}, -month = {apr}, +month = {Apr}, number = {17}, pages = {2011--2014}, title = {{Nonlinear theory of intense laser-plasma interactions}}, @@ -1548,19 +1517,19 @@ @article{Molvikpop2007 doi = {10.1063/1.2436850}, institution = {Aps, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Quantitative Experiments With Electrons In A Positively Charged Beam}}, volume = {14}, year = {2007} } @article{QuiterJAP08, -author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M -A.}, +author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M-A}, doi = {10.1063/1.2876028}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {mar}, +month = {Mar}, number = {6}, title = {{A method for high-resolution x-ray imaging of intermodal cargo containers for fissionable materials}}, volume = {103}, @@ -1568,7 +1537,7 @@ @article{QuiterJAP08 } @book{Taflove2000, address = {Norwood}, -author = {Taflove and Hagness}, +author = {Allen Taflove and Susan C. Hagness}, edition = {2Nd}, publisher = {Ma: Artech House}, title = {{Computational Electrodynamics: The Finite-Difference Time-Domain Method}}, @@ -1578,7 +1547,7 @@ @article{Schroederprl2011 author = {Schroeder, C B and Benedetti, C and Esarey, E and Leemans, W P}, doi = {10.1103/Physrevlett.106.135002}, journal = {Physical Review Letters}, -month = {mar}, +month = {Mar}, number = {13}, pages = {135002}, title = {{Nonlinear Pulse Propagation And Phase Velocity Of Laser-Driven Plasma Waves}}, @@ -1600,10 +1569,8 @@ @article{Logannim2007 year = {2007} } @article{Yu2016, - author = {Yu, Peicheng and Xu, Xinlu and Davidson, Asher and Tableman, Adam and Dalichaouch, Thamine and Li, Fei and Meyers, Michael D. and An, Weiming and Tsung, Frank S. and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2016.04.014}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Yu et al. - 2016 - Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry.pdf:pdf}, issn = {00219991}, journal = {Journal of Computational Physics}, title = {{Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}}, @@ -1611,7 +1578,7 @@ @article{Yu2016 } @article{Borisjcp73, address = {525 B St, Ste 1900, San Diego, Ca 92101-4495}, -author = {Boris, Jp and Lee, R}, +author = {Boris, J P and Lee, R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -1624,7 +1591,7 @@ @article{Borisjcp73 } @inproceedings{BorisICNSP70, address = {Naval Res. Lab., Wash., D. C.}, -author = {Boris, Jp}, +author = {Boris, J P}, booktitle = {Proc. Fourth Conf. Num. Sim. Plasmas}, pages = {3--67}, title = {{Relativistic Plasma Simulation-Optimization of a Hybrid Code}}, @@ -1638,7 +1605,7 @@ @article{Vayjcp01 author = {Vay, J.-L.}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {1}, pages = {72--98}, title = {{An Extended Fdtd Scheme For The Wave Equation: Application To Multiscale Electromagnetic Simulation}}, @@ -1663,10 +1630,10 @@ @article{Leemansnature06 doi = {10.1038/Nphys418}, issn = {1745-2473}, journal = {Nature Physics}, -month = {oct}, +month = {Oct}, number = {10}, pages = {696--699}, -title = {{Gev Electron Beams From A Centimetre-Scale Accelerator}}, +title = {{GeV electron beams from a centimetre-scale accelerator}}, volume = {2}, year = {2006} } @@ -1675,7 +1642,7 @@ @article{ChenPRSTAB13 doi = {10.1103/PhysRevSTAB.16.030701}, issn = {1098-4402}, journal = {PHYSICAL REVIEW SPECIAL TOPICS-ACCELERATORS AND BEAMS}, -month = {mar}, +month = {Mar}, number = {3}, title = {{Modeling classical and quantum radiation from laser-plasma accelerators}}, volume = {16}, @@ -1686,7 +1653,7 @@ @article{Hipace doi = {10.1088/0741-3335/56/8/084012}, issn = {0741-3335}, journal = {Plasma Physics and Controlled Fusion}, -month = {aug}, +month = {Aug}, number = {8}, pages = {084012}, publisher = {IOP Publishing}, @@ -1696,7 +1663,7 @@ @article{Hipace year = {2014} } @article{Morsenielson1971, -author = {Morse, Rl and Nielson, Cw}, +author = {Morse, R L and Nielson, C W}, doi = {10.1063/1.1693518}, issn = {1070-6631}, journal = {Phys. Fluids}, @@ -1706,14 +1673,17 @@ @article{Morsenielson1971 volume = {14}, year = {1971} } -@article{Vaydpf09, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:0909.5603}, -title = {{Speeding Up Simulations Of Relativistic Systems Using An Optimal Boosted Frame}}, +@inproceedings{Vaydpf09, +archivePrefix = {arXiv}, +author = {Vay, J.-L. and Fawley, W. M. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, +booktitle = {{Meeting of the Division of Particles and Fields of the American Physical Society (DPF 2009)}}, +eprint = {0909.5603}, +month = {Sep}, +primaryClass = {physics.acc-ph}, +title = {{Speeding up simulations of relativistic systems using an optimal boosted frame}}, year = {2009} } @misc{Vay2014, - author = {Vay, Jean-Luc and Godfrey, Brendan B.}, booktitle = {Comptes Rendus - Mecanique}, keywords = {Numerical instability,Particle-In-Cell,Plasma simulation,Special relativity}, @@ -1724,14 +1694,7 @@ @misc{Vay2014 volume = {342}, year = {2014} } -@article{Lehearxiv2015, -author = {Lehe, R and Kirchen, M and Andriyash, I.{\~{}}A. and Godfrey, B.{\~{}}B. and Vay, J.-L.}, -journal = {arXiv:1507.04790}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -year = {2015} -} @article{Friedman2014, - author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean-Luc and Haber, Irving and Kishek, Rami A.}, journal = {IEEE Transactions on Plasma Science}, keywords = {Algorithms,Maxwell,Ned Birdsall,computer,laser,numerical simulation,particle beam,particle-in-cell,plasma}, @@ -1743,7 +1706,6 @@ @article{Friedman2014 year = {2014} } @article{Lcode, - author = {Lotov, K. V.}, doi = {10.1063/1.872765}, issn = {1070664X}, @@ -1770,9 +1732,7 @@ @article{GodfreyJCP2014_2 year = {2014} } @article{Turbowave, - author = {Gordon, Daniel F and Mori, W B and Antonsen, Thomas M}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Gordon, Mori, Antonsen - 2000 - A Ponderomotive Guiding Center Particle-in-Cell Code for Efficient Modeling of Laser–Plasma Interactio.pdf:pdf}, journal = {IEEE TRANSACTIONS ON PLASMA SCIENCE}, keywords = {Index Terms—Particle code,plasma simulation}, number = {4}, @@ -1787,7 +1747,7 @@ @article{Habernim2009 institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {64--68}, title = {{Scaled Electron Studies At The University Of Maryland}}, @@ -1805,13 +1765,12 @@ @article{Folegatijpcs2011 year = {2011} } @article{YuCPC2015, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.02.018}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,Numerical dispersion relation,PARTICLE SIMULATION,PLASMA,Particle-in-cell,Plasma simulation,Relativistic drifting plasma,SHOCKS,STABILITY,Spectral solver,WAVES}, -month = {jul}, +month = {Jul}, pages = {32--47}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Elimination of the numerical Cerenkov instability for spectral EM-PIC codes}}, @@ -1824,19 +1783,13 @@ @article{PukhovJPP99 doi = {10.1017/S0022377899007515}, issn = {0022-3778}, journal = {Journal of Plasma Physics}, -month = {apr}, +month = {Apr}, number = {3}, pages = {425--433}, title = {{Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab)}}, volume = {61}, year = {1999} } -@article{Vincentiarxiv2015, -author = {Vincenti, H and Vay, J.-L.}, -journal = {arXiv:1507.05572}, -title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, -year = {2015} -} @inproceedings{Vayipac10, address = {Tokyo, Japan}, annote = {Weobra02}, @@ -1849,7 +1802,7 @@ @article{Cormierprstab2011 author = {Cormier-Michel, E and Esarey, E and Geddes, C G R and Schroeder, C B and Paul, K and Mullowney, P J and Cary, J R and Leemans, W P}, doi = {10.1103/Physrevstab.14.031303}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {31303}, title = {{Control Of Focusing Fields In Laser-Plasma Accelerators Using Higher-Order Modes}}, @@ -1857,10 +1810,8 @@ @article{Cormierprstab2011 year = {2011} } @article{Lehe2016, - author = {Lehe, R{\'{e}}mi and Kirchen, Manuel and Andriyash, Igor A. and Godfrey, Brendan B. and Vay, Jean-Luc}, doi = {10.1016/j.cpc.2016.02.007}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lehe et al. - 2016 - A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.pdf:pdf}, issn = {00104655}, journal = {Computer Physics Communications}, pages = {66--82}, @@ -1877,7 +1828,6 @@ @book{Birdsalllangdon year = {1991} } @inproceedings{Grote2005, - author = {Grote, David P. and Friedman, Alex and Vay, Jean-Luc and Haber, Irving}, booktitle = {AIP Conference Proceedings}, pages = {55--58}, @@ -1905,7 +1855,7 @@ @article{LeemansPRL2014 author = {Leemans, W P and Gonsalves, A J and Mao, H.-S. and Nakamura, K and Benedetti, C and Schroeder, C B and T{\'{o}}th, Cs. and Daniels, J and Mittelberger, D E and Bulanov, S S and Vay, J.-L. and Geddes, C G R and Esarey, E}, doi = {10.1103/PhysRevLett.113.245002}, journal = {Phys. Rev. Lett.}, -month = {dec}, +month = {Dec}, number = {24}, pages = {245002}, publisher = {American Physical Society}, @@ -1918,7 +1868,7 @@ @article{Abejcp86 author = {Abe, H and Sakairi, N and Itatani, R and Okuda, H}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {apr}, +month = {Apr}, number = {2}, pages = {247--267}, title = {{High-Order Spline Interpolations In The Particle Simulation}}, @@ -1938,11 +1888,11 @@ @article{LeeCPC2015 year = {2015} } @article{Vaylpb2002, -author = {Vay, Jl and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, D P}, doi = {10.1017/S0263034602204139}, issn = {0263-0346}, journal = {Laser And Particle Beams}, -month = {dec}, +month = {Dec}, number = {4}, pages = {569--575}, title = {{Mesh Refinement For Particle-In-Cell Plasma Simulations: Applications To And Benefits For Heavy Ion Fusion}}, @@ -1962,7 +1912,7 @@ @article{Tzoufrasprl2008 author = {Tzoufras, M and Lu, W and Tsung, F S and Huang, C and Mori, W B and Katsouleas, T and Vieira, J and Fonseca, R A and Silva, L O}, doi = {10.1103/Physrevlett.101.145002}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {14}, pages = {145002}, title = {{Beam Loading In The Nonlinear Regime Of Plasma-Based Acceleration Rid C-6436-2009 Rid B-7680-2009 Rid C-3169-2009}}, @@ -1973,7 +1923,7 @@ @article{Bulanovphysfluid1992 author = {Bulanov, S V and Inovenkov, I N and Kirsanov, V I and Naumova, N M and Sakharov, A S}, doi = {10.1063/1.860046}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7}, pages = {1935--1942}, title = {{Nonlinear Depletion Of Ultrashort And Relativistically Strong Laser-Pulses In An Underdense Plasma}}, @@ -1985,7 +1935,7 @@ @article{Martinsnaturephysics10 doi = {10.1038/Nphys1538}, issn = {1745-2473}, journal = {Nature Physics}, -month = {apr}, +month = {Apr}, number = {4}, pages = {311--316}, title = {{Exploring Laser-Wakefield-Accelerator Regimes For Near-Term Lasers Using Particle-In-Cell Simulation In Lorentz-Boosted Frames}}, @@ -1995,8 +1945,8 @@ @article{Martinsnaturephysics10 @article{Friedmanpop10, author = {Friedman, A and Barnard, J J and Cohen, R H and Grote, D P and Lund, S M and Sharp, W M and Faltens, A and Henestroza, E and Jung, J.-Y. and Kwan, J W and Lee, E P and Leitner, M A and Logan, B G and Vay, J.-L. and Waldron, W L and Davidson, R C and Dorf, M and Gilson, E P and Kaganovich, I D}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {056704 (9 Pp.)}, title = {{Beam Dynamics Of The Neutralized Drift Compression Experiment-Ii, A Novel Pulse-Compressing Ion Accelerator}}, @@ -2006,8 +1956,8 @@ @article{Friedmanpop10 @article{Schroederpop2006, author = {Schroeder, C B and Esarey, E and Shadwick, B A and Leemans, W P}, doi = {10.1063/1.2173960}, -journal = {Physics Of Plasmas}, -month = {mar}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, pages = {33103}, title = {{Trapping, Dark Current, And Wave Breaking In Nonlinear Plasma Waves}}, @@ -2016,12 +1966,12 @@ @article{Schroederpop2006 } @article{Friedmanpfb92, annote = {33Rd Annual Meeting Of The Division Of Plasma Physics Of The American Physical Soc, Tampa, Fl, Nov 04-08, 1991}, -author = {Friedman, A and Grote, Dp and Haber, I}, +author = {Friedman, A and Grote, D P and Haber, I}, doi = {10.1063/1.860024}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {0899-8221}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7, Part 2}, pages = {2203--2210}, title = {{3-Dimensional Particle Simulation Of Heavy-Ion Fusion Beams}}, @@ -2039,7 +1989,7 @@ @article{Winklehnerji2010 doi = {10.1088/1748-0221/5/12/P12001}, issn = {1748-0221}, journal = {Journal of Instrumentation}, -month = {dec}, +month = {Dec}, title = {{Comparison Of Extraction And Beam Transport Simulations With Emittance Measurements From The Ecr Ion Source Venus}}, volume = {5}, year = {2010} @@ -2047,19 +1997,18 @@ @article{Winklehnerji2010 @inproceedings{Vaypac09, address = {Vancouver, Canada}, annote = {Tu1Pbi04}, -author = {{Vay et al.}, J.-L.}, +author = {Vay, J-L and Fawley, W M and Geddes, C G R and Cormier-Michel, E and Grote, D P}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Application Of The Reduction Of Scale Range In A Lorentz Boosted Frame To The Numerical Simulation Of Particle Acceleration Devices}}, +title = {{Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices}}, year = {2009} } -@article{Vincenti2016a, - +@article{VincentiCPC2017a, author = {Vincenti, H. and Vay, J.-L.}, doi = {10.1016/j.cpc.2015.11.009}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {3D electromagnetic simulations,ABSORPTION,ALGORITHM,APPROXIMATIONS,CLOSED-FORM EXPRESSIONS,Domain decomposition technique,Effects of stencil truncation errors,PERFECTLY MATCHED LAYER,Perfectly Matched Layers,Pseudo-spectral Maxwell solver,SIMULATIONS,TAYLOR-SERIES,Very high-order Maxwell solver,WAVES}, -month = {mar}, +month = {Mar}, pages = {147--167}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, @@ -2072,7 +2021,7 @@ @article{Vayjcp02 doi = {10.1006/Jcph.2002.7175}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {367--399}, title = {{Asymmetric Perfectly Matched Layer For The Absorption Of Waves}}, @@ -2093,14 +2042,14 @@ @book{Geddesdissertation05 year = {2005} } @article{Tsungpop06, -author = {Tsung, F S and W Lu, and Tzoufras, M and Mori, W B and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, +author = {Tsung, F. S. and Lu, W. and Tzoufras, M. and Mori, W. B. and Joshi, C. and Vieira, J. M. and Silva, L. O. and Fonseca, R. A.}, doi = {10.1063/1.2198535}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56708}, -title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 Tw Lasers}}, +title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 TW Lasers}}, volume = {13}, year = {2006} } @@ -2108,7 +2057,6 @@ @inproceedings{INFERNO address = {Rostock-Warnemünde, Germany}, author = {Benedetti, Carlo and Schroeder, Carl B. and Esarey, Eric and Leemans, Wim P.}, booktitle = {ICAP}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Benedetti et al. - 2012 - Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}ampRNO.pdf:pdf}, pages = {THAAI2}, publisher = {Jacow}, title = {{Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}RNO}}, @@ -2120,7 +2068,7 @@ @article{GonsalvesNP2011 doi = {10.1038/NPHYS2071}, issn = {1745-2473}, journal = {NATURE PHYSICS}, -month = {nov}, +month = {Nov}, number = {11}, pages = {862--866}, title = {{Tunable laser plasma accelerator based on longitudinal density tailoring}}, @@ -2132,14 +2080,14 @@ @article{Ohtsuboprstab2010 doi = {10.1103/Physrevstab.13.044201}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Experimental Study Of Coherent Betatron Resonances With A Paul Trap}}, volume = {13}, year = {2010} } @inproceedings{Warp, -author = {Grote, D P and Friedman, A and Vay, J.-L. and Haber, I}, +author = {Grote, D P and Friedman, A and Vay, J-L and Haber, I}, booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, number = {749}, @@ -2148,11 +2096,11 @@ @inproceedings{Warp year = {2005} } @article{Mccorquodalejcp2004, -author = {Mccorquodale, P and Colella, P and Grote, Dp and Vay, Jl}, +author = {Mccorquodale, P and Colella, P and Grote, D P and Vay, J-L}, doi = {10.1016/J.Jcp.2004.04.022}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {34--60}, title = {{A Node-Centered Local Refinement Algorithm For Poisson's Equation In Complex Geometries}}, @@ -2162,7 +2110,6 @@ @article{Mccorquodalejcp2004 @article{Londrillo2010, author = {Londrillo, P. and Benedetti, C. and Sgattoni, A.}, doi = {10.1016/j.nima.2010.01.055}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Londrillo, Benedetti, Sgattoni - 2010 - Charge preserving high order PIC schemes.pdf:pdf}, issn = {01689002}, journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, number = {1}, @@ -2181,11 +2128,11 @@ @article{GodfreyJCP2014_FDTD year = {2014} } @article{Shortley-Weller, -author = {Shortley, Gh and Weller, R}, +author = {Shortley, G H and Weller, R}, doi = {10.1063/1.1710426}, issn = {0021-8979}, journal = {Journal of Applied Physics}, -month = {may}, +month = {May}, number = {5}, pages = {334--348}, title = {{The Numerical Solution Of Laplace's Equation}}, @@ -2193,34 +2140,50 @@ @article{Shortley-Weller year = {1938} } @article{VayPOPL2011, -author = {Vay, Jl and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, doi = {10.1063/1.3559483}, -journal = {Physics Of Plasmas}, -month = {mar}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.3559483/16019930/030701\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, -pages = {30701}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, +pages = {030701}, +title = {{Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}}, +url = {https://doi.org/10.1063/1.3559483}, volume = {18}, year = {2011} } - -@article{KirchenARXIV2016, -author = {Kirchen, M. and Lehe, R. and Godfrey, B.~B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A.~R.}, -journal = {arXiv:1608.00215}, +@article{KirchenPOP2016, +author = {Kirchen, M. and Lehe, R. and Godfrey, B. B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A. R.}, +doi = {10.1063/1.4964770}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4964770/14024121/100704\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Oct}, +number = {10}, +pages = {100704}, title = {{Stable discrete representation of relativistically drifting plasmas}}, +url = {https://doi.org/10.1063/1.4964770}, +volume = {23}, year = {2016} } - -@article{LeheARXIV2016, -author = {Lehe, R. and Kirchen, M. and Godfrey, B.~B. and Maier, A.~R. and Vay, J.-L.}, -journal = {arXiv:1608.00227}, -title = {{Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates}}, +@article{LehePRE2016, +author = {Lehe, Remi and Kirchen, Manuel and Godfrey, Brendan B. and Maier, Andreas R. and Vay, Jean-Luc}, +doi = {10.1103/PhysRevE.94.053305}, +issue = {5}, +journal = {Phys. Rev. E}, +month = {Nov}, +numpages = {16}, +pages = {053305}, +publisher = {American Physical Society}, +title = {{Elimination of numerical Cherenkov instability in flowing-plasma particle-in-cell simulations by using Galilean coordinates}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.94.053305}, +volume = {94}, year = {2016} } - @book{godfrey1985iprop, -author={Godfrey, B.B.}, -publisher={Defense Technical Information Center}, -title={{The IPROP Three-Dimensional Beam Propagation Code}}, -year={1985}, +author = {Godfrey, B. B.}, +publisher = {Defense Technical Information Center}, +title = {{The IPROP Three-Dimensional Beam Propagation Code}}, +year = {1985} } diff --git a/Docs/source/latex_theory/input_output/input_output.tex b/Docs/source/latex_theory/input_output/input_output.tex index e013e236445..446f3f2b5d6 100644 --- a/Docs/source/latex_theory/input_output/input_output.tex +++ b/Docs/source/latex_theory/input_output/input_output.tex @@ -30,7 +30,7 @@ \subsection{Inputs and outputs in a boosted frame simulation} \subsubsection{Input in a boosted frame simulation} \paragraph{Particles - } -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): \begin{enumerate} \item project positions at $z^*=0$ assuming ballistic propagation diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 0abca79a5d4..d94938775c6 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -1,38 +1,38 @@ @article{TajimaDawson1982, - author = {Tajima, T. and Dawson, J. M.}, - title = {{Laser accelerator by plasma waves}}, - journal = {AIP Conference Proceedings}, - volume = {91}, - number = {1}, - pages = {69-93}, - year = {1982}, - month = {09}, - abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", - issn = {0094-243X}, - doi = {10.1063/1.33805}, - url = {https://doi.org/10.1063/1.33805} +abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", +author = {Tajima, T. and Dawson, J. M.}, +doi = {10.1063/1.33805}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, +month = {Sep}, +number = {1}, +pages = {69--93}, +title = {{Laser accelerator by plasma waves}}, +url = {https://doi.org/10.1063/1.33805}, +volume = {91}, +year = {1982} } @article{Esarey1996, - author={Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, - journal={IEEE Transactions on Plasma Science}, - title={{Overview of plasma-based accelerator concepts}}, - year={1996}, - volume={24}, - number={2}, - pages={252-288}, - doi={10.1109/27.509991} +author = {Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, +doi = {10.1109/27.509991}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {252--288}, +title = {{Overview of plasma-based accelerator concepts}}, +volume = {24}, +year = {1996} } @ARTICLE{Birdsall1991, - author = {Birdsall, C.K.}, - journal = {IEEE Transactions on Plasma Science}, - title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, - year = {1991}, - volume = {19}, - number = {2}, - pages = {65-85}, - doi = {10.1109/27.106800} +author = {Birdsall, C. K.}, +doi = {10.1109/27.106800}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {65--85}, +title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, +volume = {19}, +year = {1991} } @misc{Lim2007, @@ -41,206 +41,294 @@ @misc{Lim2007 language = {eng}, number = {3}, title = {{The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}}, +url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087}, volume = {69}, -year = {2007}, -url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087} +year = {2007} } @article{Turner2013, author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, -title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, +doi = {10.1063/1.4775084}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {20}, +month = {Jan}, +note = {013507}, number = {1}, -year = {2013}, -month = {01}, -issn = {1070-664X}, -doi = {10.1063/1.4775084}, +title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, url = {https://doi.org/10.1063/1.4775084}, -note = {013507} -} - -@misc{winske2022hybrid, -title={{Hybrid codes (massless electron fluid)}}, -author={D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -year={2022}, -eprint={2204.01676}, -archivePrefix={arXiv}, -primaryClass={physics.plasm-ph} +volume = {20}, +year = {2013} } -@incollection{NIELSON1976, -title = {{Particle-Code Models in the Nonradiative Limit}}, -editor = {JOHN KILLEEN}, -series = {Methods in Computational Physics: Advances in Research and Applications}, -publisher = {Elsevier}, -volume = {16}, -pages = {367-388}, -year = {1976}, +@incollection{Nielson1976, +author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, +doi = {10.1016/B978-0-12-460816-0.50015-4}, +editor = {John Killeen}, issn = {0076-6860}, -doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, -author = {CLAIR W. NIELSON and H. RALPH LEWIS} +pages = {367--388}, +publisher = {Elsevier}, +series = {Methods in Computational Physics: Advances in Research and Applications}, +title = {{Particle-Code Models in the Nonradiative Limit}}, +volume = {16}, +year = {1976} } @article{MUNOZ2018, -title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, +author = {P. A. Muñoz and N. Jain and P. Kilian and J. Büchner}, +doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +issn = {0010-4655}, journal = {Computer Physics Communications}, -volume = {224}, +keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method}, pages = {245-264}, -year = {2018}, -issn = {0010-4655}, -doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, url = {https://www.sciencedirect.com/science/article/pii/S0010465517303521}, -author = {P.A. Muñoz and N. Jain and P. Kilian and J. Büchner}, -keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method} +volume = {224}, +year = {2018} } @article{Le2016, +abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, -title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, +doi = {10.1063/1.4943893}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {23}, +month = {Mar}, +note = {032114}, number = {3}, -year = {2016}, -month = {03}, -abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", -issn = {1070-664X}, -doi = {10.1063/1.4943893}, +title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, url = {https://doi.org/10.1063/1.4943893}, -note = {032114} +volume = {23}, +year = {2016} } @article{Stanier2020, -title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, +author = {A. Stanier and L. Chacón and A. Le}, +doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, -volume = {420}, +keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, pages = {109705}, -year = {2020}, -issn = {0021-9991}, -doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, url = {https://www.sciencedirect.com/science/article/pii/S0021999120304794}, -author = {A. Stanier and L. Chacón and A. Le}, -keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, +volume = {420}, +year = {2020} } @book{Stix1992, - author = {Stix, T.H.}, - date-added = {2023-06-29 13:51:16 -0700}, - date-modified = {2023-06-29 13:51:16 -0700}, - isbn = {978-0-88318-859-0}, - lccn = {lc91033341}, - publisher = {American Inst. of Physics}, - title = {{Waves in Plasmas}}, - url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, - year = {1992}, - bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC} +author = {Stix, T. H.}, +bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +date-added = {2023-06-29 13:51:16 -0700}, +date-modified = {2023-06-29 13:51:16 -0700}, +isbn = {978-0-88318-859-0}, +lccn = {lc91033341}, +publisher = {American Inst. of Physics}, +title = {{Waves in Plasmas}}, +url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +year = {1992} } @article{Macchi2013, - title = {{Ion acceleration by superintense laser-plasma interaction}}, - author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, - journal = {Rev. Mod. Phys.}, - volume = {85}, - issue = {2}, - pages = {751--793}, - numpages = {0}, - year = {2013}, - month = {May}, - publisher = {American Physical Society}, - doi = {10.1103/RevModPhys.85.751}, - url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751} +author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, +doi = {10.1103/RevModPhys.85.751}, +issue = {2}, +journal = {Rev. Mod. Phys.}, +month = {May}, +numpages = {0}, +pages = {751--793}, +publisher = {American Physical Society}, +title = {{Ion acceleration by superintense laser-plasma interaction}}, +url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751}, +volume = {85}, +year = {2013} } @article{Wilks2001, - author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, - title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, - journal = {Physics of Plasmas}, - volume = {8}, - number = {2}, - pages = {542-549}, - year = {2001}, - month = {02}, - abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", - issn = {1070-664X}, - doi = {10.1063/1.1333697}, - url = {https://doi.org/10.1063/1.1333697}, - eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", +author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, +doi = {10.1063/1.1333697}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Feb}, +number = {2}, +pages = {542-549}, +title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, +url = {https://doi.org/10.1063/1.1333697}, +volume = {8}, +year = {2001} } @article{Bulanov2008, - title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, - author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, - journal = {Phys. Rev. E}, - volume = {78}, - issue = {2}, - pages = {026412}, - numpages = {6}, - year = {2008}, - month = {Aug}, - publisher = {American Physical Society}, - doi = {10.1103/PhysRevE.78.026412}, - url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412} +author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, +doi = {10.1103/PhysRevE.78.026412}, +issue = {2}, +journal = {Phys. Rev. E}, +month = {Aug}, +numpages = {6}, +pages = {026412}, +publisher = {American Physical Society}, +title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412}, +volume = {78}, +year = {2008} } @article{Dromey2004, - author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, - title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, - journal = {Review of Scientific Instruments}, - volume = {75}, - number = {3}, - pages = {645-649}, - year = {2004}, - month = {02}, - abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", - issn = {0034-6748}, - doi = {10.1063/1.1646737}, - url = {https://doi.org/10.1063/1.1646737}, - eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", +author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, +doi = {10.1063/1.1646737}, +eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +issn = {0034-6748}, +journal = {Review of Scientific Instruments}, +month = {Feb}, +number = {3}, +pages = {645-649}, +title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, +url = {https://doi.org/10.1063/1.1646737}, +volume = {75}, +year = {2004} } @article{Roedel2010, - title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, - volume = {103}, - ISSN = {1432-0649}, - url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, - DOI = {10.1007/s00340-010-4329-7}, - number = {2}, - journal = {Applied Physics B}, - publisher = {Springer Science and Business Media LLC}, - author = {R\"{o}del, C. and Heyer, M. and Behmke, M. and K\"{u}bel, M. and J\"{a}ckel, O. and Ziegler, W. and Ehrt, D. and Kaluza, M. C. and Paulus, G. G.}, - year = {2010}, - month = nov, - pages = {295–302} +author = {R\"{o}del,  C. and Heyer,  M. and Behmke,  M. and K\"{u}bel,  M. and J\"{a}ckel,  O. and Ziegler,  W. and Ehrt,  D. and Kaluza,  M. C. and Paulus,  G. G.}, +DOI = {10.1007/s00340-010-4329-7}, +ISSN = {1432-0649}, +journal = {Applied Physics B}, +month = {Nov}, +number = {2}, +pages = {295–302}, +publisher = {Springer Science and Business Media LLC}, +title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, +url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, +volume = {103}, +year = {2010} } @misc{SandbergPASC24, - author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, - title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, - booktitle = {Proc. of PASC24}, - venue = {Zuerich, Switzerland}, - address = {Zuerich, Switzerland}, - series = {PASC'24 - Platform for Advanced Scientific Computing}, - year = {2024}, - note = {submitted} +address = {Zuerich, Switzerland}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. of PASC24}, +note = {submitted}, +series = {PASC'24 - Platform for Advanced Scientific Computing}, +title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +venue = {Zuerich, Switzerland}, +year = {2024} } @inproceedings{SandbergIPAC23, - author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, - title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, - booktitle = {Proc. 14th International Particle Accelerator Conference}, - pages = {2885-2888}, - paper = {WEPA101}, - venue = {Venice, Italy}, - address = {Venice, Italy}, - series = {IPAC'23 - 14th International Particle Accelerator Conference}, - number = {14}, - publisher = {JACoW Publishing, Geneva, Switzerland}, - month = {05}, - year = {2023}, - issn = {2673-5490}, - isbn = {978-3-95450-231-8}, - doi = {10.18429/JACoW-IPAC2023-WEPA101}, - url = {https://indico.jacow.org/event/41/contributions/2276}, - language = {English} +address = {Venice, Italy}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. 14th International Particle Accelerator Conference}, +doi = {10.18429/JACoW-IPAC2023-WEPA101}, +isbn = {978-3-95450-231-8}, +issn = {2673-5490}, +language = {English}, +month = {May}, +number = {14}, +pages = {2885-2888}, +paper = {WEPA101}, +publisher = {JACoW Publishing, Geneva, Switzerland}, +series = {IPAC'23 - 14th International Particle Accelerator Conference}, +title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, +url = {https://indico.jacow.org/event/41/contributions/2276}, +venue = {Venice, Italy}, +year = {2023} +} + +@article{HigueraPOP2017, +author = {Higuera, A. V. and Cary, J. R.}, +doi = {10.1063/1.4979989}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4979989/15988441/052104\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {04}, +number = {5}, +pages = {052104}, +title = {{Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields}}, +url = {https://doi.org/10.1063/1.4979989}, +volume = {24}, +year = {2017} +} + +@article{ShuJCP1988, +author = {Chi-Wang Shu and Stanley Osher}, +doi = {https://doi.org/10.1016/0021-9991(88)90177-5}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +number = {2}, +pages = {439-471}, +title = {Efficient implementation of essentially non-oscillatory shock-capturing schemes}, +url = {https://www.sciencedirect.com/science/article/pii/0021999188901775}, +volume = {77}, +year = {1988} +} + +@Inbook{VanLeerBookChapter1997, +author = {Van Leer, Bram}, +bookTitle = {Upwind and High-Resolution Schemes}, +doi = {10.1007/978-3-642-60543-7_3}, +editor = {Hussaini, M. Yousuff and {van Leer}, Bram and {Van Rosendale}, John}, +isbn = {978-3-642-60543-7}, +pages = {33--52}, +publisher = {Springer Berlin Heidelberg}, +title = {On The Relation Between The Upwind-Differencing Schemes Of Godunov, Engquist---Osher and Roe}, +url = {https://doi.org/10.1007/978-3-642-60543-7_3}, +year = {1997} +} + +@article{Yakimenko2019, + title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, + author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, + journal = {Phys. Rev. Lett.}, + volume = {122}, + issue = {19}, + pages = {190404}, + numpages = {7}, + year = {2019}, + month = {May}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevLett.122.190404}, + url = {https://link.aps.org/doi/10.1103/PhysRevLett.122.190404} +} + +@article{Groenewald2023, +author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, +title = "{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}", +journal = {Physics of Plasmas}, +volume = {30}, +number = {12}, +pages = {122508}, +year = {2023}, +month = {12}, +issn = {1070-664X}, +doi = {10.1063/5.0178288}, +} + +@article{Perez2012, + author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, + title = "{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}", + journal = {Physics of Plasmas}, + volume = {19}, + number = {8}, + pages = {083104}, + year = {2012}, + month = {08}, + issn = {1070-664X}, + doi = {10.1063/1.4742167}, + url = {https://doi.org/10.1063/1.4742167}, + eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4742167/13891570/083104\_1\_online.pdf}, +} + +@article{Higginson2019, + doi = {10.1016/j.jcp.2019.03.020}, + url = {https://doi.org/10.1016/j.jcp.2019.03.020}, + year = {2019}, + month = jul, + publisher = {Elsevier {BV}}, + volume = {388}, + pages = {439--453}, + author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, + title = {A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}, + journal = {Journal of Computational Physics} } diff --git a/Docs/source/theory/PML.rst b/Docs/source/theory/PML.rst deleted file mode 100644 index d60be931c5f..00000000000 --- a/Docs/source/theory/PML.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. _theory-bc: - -Boundary conditions -=================== - -Open boundary condition for electromagnetic waves -------------------------------------------------- - -For the TE case, the original Berenger’s Perfectly Matched Layer (PML) writes - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{\partial H_{z}}{\partial y}\label{PML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{\partial H_{z}}{\partial x}\label{PML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{\partial E_{y}}{\partial x}\label{PML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{\partial E_{x}}{\partial y}\label{PML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{PML_def_5}\end{aligned} - -This can be generalized to - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z}\label{APML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z}\label{APML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y}\label{APML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x}\label{APML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{APML_def_5}\end{aligned} - -For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, -this system reduces to the Berenger PML medium, while adding the additional -constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` -leads to the system of Maxwell equations in vacuum. - -.. _theory-bc-propa-plane-wave: - -Propagation of a Plane Wave in an APML Medium -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) -and pulsation :math:`\omega` propagating in the APML medium with an -angle :math:`\varphi` relative to the x axis - -.. math:: - - \begin{aligned} - E_{x} = & -E_{0}\sin \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_1} \\ - E_{y} = & E_{0}\cos \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_2} \\ - H_{zx} = & H_{zx0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_AMPL_def_3} \\ - H_{zy} = & H_{zy0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_4}\end{aligned} - -where :math:`\alpha` and\ :math:`\beta` are two complex constants to -be determined. - -Introducing (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_def_2] <#Plane_wave_APML_def_2>`__), -(`[Plane_wave_AMPL_def_3] <#Plane_wave_AMPL_def_3>`__) and (`[Plane_wave_APML_def_4] <#Plane_wave_APML_def_4>`__) -into (`[APML_def_1] <#APML_def_1>`__), (`[APML_def_2] <#APML_def_2>`__), (`[APML_def_3] <#APML_def_3>`__) -and (`[APML_def_4] <#APML_def_4>`__) gives - -.. math:: - - \begin{aligned} - \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = & \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_1} \\ - \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = & \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_2} \\ - \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = & \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi \label{Plane_wave_APML_1_3} \\ - \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = & \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi \label{Plane_wave_APML_1_4}\end{aligned} - -Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using (`[Plane_wave_APML_1_1] <#Plane_wave_APML_1_1>`__) -and (`[Plane_wave_APML_1_2] <#Plane_wave_APML_1_2>`__), we get - -.. math:: - - \begin{aligned} - \beta = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}}\label{Plane_wave_APML_beta_of_g} \\ - \alpha = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}}\label{Plane_wave_APML_alpha_of_g}\end{aligned} - -Adding :math:`H_{zx0}` and :math:`H_{zy0}` from (`[Plane_wave_APML_1_3] <#Plane_wave_APML_1_3>`__) -and (`[Plane_wave_APML_1_4] <#Plane_wave_APML_1_4>`__) and substituting the expressions -for :math:`\alpha` and :math:`\beta` from (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__) -and (`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) yields - -.. math:: - - \begin{aligned} - \frac{1}{Z} = & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber \\ - + & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi\end{aligned} - -If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then - -.. math:: - - \begin{aligned} - Z = & \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}}\label{APML_impedance}\end{aligned} - -which is the impedance of vacuum. Hence, like the PML, given some -restrictions on the parameters, the APML does not generate any reflection -at any angle and any frequency. As for the PML, this property is not -retained after discretization, as shown subsequently in this paper. - -Calling :math:`\psi` any component of the field and :math:`\psi _{0}` -its magnitude, we get from (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__), -(`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) and (`[APML_impedance] <#APML_impedance>`__) that - -.. math:: - - \label{Plane_wave_absorption} - \psi =\psi _{0}e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x}e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y} - -We assume that we have an APML layer of thickness :math:`\delta` (measured -along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` -and :math:`c_{y}=c.` Using (`[Plane_wave_absorption] <#Plane_wave_absorption>`__), we determine -that the coefficient of reflection given by this layer is - -.. math:: - - \begin{aligned} - R_{APML}\left( \theta \right) = & e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta }\nonumber \\ - = & e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }\end{aligned} - -which happens to be the same as the PML theoretical coefficient of -reflection if we assume :math:`c_{x}=c`. Hence, it follows that for -the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` -seems to be of no interest. However, although this conclusion is true -at the infinitesimal limit, it does not hold for the discretized counterpart. - -Discretization -~~~~~~~~~~~~~~ - -.. math:: - - \begin{aligned} - \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = & \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} \\ - % - \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = & - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n}_{j+1/2,k+1/2,l}}{2} = & - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n}_{j+1/2,k+1/2,l}}{2} = & \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - c_x = & c e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta x}{1-e^{-\sigma_x\Delta t}} \\ - c_y = & c e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta y}{1-e^{-\sigma_y\Delta t}} \\ - c^*_x = & c e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta x}{1-e^{-\sigma^*_x\Delta t}} \\ - c^*_y = & c e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta y}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. _theory-bc-pec: - -Perfect Electrical Conductor ----------------------------- - -This boundary can be used to model a dielectric or metallic surface. -For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic -field are set to 0. In the guard-cell region, the tangential electric field is set equal and -opposite to the respective field component in the mirror location across the PEC -boundary, and the normal electric field is set equal to the field component in the -mirror location in the domain across the PEC boundary. Similarly, the tangential -(and normal) magnetic field components are set equal (and opposite) to the respective -magnetic field components in the mirror locations across the PEC boundary. - -The PEC boundary condition also impacts the deposition of charge and current density. -On the boundary the charge density and parallel current density is set to zero. If -a reflecting boundary condition is used for the particles, density overlapping -with the PEC will be reflected back into the domain (for both charge and current -density). If absorbing boundaries are used, an image charge (equal weight but -opposite charge) is considered in the mirror location accross the boundary, and -the density from that charge is also deposited in the simulation domain. The -figure below shows the effect of this. The left boundary is absorbing while -the right boundary is reflecting. - -.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png - :alt: PEC boundary deposition - :width: 80% diff --git a/Docs/source/theory/amr.rst b/Docs/source/theory/amr.rst index 730593beea4..d83b1d7db9d 100644 --- a/Docs/source/theory/amr.rst +++ b/Docs/source/theory/amr.rst @@ -3,158 +3,64 @@ Mesh refinement =============== -.. raw:: latex - - \centering +.. _fig_ESAMR: .. figure:: ICNSP_2011_Vay_fig1.png :alt: Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. - :name: fig:ESAMR - :width: 15cm + :width: 95% Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. -The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface (J. Vay et al. 2002; Colella and Norgaard 2010); b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface (Vay 2001). The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. +The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`; b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface :cite:p:`amr-Vayjcp01`. The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. -In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors (J. Vay et al. 2002; Colella and Norgaard 2010). As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. +In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. Electrostatic ------------- -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme (Birdsall and Langdon 1991). A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) (J. Vay et al. 2002; Colella and Norgaard 2010). - -A sketch of the implementation of mesh refinement in WarpX is given in Figure \ `[fig:ESAMR] <#fig:ESAMR>`__ (left). Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. +A cornerstone of the Particle-In-Cell method is that given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme :cite:p:`amr-Birdsalllangdon`. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. -.. raw:: latex +A sketch of the implementation of mesh refinement in WarpX is given in :numref:`fig_ESAMR`. Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. - \centering +.. _fig_ESselfforce: .. figure:: ICNSP_2011_Vay_fig2.png :alt: Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. - :name: fig:ESselfforce - :width: 15cm + :width: 95% Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. -The presence of the self-force is illustrated on a simple test case that was introduced in (J. Vay et al. 2002) and also used in (Colella and Norgaard 2010): a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` (“main grid”), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled “MC” in this paper) based on the algorithm given in (Mccorquodale et al. 2004), which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in Fig. `[fig:ESselfforce] <#fig:ESselfforce>`__. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - -.. raw:: latex +The presence of the self-force is illustrated on a simple test case that was introduced in :cite:t:`amr-Vaylpb2002` and also used in :cite:t:`amr-Colellajcp2010`: a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` ("main grid"), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled "MC" in this paper) based on the algorithm given in :cite:t:`amr-Mccorquodalejcp2004`, which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in :numref:`fig_ESselfforce`. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - \centering +.. _fig_ESselfforcemap: .. figure:: ICNSP_2011_Vay_fig3.png :alt: (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). - :name: fig:ESselfforcemap - :width: 15cm + :width: 95% (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). -The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in Fig. `[fig:ESselfforcemap] <#fig:ESselfforcemap>`__ for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. +The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in :numref:`fig_ESselfforcemap` for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. We notice that the magnitude of the spurious self-force depends strongly on the distance to the edge of the patch and to the nodes of the underlying coarse grid, but weakly on the order of deposition and size of the patch. -A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra “transition” cells are added around the “effective” refined area. -Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in Fig. \ `[fig:ESselfforce] <#fig:ESselfforce>`__, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. -We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface (Colella and Norgaard 2010). +A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra "transition" cells are added around the "effective" refined area. +Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in :numref:`fig_ESselfforce`, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. +We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface :cite:p:`amr-Colellajcp2010`. Electromagnetic --------------- -The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of (Vay 2001), refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. +The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of :cite:t:`amr-Vayjcp01`, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. -To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in Fig. \ `[fig:ESAMR] <#fig:ESAMR>`__-right and described in (Vay, Adam, and Heron 2004). Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user (Berenger 1996; J.-L. Vay 2002). The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. +To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in :numref:`fig_ESAMR` and described in :cite:t:`amr-Vaycpc04`. Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user :cite:p:`amr-Berengerjcp96,amr-Vayjcp02`. The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. An option has been implemented in which various grid levels are pushed with different time steps, given as a fixed fraction of the individual grid Courant conditions (assuming same cell aspect ratio for all grids and refinement by integer factors). In this case, the fields from the coarse levels, which are advanced less often, are interpolated in time. The substitution method has two potential drawbacks due to the inexact cancellation between the coarse and fine patches of : (i) the remnants of ghost fixed charges created by the particles entering and leaving the patches (this effect is due to the use of the electromagnetic solver and is different from the spurious self-force that was described for the electrostatic case); (ii) if using a Maxwell solver with a low-order stencil, the electromagnetic waves traveling on each patch at slightly different velocity due to numerical dispersion. The first issue results in an effective spurious multipole field whose magnitude decreases very rapidly with the distance to the patch boundary, similarly to the spurious self-force in the electrostatic case. Hence, adding a few extra transition cells surrounding the patches mitigates this effect very effectively. -The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in (Vay, Adam, and Heron 2004). -Both effects and their mitigation are described in more detail in (Vay, Adam, and Heron 2004). +The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in :cite:t:`amr-Vaycpc04`. +Both effects and their mitigation are described in more detail in :cite:t:`amr-Vaycpc04`. Caustics are supported anywhere on the grid with an accuracy that is set by the local resolution, and will be adequately resolved if the grid resolution supports the necessary modes from their sources to the points of wavefront crossing. The mesh refinement method that is implemented in WarpX has the potential to provide higher efficiency than the standard use of fixed gridding, by offering a path toward adaptive gridding following wavefronts. -.. raw:: html - -
- -.. raw:: html - -
- -Berenger, Jp. 1996. “Three-Dimensional Perfectly Matched Layer for the Absorption of Electromagnetic Waves.” *Journal of Computational Physics* 127 (2): 363–79. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Colella, Phillip, and Peter C Norgaard. 2010. “Controlling Self-Force Errors at Refinement Boundaries for Amr-Pic.” *Journal of Computational Physics* 229 (4): 947–57. https://doi.org/10.1016/J.Jcp.2009.07.004. - -.. raw:: html - -
- -.. raw:: html - -
- -Mccorquodale, P, P Colella, Dp Grote, and Jl Vay. 2004. “A Node-Centered Local Refinement Algorithm For Poisson’s Equation In Complex Geometries.” *Journal of Computational Physics* 201 (1): 34–60. https://doi.org/10.1016/J.Jcp.2004.04.022. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2001. “An Extended Fdtd Scheme for the Wave Equation: Application to Multiscale Electromagnetic Simulation.” *Journal of Computational Physics* 167 (1): 72–98. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “Asymmetric Perfectly Matched Layer for the Absorption of Waves.” *Journal of Computational Physics* 183 (2): 367–99. https://doi.org/10.1006/Jcph.2002.7175. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, P Colella, P Mccorquodale, B Van Straalen, A Friedman, and Dp Grote. 2002. “Mesh Refinement for Particle-in-Cell Plasma Simulations: Applications to and Benefits for Heavy Ion Fusion.” *Laser and Particle Beams* 20 (4): 569–75. https://doi.org/10.1017/S0263034602204139. - -.. raw:: html - -
- -.. raw:: html - -
+.. bibliography:: + :keyprefix: amr- diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index 4b80c3d2ccb..ea1f662bd30 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -5,16 +5,18 @@ Moving window and optimal Lorentz boosted frame The simulations of plasma accelerators from first principles are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated particle beam into a plasma structure that is orders of magnitude longer and wider than the accelerated beam. As is customary in the modeling of particle beam dynamics in standard particle accelerators, a moving window is commonly used to follow the driver, the wake and the accelerated beam. This results in huge savings, by avoiding the meshing of the entire plasma that is orders of magnitude longer than the other length scales of interest. +.. _fig_Boosted_frame: + .. figure:: Boosted_frame.png - :alt: [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + :alt: [fig:Boosted-frame] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. - [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. -Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude (Vay 2007; J -L. Vay et al. 2011). This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in Fig. `[fig:PIC] <#fig:PIC>`__, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). +Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). The modeling of a plasma acceleration stage in a boosted frame involves the fully electromagnetic modeling of a plasma propagating at near the speed of light, for which Numerical Cerenkov -(Boris and Lee 1973; Haber et al. 1973) is a potential issue, as explained in more details below. +:cite:p:`bf-Borisjcp73,bf-Habericnsp73` is a potential issue, as explained in more details below. In addition, for a frame of reference moving in the direction of the accelerated beam (or equivalently the wake of the laser), waves emitted by the plasma in the forward direction expand while the ones emitted in the backward direction contract, following the properties of the Lorentz transformation. @@ -25,16 +27,15 @@ Backscatter is weak in the short-pulse regime, and does not interact as strongly with the beam as do the forward propagating waves which stay in phase for a long period. It is thus often assumed that the backward propagating waves can be neglected in the modeling of plasma accelerator stages. The accuracy of this assumption has been demonstrated by -comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves -(Geddes et al. 2008; Geddes et al. 2009; Cowan et al. 2009). +comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves :cite:p:`bf-Geddesjp08,bf-Geddespac09,bf-Cowanaac08`. Theoretical speedup dependency with the frame boost --------------------------------------------------- -The derivation that is given here reproduces the one given in (J -L. Vay et al. 2011), where the obtainable speedup is derived as an extension of the formula that was derived earlier(Vay 2007), taking in addition into account the group velocity of the laser as it traverses the plasma. +The derivation that is given here reproduces the one given in :cite:t:`bf-Vaypop2011`, where the obtainable speedup is derived as an extension of the formula that was derived earlier :cite:p:`bf-Vayprl07`, taking in addition into account the group velocity of the laser as it traverses the plasma. Assuming that the simulation box is a fixed number of plasma periods long, which implies the use (which is standard) of a moving window following -the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by (Esarey, Schroeder, and Leemans 2009): +the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by :cite:p:`bf-Esareyrmp09`: .. math:: v_w/c=\beta_w=\left(1-\frac{\omega_p^2}{\omega^2}\right)^{1/2} @@ -58,76 +59,78 @@ In a frame moving at :math:`\beta c`, the quantities become .. math:: \begin{aligned} - \lambda_p^*&=&\lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] \\ - L^*&=&L/\gamma \\ - \lambda^*&=& \gamma\left(1+\beta\right) \lambda\\ - \beta_w^*&=&\left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) \\ - v_p^*&=&-\beta c \\ - T^*&=&\frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} \\ - R_t^*&=&\frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*}\end{aligned} + \lambda_p^* & = \lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] + \\ + L^* & = L/\gamma + \\ + \lambda^* & = \gamma\left(1+\beta\right) \lambda + \\ + \beta_w^* & = \left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) + \\ + v_p^* & = -\beta c + \\ + T^* & = \frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} + \\ + R_t^* & = \frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*} + \end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. The expected speedup from performing the simulation in a boosted frame is given by the ratio of :math:`R_{lab}` and :math:`R_t^*` .. math:: - S=\frac{R_{lab}}{R_t^*}=\frac{\left(1+\beta\right)\left(L+\eta \lambda_p\right)}{\left(1-\beta\beta_w\right)L+\eta \lambda_p} - \label{Eq_scaling1d0} + :label: Eq_scaling1d0 -We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier (Vay 2007) for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. +We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier :cite:p:`bf-Vayprl07` for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. -For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing (Schroeder et al. 2011), which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. -Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(\ `[Eq_scaling1d0] <#Eq_scaling1d0>`__) becomes +For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing :cite:p:`bf-Schroederprl2011`, which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. +Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(:eq:`Eq_scaling1d0`) becomes .. math:: - S=\left(1+\beta\right)^2\gamma^2\frac{\xi\gamma_w^2}{\xi\gamma_w^2+\left(1+\beta\right)\gamma^2\left(\xi\beta/2+2\eta\right)} - \label{Eq_scaling1d} + :label: Eq_scaling1d -For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) reduces to +For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(:eq:`Eq_scaling1d`) reduces to .. math:: - S_{\gamma<<\gamma_w}=\left(1+\beta\right)^2\gamma^2 - \label{Eq_scaling1d_simpl2} + :label: Eq_scaling1d_simpl2 -Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) becomes +Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(`Eq_scaling1d`) becomes .. math:: - S_{\gamma\rightarrow\infty}=\frac{4}{1+4\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_inf} + :label: Eq_scaling_gamma_inf -Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) gives +Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(:eq:`Eq_scaling1d`) gives .. math:: - S_{\gamma=\gamma_w}\approx\frac{2}{1+2\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_wake} + :label: Eq_scaling_gamma_wake -Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(\ `[Eq_scaling_gamma_inf] <#Eq_scaling_gamma_inf>`__). +Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(:eq:`Eq_scaling_gamma_inf`). -Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain (Vay 2007), and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. +Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain :cite:p:`bf-Vayprl07`, and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. .. _theory-boostedframe-galilean: Numerical Stability and alternate formulation in a Galilean frame ----------------------------------------------------------------- -The numerical Cherenkov instability (NCI) (Godfrey 1974) +The numerical Cherenkov instability (NCI) :cite:p:`bf-Godfreyjcp74` is the most serious numerical instability affecting multidimensional PIC simulations of relativistic particle beams and streaming plasmas -(Martins et al. 2010; Vay et al. 2010; J L Vay et al. 2011; Sironi and Spitkovsky 2011; Godfrey and Vay 2013; Xu et al. 2013). +:cite:p:`bf-Martinscpc10,bf-VayAAC2010,bf-Vayjcp2011,bf-Spitkovsky:Icnsp2011,bf-GodfreyJCP2013,bf-XuJCP2013`. It arises from coupling between possibly numerically distorted electromagnetic modes and spurious beam modes, the latter due to the mismatch between the Lagrangian -treatment of particles and the Eulerian treatment of fields (Godfrey 1975). +treatment of particles and the Eulerian treatment of fields :cite:p:`bf-Godfreyjcp75`. In recent papers the electromagnetic dispersion -relations for the numerical Cherenkov instability were derived and solved for both FDTD (Godfrey and Vay 2013; Brendan B. Godfrey and Vay 2014) -and PSATD (Brendan B. Godfrey, Vay, and Haber 2014a, 2014b) algorithms. +relations for the numerical Cherenkov instability were derived and solved for both FDTD :cite:p:`bf-GodfreyJCP2013,bf-GodfreyJCP2014_FDTD` +and PSATD :cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyIEEE2014` algorithms. -Several solutions have been proposed to mitigate the NCI (Brendan B Godfrey, Vay, and Haber 2014; Brendan B. Godfrey, Vay, and Haber 2014b, 2014a; Godfrey and Vay 2015; Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015). Although +Several solutions have been proposed to mitigate the NCI :cite:p:`bf-GodfreyJCP2014,bf-GodfreyIEEE2014,bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015,bf-YuCPC2015,bf-YuCPC2015-Circ`. Although these solutions efficiently reduce the numerical instability, they typically introduce either strong smoothing of the currents and fields, or arbitrary numerical corrections, which are @@ -137,47 +140,46 @@ it is sometimes unclear to what extent these added corrections could impact the physics at stake for a given resolution. For instance, NCI-specific corrections include periodically smoothing -the electromagnetic field components (Martins et al. 2010), -using a special time step (Vay et al. 2010; J L Vay et al. 2011) or -applying a wide-band smoothing of the current components (Vay et al. 2010; J L Vay et al. 2011; J. Vay et al. 2011). Another set of mitigation methods +the electromagnetic field components :cite:p:`bf-Martinscpc10`, +using a special time step :cite:p:`bf-VayAAC2010,bf-Vayjcp2011` or +applying a wide-band smoothing of the current components :cite:p:`bf-VayAAC2010,bf-Vayjcp2011,bf-VayPOPL2011`. Another set of mitigation methods involve scaling the deposited currents by a carefully-designed wavenumber-dependent factor -(Brendan B. Godfrey and Vay 2014; Brendan B. Godfrey, Vay, and Haber 2014b) or slightly modifying the +:cite:p:`bf-GodfreyJCP2014_FDTD,bf-GodfreyIEEE2014` or slightly modifying the ratio of electric and magnetic fields (:math:`E/B`) before gathering their value onto the macroparticles -(Brendan B. Godfrey, Vay, and Haber 2014a; Godfrey and Vay 2015). +:cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015`. Yet another set of NCI-specific corrections -(Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015) consists +:cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` consists in combining a small timestep :math:`\Delta t`, a sharp low-pass spatial filter, and a spectral or high-order scheme that is tuned so as to create a small, artificial “bump” in the dispersion relation -(Yu, Xu, Decyk, et al. 2015). While most mitigation methods have only been applied +:cite:p:`bf-YuCPC2015`. While most mitigation methods have only been applied to Cartesian geometry, this last -set of methods ((Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015)) +set of methods :cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` has the remarkable property that it can be applied -(Yu, Xu, Tableman, et al. 2015) to both Cartesian geometry and +:cite:p:`bf-YuCPC2015-Circ` to both Cartesian geometry and quasi-cylindrical geometry (i.e. cylindrical geometry with -azimuthal Fourier decomposition (Lifschitz et al. 2009; Davidson et al. 2015; R. Lehe et al. 2016)). However, +azimuthal Fourier decomposition :cite:p:`bf-LifschitzJCP2009,bf-DavidsonJCP2015,bf-Lehe2016`). However, the use of a small timestep proportionally slows down the progress of the simulation, and the artificial “bump” is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in (Kirchen et al. 2016; Lehe et al. 2016), which +A new scheme was recently proposed, in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity – with no arbitrary correction – by simply integrating the PIC equations in *Galilean coordinates* (also known as *comoving coordinates*). More precisely, in the new method, the Maxwell equations *in Galilean coordinates* are integrated analytically, using only natural hypotheses, within the PSATD -framework (Pseudo-Spectral-Analytical-Time-Domain (Haber et al. 1973; Vay, Haber, and Godfrey 2013)). +framework (Pseudo-Spectral-Analytical-Time-Domain :cite:p:`bf-Habericnsp73,bf-VayJCP2013`). The idea of the proposed scheme is to perform a Galilean change of coordinates, and to carry out the simulation in the new coordinates: .. math:: - - \label{eq:change-var} \boldsymbol{x}' = \boldsymbol{x} - \boldsymbol{v}_{gal}t + :label: change-var where :math:`\boldsymbol{x} = x\,\boldsymbol{u}_x + y\,\boldsymbol{u}_y + z\,\boldsymbol{u}_z` and :math:`\boldsymbol{x}' = x'\,\boldsymbol{u}_x + y'\,\boldsymbol{u}_y + z'\,\boldsymbol{u}_z` are the @@ -190,10 +192,10 @@ plasma, the plasma does not move with respect to the grid in the Galilean coordinates :math:`\boldsymbol{x}'` – or, equivalently, in the standard coordinates :math:`\boldsymbol{x}`, the grid moves along with the plasma. The heuristic intuition behind this scheme is that these coordinates should prevent the discrepancy between the Lagrangian and -Eulerian point of view, which gives rise to the NCI (Godfrey 1975). +Eulerian point of view, which gives rise to the NCI :cite:p:`bf-Godfreyjcp75`. An important remark is that the Galilean change of -coordinates (`[eq:change-var] <#eq:change-var>`__) is a simple translation. Thus, when used in +coordinates in Eq. (:eq:`change-var`) is a simple translation. Thus, when used in the context of Lorentz-boosted simulations, it does of course preserve the relativistic dilatation of space and time which gives rise to the characteristic computational speedup of the boosted-frame technique. @@ -206,378 +208,81 @@ translate the boundaries, in the Galilean scheme the gridpoints *themselves* are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current :math:`\boldsymbol{J}` within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme (Lehe et al. 2016). +window and in a Galilean PSATD scheme :cite:p:`bf-LehePRE2016`. In the Galilean coordinates :math:`\boldsymbol{x}'`, the equations of particle motion and the Maxwell equations take the form .. math:: + \frac{d\boldsymbol{x}'}{dt} = \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal} + :label: motion1 - \begin{aligned} - \frac{d\boldsymbol{x}'}{dt} &= \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal}\label{eq:motion1} \\ - \frac{d\boldsymbol{p}}{dt} &= q \left( \boldsymbol{E} + - \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) \label{eq:motion2}\\ - \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} &= -\boldsymbol{\nabla'}\times\boldsymbol{E} \label{eq:maxwell1}\\ - \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} &= \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} \label{eq:maxwell2}\end{aligned} +.. math:: + \frac{d\boldsymbol{p}}{dt} = q \left( \boldsymbol{E} + \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) + :label: motion2 + +.. math:: + \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} = -\boldsymbol{\nabla'}\times\boldsymbol{E} + :label: maxwell1 + +.. math:: + \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} = \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} + :label: maxwell2 where :math:`\boldsymbol{\nabla'}` denotes a spatial derivative with respect to the Galilean coordinates :math:`\boldsymbol{x}'`. Integrating these equations from :math:`t=n\Delta t` to :math:`t=(n+1)\Delta t` results in the following update equations (see -(Lehe et al. 2016) for the details of the derivation): +:cite:t:`bf-LehePRE2016` for the details of the derivation): .. math:: - \begin{aligned} - \mathbf{\tilde{B}}^{n+1} &= \theta^2 C \mathbf{\tilde{B}}^n - -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber \\ - & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times - \mathbf{\tilde{J}}^{n+1/2} \label{eq:disc-maxwell1}\\ - \mathbf{\tilde{E}}^{n+1} &= \theta^2 C \mathbf{\tilde{E}}^n - +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber \\ - & +\frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber \\ - & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} \label{eq:disc-maxwell2}\end{aligned} - -where we used the short-hand notations :math:`\mathbf{\tilde{E}}^n \equiv -% -\mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, :math:`\mathbf{\tilde{B}}^n \equiv -\mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: + \mathbf{\tilde{B}}^{n+1} & = \theta^2 C \mathbf{\tilde{B}}^n -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber + \\ + & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times \mathbf{\tilde{J}}^{n+1/2} + \end{aligned} + :label: disc-maxwell1 .. math:: - \begin{aligned} - &C = \cos(ck\Delta t) \quad S = \sin(ck\Delta t) \quad k - = |\boldsymbol{k}| \label{eq:def-C-S}\\& - \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck} \quad \theta = - e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \quad \theta^* = - e^{-i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \label{eq:def-nu-theta}\\& - \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i - \nu \theta S \right) \label{eq:def-chi1}\\& - \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} \quad - \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} \label{eq:def-chi23}\end{aligned} - -Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, -(`[eq:disc-maxwell1] <#eq:disc-maxwell1>`__) and (`[eq:disc-maxwell2] <#eq:disc-maxwell2>`__) reduce to the standard PSATD -equations (Haber et al. 1973), as expected. -As shown in (Kirchen et al. 2016; Lehe et al. 2016), -the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp, and R Lee. 1973. “Nonphysical Self Forces in Some Electromagnetic Plasma-Simulation Algorithms.” Note. *Journal of Computational Physics* 12 (1). 525 B St, Ste 1900, San Diego, Ca 92101-4495: Academic Press Inc Jnl-Comp Subscriptions: 131–36. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, B, D Bruhwiler, E Cormier-Michel, E Esarey, C G R Geddes, P Messmer, and K Paul. 2009. “Laser Wakefield Simulation Using A Speed-of-Light Frame Envelope Model.” In *Aip Conference Proceedings*, 1086:309–14. - -.. raw:: html - -
- -.. raw:: html - -
- -Davidson, A., A. Tableman, W. An, F.S. Tsung, W. Lu, J. Vieira, R.A. Fonseca, L.O. Silva, and W.B. Mori. 2015. “Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in :math:`\Phi` into OSIRIS.” *Journal of Computational Physics* 281: 1063–77. https://doi.org/10.1016/j.jcp.2014.10.064. - -.. raw:: html - -
- -.. raw:: html - -
- -Esarey, E, C B Schroeder, and W P Leemans. 2009. “Physics of Laser-Driven Plasma-Based Electron Accelerators.” *Rev. Mod. Phys.* 81 (3): 1229–85. https://doi.org/10.1103/Revmodphys.81.1229. - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes, C G R, D L Bruhwiler, J R Cary, W B Mori, J.-L. Vay, S F Martins, T Katsouleas, et al. 2008. “Computational Studies and Optimization of Wakefield Accelerators.” In *Journal of Physics: Conference Series*, 125:012002 (11 Pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes et al., C G R. 2009. “Scaled Simulation Design of High Quality Laser Wakefield Accelerator Stages.” In *Proc. Particle Accelerator Conference*. Vancouver, Canada. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Bb. 1974. “Numerical Cherenkov Instabilities in Electromagnetic Particle Codes.” *Journal of Computational Physics* 15 (4): 504–21. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 1975. “Canonical Momenta and Numerical Instabilities in Particle Codes.” *Journal of Computational Physics* 19 (1): 58–76. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, and Jean-Luc Vay. 2013. “Numerical stability of relativistic beam multidimensional {PIC} simulations employing the Esirkepov algorithm.” *Journal of Computational Physics* 248 (0): 33–46. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.04.006. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., and Jean Luc Vay. 2014. “Suppressing the numerical Cherenkov instability in FDTD PIC codes.” *Journal of Computational Physics* 267: 1–6. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2015. “Improved numerical Cherenkov instability suppression in the generalized PSTD PIC algorithm.” *Computer Physics Communications* 196. Elsevier: 221–25. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., Jean Luc Vay, and Irving Haber. 2014a. “Numerical stability analysis of the pseudo-spectral analytical time-domain PIC algorithm.” *Journal of Computational Physics* 258: 689–704. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2014b. “Numerical stability improvements for the pseudospectral EM PIC algorithm.” *IEEE Transactions on Plasma Science* 42 (5). Institute of Electrical; Electronics Engineers Inc.: 1339–44. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, Jean-Luc Vay, and Irving Haber. 2014. “Numerical stability analysis of the pseudo-spectral analytical time-domain {PIC} algorithm.” *Journal of Computational Physics* 258 (0): 689–704. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.10.053. - -.. raw:: html + \mathbf{\tilde{E}}^{n+1} & = \theta^2 C \mathbf{\tilde{E}}^n +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber + \\ + & + \frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber + \\ + & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} + \end{aligned} + :label: disc-maxwell2 + +where we used the short-hand notations +:math:`\mathbf{\tilde{E}}^n \equiv \mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, +:math:`\mathbf{\tilde{B}}^n \equiv \mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Kirchen, M., R. Lehe, B. B. Godfrey, I. Dornmair, S. Jalas, K. Peters, J.-L. Vay, and A. R. Maier. 2016. “Stable discrete representation of relativistically drifting plasmas.” *arXiv:1608.00215*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, Rémi, Manuel Kirchen, Igor A. Andriyash, Brendan B. Godfrey, and Jean-Luc Vay. 2016. “A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.” *Computer Physics Communications* 203: 66–82. https://doi.org/10.1016/j.cpc.2016.02.007. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R., M. Kirchen, B. B. Godfrey, A. R. Maier, and J.-L. Vay. 2016. “Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates.” *arXiv:1608.00227*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lifschitz, A F, X Davoine, E Lefebvre, J Faure, C Rechatin, and V Malka. 2009. “Particle-in-Cell modelling of laser-plasma interaction using Fourier decomposition.” *Journal of Computational Physics* 228 (5): 1803–14. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2008.11.017. - -.. raw:: html - -
- -.. raw:: html - -
- -Martins, Samuel F, Ricardo A Fonseca, Luis O Silva, Wei Lu, and Warren B Mori. 2010. “Numerical Simulations of Laser Wakefield Accelerators in Optimal Lorentz Frames.” *Computer Physics Communications* 181 (5): 869–75. https://doi.org/10.1016/J.Cpc.2009.12.023. - -.. raw:: html - -
- -.. raw:: html - -
- -Schroeder, C B, C Benedetti, E Esarey, and W P Leemans. 2011. “Nonlinear Pulse Propagation and Phase Velocity of Laser-Driven Plasma Waves.” *Physical Review Letters* 106 (13): 135002. https://doi.org/10.1103/Physrevlett.106.135002. - -.. raw:: html - -
- -.. raw:: html - -
- -Sironi, L, and A Spitkovsky. 2011. “No Title.” - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -. L, C G R Geddes, C Benedetti, D L Bruhwiler, E Cormier-Michel, B M Cowan, J R Cary, and D P Grote. 2010. “Modeling Laser Wakefield Accelerators in A Lorentz Boosted Frame.” *Aip Conference Proceedings* 1299: 244–49. https://doi.org/10.1063/1.3520322. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Effects of Hyperbolic Rotation in Minkowski Space on the Modeling of Plasma Accelerators in A Lorentz Boosted Frame.” *Physics of Plasmas* 18 (3): 30701. https://doi.org/10.1063/1.3559483. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html - -
- -Xu, Xinlu, Peicheng Yu, Samual F Martins, Frank S Tsung, Viktor K Decyk, Jorge Vieira, Ricardo A Fonseca, Wei Lu, Luis O Silva, and Warren B Mori. 2013. “Numerical instability due to relativistic plasma drift in EM-PIC simulations.” *Computer Physics Communications* 184 (11): 2503–14. https://doi.org/http://dx.doi.org/10.1016/j.cpc.2013.07.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Yu, Peicheng, Xinlu Xu, Viktor K. Decyk, Frederico Fiuza, Jorge Vieira, Frank S. Tsung, Ricardo A. Fonseca, Wei Lu, Luis O. Silva, and Warren B. Mori. 2015. “Elimination of the numerical Cerenkov instability for spectral EM-PIC codes.” *Computer Physics Communications* 192 (July). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 32–47. https://doi.org/10.1016/j.cpc.2015.02.018. - -.. raw:: html - -
- -.. raw:: html +.. math:: + C = \cos(ck\Delta t), \quad S = \sin(ck\Delta t), \quad k = |\boldsymbol{k}|, + :label: def-C-S -
+.. math:: + \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck}, \quad \theta = e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2}, + :label: def-nu-theta -Yu, Peicheng, Xinlu Xu, Adam Tableman, Viktor K. Decyk, Frank S. Tsung, Frederico Fiuza, Asher Davidson, et al. 2015. “Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit.” *Computer Physics Communications* 197 (December). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 144–52. https://doi.org/10.1016/j.cpc.2015.08.026. +.. math:: + \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i \nu \theta S \right), + :label: def-chi1 -.. raw:: html +.. math:: + \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} + :label: def-chi2 -
+.. math:: + \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} + :label: def-chi3 -.. raw:: html +Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, +Eqs. (:eq:`disc-maxwell1`) and (:eq:`disc-maxwell2`) reduce to the standard PSATD +equations :cite:p:`bf-Habericnsp73`, as expected. +As shown in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, +the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. -
+.. bibliography:: + :keyprefix: bf- diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst new file mode 100644 index 00000000000..395b072ccbe --- /dev/null +++ b/Docs/source/theory/boundary_conditions.rst @@ -0,0 +1,303 @@ +.. _theory-bc: + +Boundary conditions +=================== + +.. _theory-bc-PML: + +Perfectly Matched Layer: open boundary condition for electromagnetic waves +-------------------------------------------------------------------------- + +For the transverse electric (TE) case, the original Berenger’s Perfectly Matched Layer (PML) paper :cite:p:`bc-Berengerjcp94` writes + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{\partial H_{z}}{\partial y} + :label: PML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{\partial H_{z}}{\partial x} + :label: PML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{\partial E_{y}}{\partial x} + :label: PML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{\partial E_{x}}{\partial y} + :label: PML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: PML_def_5 + +This can be generalized to + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z} + :label: APML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z} + :label: APML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y} + :label: APML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x} + :label: APML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: APML_def_5 + +For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, +this system reduces to the Berenger PML medium, while adding the additional +constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` +leads to the system of Maxwell equations in vacuum. + +.. _theory-bc-propa-plane-wave: + +Propagation of a Plane Wave in an APML Medium +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) +and pulsation :math:`\omega` propagating in the APML medium with an +angle :math:`\varphi` relative to the x axis + +.. math:: + E_{x} = -E_{0}\sin \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_1 + +.. math:: + E_{y} = E_{0}\cos \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_2 + +.. math:: + H_{zx} = H_{zx0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_AMPL_def_3 + +.. math:: + H_{zy} = H_{zy0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_4 + +where :math:`\alpha` and :math:`\beta` are two complex constants to +be determined. + +Introducing Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_def_2`), +(:eq:`Plane_wave_AMPL_def_3`) and (:eq:`Plane_wave_APML_def_4`) +into Eqs. (:eq:`APML_def_1`), (:eq:`APML_def_2`), (:eq:`APML_def_3`) +and (:eq:`APML_def_4`) gives + +.. math:: + \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_1 + +.. math:: + \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_2 + +.. math:: + \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi + :label: Plane_wave_APML_1_3 + +.. math:: + \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi + :label: Plane_wave_APML_1_4 + +Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using Eqs. (:eq:`Plane_wave_APML_1_1`) +and (:eq:`Plane_wave_APML_1_2`), we get + +.. math:: + \beta = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}} + :label: Plane_wave_APML_beta_of_g + +.. math:: + \alpha = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}} + :label: Plane_wave_APML_alpha_of_g + +Adding :math:`H_{zx0}` and :math:`H_{zy0}` from Eqs. (:eq:`Plane_wave_APML_1_3`) +and (:eq:`Plane_wave_APML_1_4`) and substituting the expressions +for :math:`\alpha` and :math:`\beta` from Eqs. (:eq:`Plane_wave_APML_beta_of_g`) +and (:eq:`Plane_wave_APML_alpha_of_g`) yields + +.. math:: + + \begin{aligned} + \frac{1}{Z} & = \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber + \\ + & + \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi + \end{aligned} + +If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then + +.. math:: + Z = \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}} + :label: APML_impedance + +which is the impedance of vacuum. Hence, like the PML, given some +restrictions on the parameters, the APML does not generate any reflection +at any angle and any frequency. As for the PML, this property is not +retained after discretization, as shown subsequently. + +Calling :math:`\psi` any component of the field and :math:`\psi _{0}` +its magnitude, we get from Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_beta_of_g`), +(:eq:`Plane_wave_APML_alpha_of_g`) and (:eq:`APML_impedance`) that + +.. math:: + \psi =\psi _{0} \: e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x} e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y}. + :label: Plane_wave_absorption + +We assume that we have an APML layer of thickness :math:`\delta` (measured +along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` +and :math:`c_{y}=c.` Using (:eq:`Plane_wave_absorption`), we determine +that the coefficient of reflection given by this layer is + +.. math:: + + \begin{aligned} + R_{\mathrm{APML}}\left( \theta \right) & = e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta },\nonumber + \\ + & = e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }, + \end{aligned} + +which happens to be the same as the PML theoretical coefficient of +reflection if we assume :math:`c_{x}=c`. Hence, it follows that for +the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` +seems to be of no interest. However, although this conclusion is true +at the infinitesimal limit, it does not hold for the discretized counterpart. + +Discretization +~~~~~~~~~~~~~~ + +In the following we set :math:`\varepsilon_0 = \mu_0 = 1`. We discretize Eqs. (:eq:`PML_def_1`), (:eq:`PML_def_2`), (:eq:`PML_def_3`), and (:eq:`PML_def_4`) to obtain + +.. math:: + \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} + +.. math:: + \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} + +and this can be solved to obtain the following leapfrog integration equations + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we account for higher order :math:`\Delta t` terms, a better approximation is given by + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +More generally, this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we set + +.. math:: + + \begin{aligned} + c_x & = c \: e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta t}{1-e^{-\sigma_x\Delta t}} \\ + c_y & = c \: e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta t}{1-e^{-\sigma_y\Delta t}} \\ + c^*_x & = c \: e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta t}{1-e^{-\sigma^*_x\Delta t}} \\ + c^*_y & = c \: e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta t}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} + +then this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] + \end{aligned} + +When the generalized conductivities are zero, the update equations are + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +as expected. + +.. _theory-bc-pec: + +Perfect Electrical Conductor +---------------------------- + +This boundary can be used to model a dielectric or metallic surface. +For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic +field are set to 0. In the guard-cell region, the tangential electric field is set equal and +opposite to the respective field component in the mirror location across the PEC +boundary, and the normal electric field is set equal to the field component in the +mirror location in the domain across the PEC boundary. Similarly, the tangential +(and normal) magnetic field components are set equal (and opposite) to the respective +magnetic field components in the mirror locations across the PEC boundary. + +The PEC boundary condition also impacts the deposition of charge and current density. +On the boundary the charge density and parallel current density is set to zero. If +a reflecting boundary condition is used for the particles, density overlapping +with the PEC will be reflected back into the domain (for both charge and current +density). If absorbing boundaries are used, an image charge (equal weight but +opposite charge) is considered in the mirror location accross the boundary, and +the density from that charge is also deposited in the simulation domain. :numref:`fig_PEC_boundary_deposition` +shows the effect of this. The left boundary is absorbing while +the right boundary is reflecting. + +.. _fig_PEC_boundary_deposition: + +.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png + :alt: Plot of PEC boundary current deposition showing current vs position along the ``x``-axis. + :width: 100% + + PEC boundary current deposition along the ``x``-axis. The left boundary is absorbing while the right boundary is reflecting. + +.. bibliography:: + :keyprefix: bc- diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst index 7e69516270b..813a568c540 100644 --- a/Docs/source/theory/cold_fluid_model.rst +++ b/Docs/source/theory/cold_fluid_model.rst @@ -42,15 +42,19 @@ where the particle quantities are calculated by the PIC algorithm. Implementation details ---------------------- +.. _fig_fluid_loop: + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 + :alt: Figure showing fluid Loop embedded within the overall PIC loop. + + Fluid Loop embedded within the overall PIC loop. + The fluid timeloop is embedded inside the standard PIC timeloop and consists of the following steps: 1. Higuera and Cary push of the momentum 2. Non-inertial (momentum source) terms (only in cylindrical geometry) 3. boundary conditions and MPI Communications 4. MUSCL -scheme for advection terms 5. Current and Charge Deposition. The figure here gives +scheme for advection terms 5. Current and Charge Deposition. :numref:`fig_fluid_loop` gives a visual representation of these steps, and we describe each of these in more detail. -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 - :alt: Fluid Loop embedded within the overall PIC loop. - Step 0: **Preparation** Before the fluid loop begins, it is assumed that the program is in the state where fields :math:`\mathbf{E}` and :math:`\mathbf{B}` are available integer timestep. The @@ -59,14 +63,14 @@ Step 0: **Preparation** on a nodal grid and at half-integer timestep. Step 1: **Higuera and Cary Push** - The time staggering of the fields is used by the momentum source term, which is solved with a the - Higeura and Cary push (Higuera et al, 2017). We do not adopt spatial + The time staggering of the fields is used by the momentum source term, which is solved with a + Higuera and Cary push :cite:p:`cfm-HigueraPOP2017`. We do not adopt spatial grid staggering, all discretized fluid quantities exist on the nodal grid. External fields can be included at this step. Step 2: **Non-inertial Terms** In RZ, the divergence of the flux terms has additional non-zero elements outside of the - derivatives. These terms are Strang split and are time integrated via equation 2.18 from (Osher et al, 1988), + derivatives. These terms are Strang split and are time integrated via equation 2.18 from :cite:t:`cfm-ShuJCP1988`, which is the SSP-RK3 integrator. Step 3: **Boundary Conditions and Communications** @@ -79,7 +83,7 @@ Step 4: **Advective Push** limiting is used. We further simplify the conservative equations in terms of primitive variables, :math:`\{ N, U_x, U_y, U_z \}`. Which we found to be more stable than conservative variables for the MUSCL reconstruction. Details of - the scheme can be found here (Van Leer et al, 1984). + the scheme can be found in :cite:t:`cfm-VanLeerBookChapter1997`. Step 5: **Current and Charge Deposition** Once this series of steps is complete and the fluids have been evolved by an entire @@ -104,33 +108,5 @@ Step 5: **Current and Charge Deposition** If using the fluid model with the Kinetic-Fluid Hybrid model or the electrostatic solver, there is a known issue that the fluids deposit at a half-timestep offset in the charge-density. -.. raw:: html - -
- -.. raw:: html - -
- -Higuera, Adam V., and John R. Cary. "Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields." Physics of Plasmas 24.5 (2017). - -.. raw:: html - -
- -.. raw:: html - -
- -Osher, Stanley, and Chi-Wang Shu. "Efficient implementation of essentially non-oscillatory shock-capturing schemes." J. Comput. Phys 77.2 (1988): 439-471. - - -.. raw:: html - -
- -.. raw:: html - -
- -Van Leer, Bram. "On the relation between the upwind-differencing schemes of Godunov, Engquist–Osher and Roe." SIAM Journal on Scientific and statistical Computing 5.1 (1984): 1-20. +.. bibliography:: + :keyprefix: cfm- diff --git a/Docs/source/theory/collisions.rst b/Docs/source/theory/collisions.rst index 3f24cd8f331..52e36521125 100644 --- a/Docs/source/theory/collisions.rst +++ b/Docs/source/theory/collisions.rst @@ -3,18 +3,36 @@ Collisions ========== -Monte Carlo Collisions ----------------------- - -The Monte Carlo collisions (MCC) module can be found in *Source/Particles/Collisions/BackgroundMCC/*. -Several types of collisions between simulation particles and a neutral background gas are supported including elastic scattering, back scattering, charge exchange, excitation collisions and impact ionization. -An instance of the class :cpp:class:`MCCProcess` is created for each type of collision included in a simulation. This class saves information about the type of collision and the collision cross-section as a function of energy. - -The so-called null collision strategy is used in order to minimize the computational burden of the MCC module. -This strategy is standard in PIC-MCC and a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. -In short the maximum collision probability is found over a sensible range of energies and is used to pre-select the appropriate number of macroparticles for collision consideration. Only these pre-selected particles are then individually considered for a collision based on their energy and the cross-sections of all the different collisional processes included. - -The MCC implementation assumes that the background neutral particles are **thermal**, and are moving at non-relativistic velocities in the lab frame. For each simulation particle considered for a collision, a velocity vector for a neutral particle is randomly chosen. The particle velocity is then boosted to the stationary frame of the neutral through a Galilean transformation. The energy of the collision is calculated using the particle utility function, ``ParticleUtils::getCollisionEnergy()``, as +WarpX includes several different models to capture collisional processes +including collisions between kinetic particles (Coulomb collisions, DSMC, +nuclear fusion) as well as collisions between kinetic particles and a fixed +(i.e. non-evolving) background species (MCC, background stopping). + +.. _theory-collisions-mcc: + +Background Monte Carlo Collisions (MCC) +--------------------------------------- + +Several types of collisions between simulation particles and a neutral +background gas are supported including elastic scattering, back scattering, +charge exchange, excitation collisions and impact ionization. + +The so-called null collision strategy is used in order to minimize the +computational burden of the MCC module. This strategy is standard in PIC-MCC and +a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. +In short the maximum collision probability is found over a sensible range of +energies and is used to pre-select the appropriate number of macroparticles for +collision consideration. Only these pre-selected particles are then individually +considered for a collision based on their energy and the cross-sections of all +the different collisional processes included. + +The MCC implementation assumes that the background neutral particles are **thermal**, +and are moving at non-relativistic velocities in the lab frame. For each +simulation particle considered for a collision, a velocity vector for a neutral +particle is randomly chosen given the user specified neutral temperature. The +particle velocity is then boosted to the stationary frame of the neutral through +a Galilean transformation. The energy of the collision is calculated using the +particle utility function, ``ParticleUtils::getCollisionEnergy()``, as .. math:: @@ -23,19 +41,55 @@ The MCC implementation assumes that the background neutral particles are **therm &= \frac{2Mmu^2}{M + m + \sqrt{M^2+m^2+2\gamma mM}}\frac{1}{\gamma + 1} \end{aligned} -where :math:`u` is the speed of the particle as tracked in WarpX (i.e. :math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and :math:`M` are the rest masses of the simulation and background species, respectively. The Lorentz factor is defined in the usual way, :math:`\gamma = \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above expression clearly reduces to the classical equation :math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections for all scattering processes are evaluated at the energy as calculated above. +where :math:`u` is the speed of the particle as tracked in WarpX (i.e. +:math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and +:math:`M` are the rest masses of the simulation and background species, +respectively. The Lorentz factor is defined in the usual way, +:math:`\gamma \def \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above +expression reduces to the classical equation +:math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections +for all scattering processes are evaluated at the energy as calculated above. Once a particle is selected for a specific collision process, that process determines how the particle is scattered as outlined below. +.. _theory-collisions-dsmc: + +Direct Simulation Monte Carlo (DSMC) +------------------------------------ + +The algorithm by which binary collisions are treated is outlined below. The +description assumes collisions between different species. + +1. Particles from both species are sorted by grid-cells. +2. The order of the particles in each cell is shuffled. +3. Within each cell, particles are paired to form collision partners. Particles + of the species with fewer members in a given cell is split in half so that + each particle has exactly one partner of the other species. +4. Each collision pair is considered for a collision using the same logic as in + the MCC description above. +5. Particles that are chosen for collision are scattered according to the + selected collision process. + +Scattering processes +-------------------- + Charge exchange ^^^^^^^^^^^^^^^ -This is the simplest scattering process. Under charge exchange the simulation particle's velocity is simply replaced with the sampled velocity of the neutral particle. Charge exchange usually has a cooling effect on the ions in the simulation by exchanging energetic ions for lower energy neutrals. +This process can occur when an ion and neutral (of the same species) collide +and results in the exchange of an electron. The ion velocity is simply replaced +with the neutral velocity and vice-versa. Elastic scattering ^^^^^^^^^^^^^^^^^^ -This scattering process as well as the ones below that relate to it, are all performed in the center-of-momentum (COM) frame. Designating the COM velocity of the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, the transformation from lab frame to COM frame is done with a general Lorentz boost (see function ``ParticleUtils::doLorentzTransform()``): +The ``elastic`` option uses isotropic scattering, i.e., with a differential +cross section that is independent of angle. +This scattering process as well as the ones below that relate to it, are all +performed in the center-of-momentum (COM) frame. Designating the COM velocity of +the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, +the transformation from lab frame to COM frame is done with a general Lorentz +boost (see function ``ParticleUtils::doLorentzTransform()``): .. math:: \begin{bmatrix} @@ -74,6 +128,12 @@ Excitation The process is also the same as for elastic scattering except the excitation energy cost is subtracted from the particle energy. This is done by reducing the velocity before a scattering angle is chosen. +Benchmarks +---------- + +See the :ref:`MCC example ` for a benchmark of the MCC +implementation against literature results. + Particle cooling due to elastic collisions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Docs/source/theory/input_output.rst b/Docs/source/theory/input_output.rst index 75427ad9c6d..21a5f5c8d2c 100644 --- a/Docs/source/theory/input_output.rst +++ b/Docs/source/theory/input_output.rst @@ -3,7 +3,7 @@ Inputs and Outputs ================== -Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings (Vay 2008; Cowan et al. 2013). +Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings :cite:p:`io-Vaypop2008, io-CowanPRSTAB13`. Outputs include dumps of particle and field quantities at regular intervals, histories of particle distributions moments, spectra, etc, and plots of the various quantities. In parallel simulations, the diagnostic subroutines need to handle additional complexity from the domain decomposition, as well as large amount of data that may necessitate data reduction in some form before saving to disk. @@ -12,16 +12,17 @@ Simulations in a Lorentz boosted frame require additional considerations, as des Inputs and outputs in a boosted frame simulation ------------------------------------------------ -.. _Fig_inputoutput: +.. _fig_inputoutput: + .. figure:: Input_output.png :alt: (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. - :width: 120mm + :width: 100% (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. The input and output data are often known from, or compared to, experimental data. Thus, calculating in a frame other than the laboratory entails transformations of the data between the calculation frame and the laboratory -frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp (Grote et al. 2005) to handle the input and output of data between the frame of calculation and the laboratory frame (Vay et al. 2011). Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. +frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp :cite:p:`io-Warp` to handle the input and output of data between the frame of calculation and the laboratory frame :cite:p:`io-Vaypop2011`. Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. Input in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,17 +30,17 @@ Input in a boosted frame simulation Particles - ^^^^^^^^^^^^ -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability (Vay 2007). In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability :cite:p:`io-Vayprl07`. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): #. project positions at :math:`z^*=0` assuming ballistic propagation .. math:: \begin{aligned} - t^* &=& \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ - x^* &=& x-v_x t^* \label{Eq:x*}\\ - y^* &=& y-v_y t^* \label{Eq:y*}\\ - z^* &=& 0 \label{Eq:z*}\end{aligned} + t^* &= \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ + x^* &= x-v_x t^* \label{Eq:x*}\\ + y^* &= y-v_y t^* \label{Eq:y*}\\ + z^* &= 0 \label{Eq:z*}\end{aligned} the velocity components being left unchanged, @@ -48,13 +49,13 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - t'^* &=& -\gamma t^* \label{Eq:tp*}\\ - x'^* &=& x^* \label{Eq:xp*}\\ - y'^* &=& y^* \label{Eq:yp*}\\ - z'^* &=& \gamma\beta c t^* \label{Eq:zp*}\\ - v'^*_x&=&\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ - v'^*_y&=&\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ - v'^*_z&=&\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} + t'^* &= -\gamma t^* \label{Eq:tp*}\\ + x'^* &= x^* \label{Eq:xp*}\\ + y'^* &= y^* \label{Eq:yp*}\\ + z'^* &= \gamma\beta c t^* \label{Eq:zp*}\\ + v'^*_x&=\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ + v'^*_y&=\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ + v'^*_z&=\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. With the knowledge of the time at which each beam macroparticle crosses the plane into consideration, one can inject each beam macroparticle in the simulation at the appropriate location and time. @@ -63,11 +64,11 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - z' &=& z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} + z' &= z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} - This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other (Vay 2008)) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. + This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other :cite:p:`io-Vaypop2008`) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. -Figure :numref:`Fig_inputoutput` (top) shows a snapshot of a beam that has passed partly through the injection plane. As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. +A snapshot of a beam that has passed partly through the injection plane in shown in :numref:`fig_inputoutput` (top). As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. Laser - ^^^^^^^^ @@ -89,93 +90,28 @@ If, for convenience, the injection plane is moving at constant velocity :math:`\ .. math:: \begin{aligned} - E_\perp\left(x,y,t\right)&=&\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\nonumber \\ - &\times& \sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right].\end{aligned} + E_\perp\left(x,y,t\right)&=\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right] + \end{aligned} The injection of a laser of frequency :math:`\omega` is considered for a simulation using a boosted frame moving at :math:`\beta c` with respect to the laboratory. Assuming that the laser is injected at a plane that is fixed in the laboratory, and thus moving at :math:`\beta_s=-\beta` in the boosted frame, the injection in the boosted frame is given by .. math:: \begin{aligned} - E_\perp\left(x',y',t'\right)&=&\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\nonumber \\ - &\times&\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right]\\ - &=&\left(E_0/\gamma\right) f\left(x',y',t'\right) \nonumber\\ - &\times&\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right]\end{aligned} + E_\perp\left(x',y',t'\right)&=\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right] + \\ + &=\left(E_0/\gamma\right) f\left(x',y',t'\right)\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right] + \end{aligned} since :math:`E'_0/E_0=\omega'/\omega=1/\left(1+\beta\right)\gamma`. -The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme (Esirkepov 2001). +The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme :cite:p:`io-Esirkepovcpc01`. Output in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some quantities, e.g. charge or dimensions perpendicular to the boost velocity, are Lorentz invariant. -Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on Fig. :numref:`Fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Grote, D P, A Friedman, J.-L. Vay, and I Haber. 2005. “The Warp Code: Modeling High Intensity Ion Beams.” In *Aip Conference Proceedings*, 55–58. 749. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html +Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on :numref:`fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. -
+.. bibliography:: + :keyprefix: io- diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 69797352bee..854c4d5ada1 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -18,8 +18,8 @@ has to resolve the electron Debye length and CFL-condition based on the speed of light. Many authors have described variations of the kinetic ion & fluid electron model, -generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`c-winske2022hybrid`. +generally referred to as particle-fluid hybrid or just hybrid-PIC models. The +implementation in WarpX is described in detail in :cite:t:`kfhm-Groenewald2023`. This description follows mostly from that reference. Model @@ -29,7 +29,7 @@ The basic justification for the hybrid model is that the system to which it is applied is dominated by ion kinetics, with ions moving much slower than electrons and photons. In this scenario two critical approximations can be made, namely, neutrality (:math:`n_e=n_i`) and the Maxwell-Ampere equation can be simplified by -neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, +neglecting the displacement current term :cite:p:`kfhm-Nielson1976`, giving, .. math:: @@ -168,4 +168,4 @@ The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. .. bibliography:: - :keyprefix: c- + :keyprefix: kfhm- diff --git a/Docs/source/theory/pic.rst b/Docs/source/theory/pic.rst new file mode 100644 index 00000000000..820cdba50e6 --- /dev/null +++ b/Docs/source/theory/pic.rst @@ -0,0 +1,695 @@ +.. _theory-pic: + +Particle-in-Cell Method +======================= + +.. _fig-pic: + +.. figure:: PIC.png + :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + + The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + +In the *electromagnetic particle-in-cell method* :cite:p:`pt-Birdsalllangdon,pt-HockneyEastwoodBook`, +the electromagnetic fields are solved on a grid, usually using Maxwell’s +equations + +.. math:: + \frac{\mathbf{\partial B}}{\partial t} = -\nabla\times\mathbf{E} + :label: Faraday-1 + +.. math:: + \frac{\mathbf{\partial E}}{\partial t} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-1 + +.. math:: + \nabla\cdot\mathbf{E} = \rho + :label: Gauss-1 + +.. math:: + \nabla\cdot\mathbf{B} = 0 + :label: divb-1 + +given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and +:math:`\mathbf{B}` are the electric and magnetic field components, and +:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The +charged particles are advanced in time using the Newton-Lorentz equations +of motion + +.. math:: + \frac{d\mathbf{x}}{dt} = \mathbf{v}, + :label: Lorentz_x-1 + +.. math:: + \frac{d\left(\gamma\mathbf{v}\right)}{dt} = \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right), + :label: Lorentz_v-1 + +where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` +are respectively the mass, charge, position, velocity and relativistic +factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated +on the grid from the particles’ positions and velocities, while the +electric and magnetic field components are interpolated from the grid +to the particles’ positions for the velocity update. + +.. _theory-pic-push: + +Particle push +------------- + +A centered finite-difference discretization of the Newton-Lorentz +equations of motion is given by + +.. math:: + \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t} = \mathbf{v}^{i+1/2}, + :label: leapfrog_x + +.. math:: + \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t} = \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right). + :label: leapfrog_v + +In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be +expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. + +.. _theory-pic-push-boris: + +Boris relativistic velocity rotation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The solution proposed by Boris :cite:p:`pt-BorisICNSP70` is given by + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}} + :label: boris_v + +where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. + +The system (:eq:`leapfrog_v`, :eq:`boris_v`) is solved very +efficiently following Boris’ method, where the electric field push +is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the +velocity is updated using the following sequence: + +.. math:: + + \begin{aligned} + \mathbf{u^{-}} & = \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \\ + \mathbf{u'} & = \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t} + \\ + \mathbf{u}^{+} & = \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+\mathbf{t}^{2}) + \\ + \mathbf{u}^{i+1/2} & = \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \end{aligned} + +where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where +:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. + +The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. + +.. _theory-pic-push-vay: + +Vay Lorentz-invariant formulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It was shown in :cite:t:`pt-Vaypop2008` that the Boris formulation is +not Lorentz invariant and can lead to significant errors in the treatment +of relativistic dynamics. A Lorentz invariant formulation is obtained +by considering the following velocity average + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2}. + :label: new_v + +This gives a system that is solvable analytically (see :cite:t:`pt-Vaypop2008` +for a detailed derivation), giving the following velocity update: + +.. math:: + \mathbf{u^{*}} = \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right), + :label: pusher_gamma + +.. math:: + \mathbf{u}^{i+1/2} = \frac{\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}}{1+\mathbf{t}^{2}}, + :label: pusher_upr + +where + +.. math:: + + \begin{align} + \mathbf{t} & = \boldsymbol{\tau}/\gamma^{i+1/2}, + \\ + \boldsymbol{\tau} & = \left(q\Delta t/2m\right)\mathbf{B}^{i}, + \\ + \gamma^{i+1/2} & = \sqrt{\sigma+\sqrt{\sigma^{2}+\left(\boldsymbol{\tau}^{2}+w^{2}\right)}}, + \\ + w & = \mathbf{u^{*}}\cdot\boldsymbol{\tau}, + \\ + \sigma & = \left(\gamma'^{2}-\boldsymbol{\tau}^{2}\right)/2, + \\ + \gamma' & = \sqrt{1+(\mathbf{u}^{*}/c)^{2}}. + \end{align} + +This Lorentz invariant formulation +is particularly well suited for the modeling of ultra-relativistic +charged particle beams, where the accurate account of the cancellation +of the self-generated electric and magnetic fields is essential, as +shown in :cite:t:`pt-Vaypop2008`. + +.. _theory-pic-mwsolve: + +Field solve +----------- + +Various methods are available for solving Maxwell’s equations on a +grid, based on finite-differences, finite-volume, finite-element, +spectral, or other discretization techniques that apply most commonly +on single structured or unstructured meshes and less commonly on multiblock +multiresolution grid structures. In this chapter, we summarize the widespread +second order finite-difference time-domain (FDTD) algorithm, its extension +to non-standard finite-differences as well as the pseudo-spectral +analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) +algorithms. Extension to multiresolution (or mesh refinement) PIC +is described in, e.g., :cite:t:`pt-VayCSD12,pt-Vaycpc04`. + +.. _fig_yee_grid: + +.. figure:: Yee_grid.png + :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + + (left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + +.. _theory-pic-mwsolve-fdtd: + +Finite-Difference Time-Domain (FDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The most popular algorithm for electromagnetic PIC codes is the Finite-Difference +Time-Domain (or FDTD) solver + +.. math:: + D_{t}\mathbf{B} = -\nabla\times\mathbf{E} + :label: Faraday-2 + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-2 + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss-2 + +.. math:: + \left[\nabla\cdot\mathbf{B} = 0\right]. + :label: divb-2 + +The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` +and the finite-difference operators in time and space are defined +respectively as + +.. math:: + + \begin{align} + D_{t}G|_{i,j,k}^{n} & = \frac{(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2})}{\Delta t}, + \\ + D_{x}G|_{i,j,k}^{n} & = \frac{G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}}{\Delta x}, + \end{align} + +where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and +the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` +and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. +The difference operators along :math:`y` and :math:`z` are obtained by circular +permutation. The equations in brackets are given for completeness, +as they are often not actually solved, thanks to the usage of a so-called +charge conserving algorithm, as explained below. As shown in :numref:`fig_yee_grid`, +the quantities are given on a staggered (or “Yee”) +grid :cite:p:`pt-Yee`, where the electric field components are located +between nodes and the magnetic field components are located in the +center of the cell faces. Knowing the current densities at half-integer steps, +the electric field components are updated alternately with the magnetic +field components at integer and half-integer steps respectively. + +.. _theory-pic-mwsolve-nsfdtd: + +Non-Standard Finite-Difference Time-Domain (NSFDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An implementation of the source-free Maxwell’s wave equations for narrow-band +applications based on non-standard finite-differences (NSFD) +was introduced in :cite:t:`pt-Coleieee1997,pt-Coleieee2002`, and +was adapted for wideband applications in :cite:t:`pt-Karkicap06`. At +the Courant limit for the time step and for a given set of parameters, +the stencil proposed in :cite:t:`pt-Karkicap06` has no numerical dispersion +along the principal axes, provided that the cell size is the same +along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainen” +(or CK) solver uses the non-standard finite difference formulation +(based on extended stencils) of the Maxwell-Ampere equation and can be +implemented as follows :cite:p:`pt-Vayjcp2011`: + +.. math:: + D_{t}\mathbf{B} = -\nabla^{*}\times\mathbf{E} + :label: Faraday + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss + +.. math:: + \left[\nabla^{*}\cdot\mathbf{B}= 0\right] + :label: divb + +Eqs. (:eq:`Gauss`) and (:eq:`divb`) are not being solved explicitly +but verified via appropriate initial conditions and current deposition +procedure. The NSFD differential operator is given by + +.. math:: + + \nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}} + +where + +.. math:: + + D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x} + +with + +.. math:: + + \begin{align} + S_{x}^{1}G|_{i,j,k}^{n} & = G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}, + \\ + S_{x}^{2}G|_{i,j,k}^{n} & = G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}. + \end{align} + +Here :math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` +are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with +the FDTD algorithm, the quantities with half-integer are located between +the nodes (electric field components) or in the center of the cell +faces (magnetic field components). The operators along :math:`y` and :math:`z`, +i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, +:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular +permutation of the indices. + +Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients +given in :cite:t:`pt-Karkicap06` (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) +allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which +equates to having no numerical dispersion along the principal axes. +The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. +An extension to non-cubic cells is provided in 3-D by :cite:t:`pt-CowanPRSTAB13` and in 2-D by +:cite:t:`pt-PukhovJPP99`. An alternative NSFDTD implementation that enables superluminous waves is also +given in :cite:t:`pt-LehePRSTAB13`. + +As mentioned above, a key feature of the algorithms based on NSFDTD +is that some implementations :cite:p:`pt-Karkicap06,pt-CowanPRSTAB13` enable the time step :math:`\Delta t=\Delta x` along one or +more axes and no numerical dispersion along those axes. However, as +shown in :cite:t:`pt-Vayjcp2011`, an instability develops at the Nyquist +wavelength at (or very near) such a timestep. It is also shown in +the same paper that removing the Nyquist component in all the source +terms using a bilinear filter (see description of the filter below) +suppresses this instability. + +.. _theory-pic-mwsolve-psatd: + +Pseudo Spectral Analytical Time Domain (PSATD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Maxwell’s equations in Fourier space are given by + +.. math:: \frac{\partial\mathbf{\tilde{E}}}{\partial t} = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}} +.. math:: \frac{\partial\mathbf{\tilde{B}}}{\partial t} = -i\mathbf{k}\times\mathbf{\tilde{E}} +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{E}} = \tilde{\rho}] +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{B}} = 0] + +where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. +As with the real space formulation, provided that the continuity equation +:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then +the last two equations will automatically be satisfied at any time +if satisfied initially and do not need to be explicitly integrated. + +Decomposing the electric field and current between longitudinal and +transverse components + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}} & = \mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}}) + \\ + \mathbf{\tilde{J}} & = \mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}}) + \end{aligned} + +gives + +.. math:: + + \begin{aligned} + \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}} + \\ + \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = -\mathbf{\tilde{J}_{L}} + \\ + \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = -i\mathbf{k}\times\mathbf{\tilde{E}} + \end{aligned} + +with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. + +If the sources are assumed to be constant over a time interval :math:`\Delta t`, +the system of equations is solvable analytically and is given by (see :cite:t:`pt-Habericnsp73` for the original formulation and :cite:t:`pt-VayJCP2013` +for a more detailed derivation): + +.. math:: + \mathbf{\tilde{E}}_{T}^{n+1} = C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2} + :label: PSATD_transverse_1 + +.. math:: + \mathbf{\tilde{E}}_{L}^{n+1} = \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2} + :label: PSATD_longitudinal + +.. math:: + \mathbf{\tilde{B}}^{n+1} = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2} + :label: PSATD_transverse_2 + +with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. + +Combining the transverse and longitudinal components, gives + +.. math:: + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2} + \\ + & + (1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber + \\ + & + \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right), + \end{aligned} + :label: Eq_PSATD_1 + +.. math:: + \begin{aligned} + \mathbf{\tilde{B}}^{n+1} & = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + \\ + & + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}. + \end{aligned} + :label: Eq_PSATD_2 + +For fields generated by the source terms without the self-consistent +dynamics of the charged particles, this algorithm is free of numerical +dispersion and is not subject to a Courant condition. Furthermore, +this solution is exact for any time step size subject to the assumption +that the current source is constant over that time step. + +As shown in :cite:t:`pt-VayJCP2013`, by expanding the coefficients :math:`S_{h}` +and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD +formulation reduces to the perhaps better known pseudo-spectral time-domain +(PSTD) formulation :cite:p:`pt-DawsonRMP83,pt-Liumotl1997`: + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2}, + \\ + \mathbf{\tilde{B}}^{n+3/2} & = \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}. + \end{aligned} + +The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` +In contrast to the PSATD solver, the PSTD solver is subject to numerical +dispersion for a finite time step and to a Courant condition that +is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta z^{2}}\right)^{-1/2}`. + +The PSATD and PSTD formulations that were just given apply to the +field components located at the nodes of the grid. As noted in :cite:t:`pt-Ohmurapiers2010`, +they can also be easily recast on a staggered Yee grid by multiplication +of the field components by the appropriate phase factors to shift +them from the collocated to the staggered locations. The choice between +a collocated and a staggered formulation is application-dependent. + +Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods :cite:p:`pt-VayJCP2013`, at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in :cite:t:`pt-VincentiCPC2017a` for stencils of arbitrary order (up-to the infinite “spectral” order). + +WarpX also includes a kinetic-fluid hybrid model in which the electric field is +calculated using Ohm's law instead of directly evolving Maxwell's equations. This +approach allows reduced physics simulations to be done with significantly lower +spatial and temporal resolution than in the standard, fully kinetic, PIC. Details +of this model can be found in the section +:ref:`Kinetic-fluid hybrid model `. + +.. _current_deposition: + +Current deposition +------------------ + +The current densities are deposited on the computational grid from +the particle position and velocities, employing splines of various +orders :cite:p:`pt-Abejcp86`. + +.. math:: + + \begin{aligned} + \rho & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n + \\ + \mathbf{J} & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n + \end{aligned} + +In most applications, it is essential to prevent the accumulation +of errors resulting from the violation of the discretized Gauss’ Law. +This is accomplished by providing a method for depositing the current +from the particles to the grid that preserves the discretized Gauss’ +Law, or by providing a mechanism for “divergence cleaning” :cite:p:`pt-Birdsalllangdon,pt-Langdoncpc92,pt-Marderjcp87,pt-Vaypop98,pt-Munzjcp2000`. +For the former, schemes that allow a deposition of the current that +is exact when combined with the Yee solver is given in :cite:t:`pt-Villasenorcpc92` +for linear splines and in :cite:t:`pt-Esirkepovcpc01` for splines of arbitrary order. + +The NSFDTD formulations given above and in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13` +apply to the Maxwell-Faraday +equation, while the discretized Maxwell-Ampere equation uses the FDTD +formulation. Consequently, the charge conserving algorithms developed +for current deposition :cite:p:`pt-Villasenorcpc92,pt-Esirkepovcpc01` apply +readily to those NSFDTD-based formulations. More details concerning +those implementations, including the expressions for the numerical +dispersion and Courant condition are given +in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13`. + +Current correction +~~~~~~~~~~~~~~~~~~ + +In the case of the pseudospectral solvers, the current deposition +algorithm generally does not satisfy the discretized continuity equation +in Fourier space: + +.. math:: + \tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}. + +In this case, a Boris correction :cite:p:`pt-Birdsalllangdon` can be applied +in :math:`k` space in the form + +.. math:: + \mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\frac{\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}}{k}\mathbf{\hat{k}}, + +where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction +to the current can be applied (with some similarity to the current +deposition presented by Morse and Nielson in their potential-based +model in :cite:t:`pt-Morsenielson1971`) using + +.. math:: \mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k, + +where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse +component of the current is left untouched while the longitudinal +component is effectively replaced by the one obtained from integration +of the continuity equation, ensuring that the corrected current satisfies +the continuity equation. The advantage of correcting the current rather than +the electric field is that it is more local and thus more compatible with +domain decomposition of the fields for parallel computation :cite:p:`pt-VayJCP2013`. + +Vay deposition +~~~~~~~~~~~~~~ + +Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space :cite:p:`pt-Morsenielson1971,pt-Villasenorcpc92,pt-Esirkepovcpc01`. + +The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils :cite:p:`pt-VayJCP2013`. +The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. +The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as + +- 2D Cartesian geometry: + +.. math:: + \begin{align} + D_x & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_y & = \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_z & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \end{align} + +- 3D Cartesian geometry: + +.. math:: + \begin{align} + \begin{split} + D_x & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_y & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_z & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \end{align} + +Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. +Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. + +Field gather +------------ + +In general, the field is gathered from the mesh onto the macroparticles +using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. +Three variations are considered: + +- “momentum conserving”: fields are interpolated from the grid nodes + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at staggered positions, + they are first interpolated to the nodes on an auxiliary grid), + +- “energy conserving (or Galerkin)”: fields are interpolated from + the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` + for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, + :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` + for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` + and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields + are known at the nodes, they are first interpolated to the staggered + positions on an auxiliary grid), + +- “uniform”: fields are interpolated directly form the Yee grid + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at the nodes, they + are first interpolated to the staggered positions on an auxiliary + grid). + +As shown in :cite:t:`pt-Birdsalllangdon,pt-HockneyEastwoodBook,pt-LewisJCP1972`, +the momentum and energy conserving schemes conserve momentum and energy +respectively at the limit of infinitesimal time steps and generally +offer better conservation of the respective quantities for a finite +time step. The uniform scheme does not conserve momentum nor energy +in the sense defined for the others but is given for completeness, +as it has been shown to offer some interesting properties in the modeling +of relativistically drifting plasmas :cite:p:`pt-GodfreyJCP2013`. + +.. _theory-pic-filter: + +Filtering +--------- + +It is common practice to apply digital filtering to the charge or +current density in Particle-In-Cell simulations as a complement or +an alternative to using higher order splines :cite:p:`pt-Birdsalllangdon`. +A commonly used filter in PIC simulations is the three points filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2 + +where :math:`\phi^{f}` is the filtered quantity. This filter is called +a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and +:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is +given as a function of the filtering coefficient :math:`\alpha` and +the wavenumber :math:`k` by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +The total attenuation :math:`G` for :math:`n` successive applications of filters +of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by + +.. math:: + G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, +so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” +step :cite:p:`pt-Birdsalllangdon`. For the bilinear filter (:math:`\alpha=1/2`), +the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession +of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. + +It is sometimes necessary to filter on a relatively wide band of wavelength, +necessitating the application of a large number of passes of the bilinear +filter or on the use of filters acting on many points. The former +can become very intensive computationally while the latter is problematic +for parallel computations using domain decomposition, as the footprint +of the filter may eventually surpass the size of subdomains. A workaround +is to use a combination of filters of limited footprint. A solution +based on the combination of three point filters with various strides +was proposed in :cite:t:`pt-Vayjcp2011` and operates as follows. + +The bilinear filter provides complete suppression of the signal at +the grid Nyquist wavelength (twice the grid cell size). Suppression +of the signal at integer multiples of the Nyquist wavelength can be +obtained by using a stride :math:`s` in the filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2 + +for which the gain is given by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right). + +For a given stride, the gain is given by the gain of the bilinear +filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength +:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, +as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. +The resulting filter is pass band between the poles, but since the +poles are spread at different integer values in k space, a wide band +low pass filter can be constructed by combining filters using different +strides. As shown in :cite:t:`pt-Vayjcp2011`, the successive application +of 4-passes + compensation of filters with strides 1, 2 and 4 has +a nearly equivalent fall-off in gain as 80 passes + compensation of +a bilinear filter. Yet, the strided filter solution needs only 15 +passes of a three-point filter, compared to 81 passes for an equivalent +n-pass bilinear filter, yielding a gain of 5.4 in number of operations +in favor of the combination of filters with stride. The width of the +filter with stride 4 extends only on 9 points, compared to 81 points +for a single pass equivalent filter, hence giving a gain of 9 in compactness +for the stride filters combination in comparison to the single-pass +filter with large stencil, resulting in more favorable scaling with the number +of computational cores for parallel calculations. + +.. bibliography:: + :keyprefix: pt- diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst deleted file mode 100644 index 0aadbd558f0..00000000000 --- a/Docs/source/theory/picsar_theory.rst +++ /dev/null @@ -1,867 +0,0 @@ -.. raw:: latex - - \markboth{J.-L. Vay, R. Lehe}{Simulations for plasma and laser acceleration.} - -.. raw:: latex - - \maketitle - -.. raw:: latex - - \linenumbers - -.. _theory-pic: - -Particle-in-Cell Method -======================= - -.. figure:: PIC.png - :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - - [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - -In the *electromagnetic particle-in-cell method* (Birdsall and Langdon 1991), -the electromagnetic fields are solved on a grid, usually using Maxwell’s -equations - -.. math:: - - \begin{aligned} - \frac{\mathbf{\partial B}}{\partial t} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-1}\\ - \frac{\mathbf{\partial E}}{\partial t} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-1}\\ - \nabla\cdot\mathbf{E} & = & \rho\label{Eq:Gauss-1}\\ - \nabla\cdot\mathbf{B} & = & 0\label{Eq:divb-1}\end{aligned} - -given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and -:math:`\mathbf{B}` are the electric and magnetic field components, and -:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The -charged particles are advanced in time using the Newton-Lorentz equations -of motion - -.. math:: - - \begin{aligned} - \frac{d\mathbf{x}}{dt}= & \mathbf{v},\label{Eq:Lorentz_x-1}\\ - \frac{d\left(\gamma\mathbf{v}\right)}{dt}= & \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right),\label{Eq:Lorentz_v-1}\end{aligned} - -where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` -are respectively the mass, charge, position, velocity and relativistic -factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated -on the grid from the particles’ positions and velocities, while the -electric and magnetic field components are interpolated from the grid -to the particles’ positions for the velocity update. - -.. _theory-pic-push: - -Particle push -------------- - -A centered finite-difference discretization of the Newton-Lorentz -equations of motion is given by - -.. math:: - - \begin{aligned} - \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t}= & \mathbf{v}^{i+1/2},\label{Eq:leapfrog_x}\\ - \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t}= & \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right).\label{Eq:leapfrog_v}\end{aligned} - -In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be -expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. - -.. _theory-pic-push-boris: - -Boris relativistic velocity rotation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The solution proposed by Boris (Boris 1970) is given by - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}}.\label{Eq:boris_v}\end{aligned} - -where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. - -The system (`[Eq:leapfrog_v] <#Eq:leapfrog_v>`__,\ `[Eq:boris_v] <#Eq:boris_v>`__) is solved very -efficiently following Boris’ method, where the electric field push -is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the -velocity is updated using the following sequence: - -.. math:: - - \begin{aligned} - \mathbf{u^{-}}= & \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\\ - \mathbf{u'}= & \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t}\\ - \mathbf{u}^{+}= & \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+t^{2})\\ - \mathbf{u}^{i+1/2}= & \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\end{aligned} - -where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where -:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. - -The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. - -.. _theory-pic-push-vay: - -Vay Lorentz-invariant formulation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It was shown in (Vay 2008) that the Boris formulation is -not Lorentz invariant and can lead to significant errors in the treatment -of relativistic dynamics. A Lorentz invariant formulation is obtained -by considering the following velocity average - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2},\label{Eq:new_v}\end{aligned} - -This gives a system that is solvable analytically (see (Vay 2008) -for a detailed derivation), giving the following velocity update: - -.. math:: - - \begin{aligned} - \mathbf{u^{*}}= & \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right),\label{pusher_gamma}\\ - \mathbf{u}^{i+1/2}= & \left[\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}\right]/\left(1+t^{2}\right),\label{pusher_upr}\end{aligned} - -where :math:`\mathbf{t}=\boldsymbol{\tau}/\gamma^{i+1/2}`, :math:`\boldsymbol{\tau}=\left(q\Delta t/2m\right)\mathbf{B}^{i}`, -:math:`\gamma^{i+1/2}=\sqrt{\sigma+\sqrt{\sigma^{2}+\left(\tau^{2}+w^{2}\right)}}`, -:math:`w=\mathbf{u^{*}}\cdot\boldsymbol{\tau}`, :math:`\sigma=\left(\gamma'^{2}-\tau^{2}\right)/2` -and :math:`\gamma'=\sqrt{1+(\mathbf{u}^{*}/c)^{2}}`. This Lorentz invariant formulation -is particularly well suited for the modeling of ultra-relativistic -charged particle beams, where the accurate account of the cancellation -of the self-generated electric and magnetic fields is essential, as -shown in (Vay 2008). - -.. _theory-pic-mwsolve: - -Field solve ------------ - -Various methods are available for solving Maxwell’s equations on a -grid, based on finite-differences, finite-volume, finite-element, -spectral, or other discretization techniques that apply most commonly -on single structured or unstructured meshes and less commonly on multiblock -multiresolution grid structures. In this chapter, we summarize the widespread -second order finite-difference time-domain (FDTD) algorithm, its extension -to non-standard finite-differences as well as the pseudo-spectral -analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) -algorithms. Extension to multiresolution (or mesh refinement) PIC -is described in, e.g. (Vay et al. 2012; Vay, Adam, and Heron 2004). - -.. _theory-pic-mwsolve-fdtd: - -Finite-Difference Time-Domain (FDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The most popular algorithm for electromagnetic PIC codes is the Finite-Difference -Time-Domain (or FDTD) solver - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-2}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-2}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss-2}\\ - \left[\nabla\cdot\mathbf{B}\right. & = & \left.0\right].\label{Eq:divb-2}\end{aligned} - -.. figure:: Yee_grid.png - :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - - [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - -The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` -and the finite-difference operators in time and space are defined -respectively as - -.. math:: D_{t}G|_{i,j,k}^{n}=\left(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2}\right)/\Delta t - -and :math:`D_{x}G|_{i,j,k}^{n}=\left(G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}\right)/\Delta x`, -where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and -the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` -and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. -The difference operators along :math:`y` and :math:`z` are obtained by circular -permutation. The equations in brackets are given for completeness, -as they are often not actually solved, thanks to the usage of a so-called -charge conserving algorithm, as explained below. As shown in Figure -`[fig:yee_grid] <#fig:yee_grid>`__, the quantities are given on a staggered (or “Yee”) -grid (Yee 1966), where the electric field components are located -between nodes and the magnetic field components are located in the -center of the cell faces. Knowing the current densities at half-integer steps, -the electric field components are updated alternately with the magnetic -field components at integer and half-integer steps respectively. - -.. _theory-pic-mwsolve-nsfdtd: - -Non-Standard Finite-Difference Time-Domain (NSFDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In (Cole 1997, 2002), Cole introduced an implementation -of the source-free Maxwell’s wave equations for narrow-band applications -based on non-standard finite-differences (NSFD). In (Karkkainen et al. 2006), -Karkkainen *et al.* adapted it for wideband applications. At -the Courant limit for the time step and for a given set of parameters, -the stencil proposed in (Karkkainen et al. 2006) has no numerical dispersion -along the principal axes, provided that the cell size is the same -along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainnen” -(or CK) solver uses the non-standard finite difference formulation -(based on extended stencils) of the Maxwell-Ampere equation and can be -implemented as follows (Vay et al. 2011): - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla^{*}\times\mathbf{E}\label{Eq:Faraday}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss}\\ - \left[\nabla^{*}\cdot\mathbf{B}\right. & = & \left.0\right]\label{Eq:divb}\end{aligned} - -Eq. `[Eq:Gauss] <#Eq:Gauss>`__ and `[Eq:divb] <#Eq:divb>`__ are not being solved explicitly -but verified via appropriate initial conditions and current deposition -procedure. The NSFD differential operators is given by :math:`\nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}}` -where :math:`D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x}` -with :math:`S_{x}^{1}G|_{i,j,k}^{n}=G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}`, -:math:`S_{x}^{2}G|_{i,j,k}^{n}=G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}`. -:math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` -are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with -the FDTD algorithm, the quantities with half-integer are located between -the nodes (electric field components) or in the center of the cell -faces (magnetic field components). The operators along :math:`y` and :math:`z`, -i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, -:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular -permutation of the indices. - -Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients -given in (Karkkainen et al. 2006) (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) -allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which -equates to having no numerical dispersion along the principal axes. -The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. -An extension to non-cubic cells is provided by Cowan, *et al.* -in 3-D in (Cowan et al. 2013) and was given by Pukhov in 2-D in -(Pukhov 1999). An alternative NSFDTD implementation that enables superluminous waves is also -given by Lehe et al. in (Lehe et al. 2013). - -As mentioned above, a key feature of the algorithms based on NSFDTD -is that some implementations (Karkkainen et al. 2006; Cowan et al. 2013) enable the time step :math:`\Delta t=\Delta x` along one or -more axes and no numerical dispersion along those axes. However, as -shown in (Vay et al. 2011), an instability develops at the Nyquist -wavelength at (or very near) such a timestep. It is also shown in -the same paper that removing the Nyquist component in all the source -terms using a bilinear filter (see description of the filter below) -suppresses this instability. - -.. _theory-pic-mwsolve-psatd: - -Pseudo Spectral Analytical Time Domain (PSATD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Maxwell’s equations in Fourier space are given by - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{E}}& = & \tilde{\rho}]\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{B}}& = & 0]\end{aligned} - -where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. -As with the real space formulation, provided that the continuity equation -:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then -the last two equations will automatically be satisfied at any time -if satisfied initially and do not need to be explicitly integrated. - -Decomposing the electric field and current between longitudinal and -transverse components :math:`\mathbf{\tilde{E}}=\mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}})` -and :math:`\mathbf{\tilde{J}}=\mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}})` -gives - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}}\\ - \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = & -\mathbf{\tilde{J}_{L}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\end{aligned} - -with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. - -If the sources are assumed to be constant over a time interval :math:`\Delta t`, -the system of equations is solvable analytically and is given by (see -(Haber et al. 1973) for the original formulation and (Jean-Luc Vay, Haber, and Godfrey 2013) -for a more detailed derivation): - -[Eq:PSATD] - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}_{T}^{n+1} & = & C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2}\label{Eq:PSATD_transverse_1}\\ - \mathbf{\tilde{E}}_{L}^{n+1} & = & \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}\label{Eq:PSATD_transverse_2}\end{aligned} - -with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. - -Combining the transverse and longitudinal components, gives - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2}\\ - & + &(1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber \\ - & + & \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right),\label{Eq_PSATD_1}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}.\label{Eq_PSATD_2}\end{aligned} - -For fields generated by the source terms without the self-consistent -dynamics of the charged particles, this algorithm is free of numerical -dispersion and is not subject to a Courant condition. Furthermore, -this solution is exact for any time step size subject to the assumption -that the current source is constant over that time step. - -As shown in (Jean-Luc Vay, Haber, and Godfrey 2013), by expanding the coefficients :math:`S_{h}` -and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD -formulation reduces to the perhaps better known pseudo-spectral time-domain -(PSTD) formulation (Dawson 1983; Liu 1997): - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2},\\ - \mathbf{\tilde{B}}^{n+3/2} & = & \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}.\end{aligned} - -The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` -In contrast to the PSATD solver, the PSTD solver is subject to numerical -dispersion for a finite time step and to a Courant condition that -is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta x^{2}}\right)^{-1/2}.` - -The PSATD and PSTD formulations that were just given apply to the -field components located at the nodes of the grid. As noted in (Ohmura and Okamura 2010), -they can also be easily recast on a staggered Yee grid by multiplication -of the field components by the appropriate phase factors to shift -them from the collocated to the staggered locations. The choice between -a collocated and a staggered formulation is application-dependent. - -Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods (Jean-Luc Vay, Haber, and Godfrey 2013), at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in (Vincenti and Vay 2016) for stencils of arbitrary order (up-to the infinite “spectral” order). - -WarpX also includes a kinetic-fluid hybrid model in which the electric field is -calculated using Ohm's law instead of directly evolving Maxwell's equations. This -approach allows reduced physics simulations to be done with significantly lower -spatial and temporal resolution than in the standard, fully kinetic, PIC. Details -of this model can be found in the section -:ref:`Kinetic-fluid hybrid model `. - -.. _current_deposition: - -Current deposition ------------------- - -The current densities are deposited on the computational grid from -the particle position and velocities, employing splines of various -orders (Abe et al. 1986). - -.. math:: - - \begin{aligned} - \rho & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n\\ - \mathbf{J} & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n\end{aligned} - -In most applications, it is essential to prevent the accumulation -of errors resulting from the violation of the discretized Gauss’ Law. -This is accomplished by providing a method for depositing the current -from the particles to the grid that preserves the discretized Gauss’ -Law, or by providing a mechanism for “divergence cleaning” (Birdsall and Langdon 1991; Langdon 1992; Marder 1987; Vay and Deutsch 1998; Munz et al. 2000). -For the former, schemes that allow a deposition of the current that -is exact when combined with the Yee solver is given in (Villasenor and Buneman 1992) -for linear splines and in (Esirkepov 2001) for splines of arbitrary order. - -The NSFDTD formulations given above and in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013) -apply to the Maxwell-Faraday -equation, while the discretized Maxwell-Ampere equation uses the FDTD -formulation. Consequently, the charge conserving algorithms developed -for current deposition (Villasenor and Buneman 1992; Esirkepov 2001) apply -readily to those NSFDTD-based formulations. More details concerning -those implementations, including the expressions for the numerical -dispersion and Courant condition are given -in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013). - -Current correction -~~~~~~~~~~~~~~~~~~ - -In the case of the pseudospectral solvers, the current deposition -algorithm generally does not satisfy the discretized continuity equation -in Fourier space :math:`\tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}`. -In this case, a Boris correction (Birdsall and Langdon 1991) can be applied -in :math:`k` space in the form :math:`\mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\left(\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}\right)\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction -to the current can be applied (with some similarity to the current -deposition presented by Morse and Nielson in their potential-based -model in (Morse and Nielson 1971)) using :math:`\mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse -component of the current is left untouched while the longitudinal -component is effectively replaced by the one obtained from integration -of the continuity equation, ensuring that the corrected current satisfies -the continuity equation. The advantage of correcting the current rather than -the electric field is that it is more local and thus more compatible with -domain decomposition of the fields for parallel computation (Jean Luc Vay, Haber, and Godfrey 2013). - -Vay deposition -~~~~~~~~~~~~~~ - -Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space (`Morse and Nielson, 1971 `_; `Villasenor and Buneman, 1992 `_; `Esirkepov, 2001 `_). - -The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils `(Vay et al, 2013) `_. -The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. -The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as - -- 2D Cartesian geometry: - -.. math:: - \begin{align} - D_x = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_y = & \: \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_z = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \end{align} - -- 3D Cartesian geometry: - -.. math:: - \begin{align} - \begin{split} - D_x = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_y = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_z = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} - \end{align} - -Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. -Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. - -Field gather ------------- - -In general, the field is gathered from the mesh onto the macroparticles -using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. -Three variations are considered: - -- “momentum conserving”: fields are interpolated from the grid nodes - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at staggered positions, - they are first interpolated to the nodes on an auxiliary grid), - -- “energy conserving (or Galerkin)”: fields are interpolated from - the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` - for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, - :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` - for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` - and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields - are known at the nodes, they are first interpolated to the staggered - positions on an auxiliary grid), - -- “uniform”: fields are interpolated directly form the Yee grid - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at the nodes, they - are first interpolated to the staggered positions on an auxiliary - grid). - -As shown in :raw-latex:`\cite{BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972}`, -the momentum and energy conserving schemes conserve momentum and energy -respectively at the limit of infinitesimal time steps and generally -offer better conservation of the respective quantities for a finite -time step. The uniform scheme does not conserve momentum nor energy -in the sense defined for the others but is given for completeness, -as it has been shown to offer some interesting properties in the modeling -of relativistically drifting plasmas :raw-latex:`\cite{GodfreyJCP2013}`. - -.. _theory-pic-filter: - -Filtering -========= - -It is common practice to apply digital filtering to the charge or -current density in Particle-In-Cell simulations as a complement or -an alternative to using higher order splines (Birdsall and Langdon 1991). -A commonly used filter in PIC simulations is the three points filter -:math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2` -where :math:`\phi^{f}` is the filtered quantity. This filter is called -a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and -:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is -given as a function of the filtering coefficient :math:`\alpha` and -the wavenumber :math:`k` by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -The total attenuation :math:`G` for :math:`n` successive applications of filters -of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by :math:`G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, -so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” -step (Birdsall and Langdon 1991). For the bilinear filter (:math:`\alpha=1/2`), -the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession -of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. - -It is sometimes necessary to filter on a relatively wide band of wavelength, -necessitating the application of a large number of passes of the bilinear -filter or on the use of filters acting on many points. The former -can become very intensive computationally while the latter is problematic -for parallel computations using domain decomposition, as the footprint -of the filter may eventually surpass the size of subdomains. A workaround -is to use a combination of filters of limited footprint. A solution -based on the combination of three point filters with various strides -was proposed in (Vay et al. 2011) and operates as follows. - -The bilinear filter provides complete suppression of the signal at -the grid Nyquist wavelength (twice the grid cell size). Suppression -of the signal at integer multiples of the Nyquist wavelength can be -obtained by using a stride :math:`s` in the filter :math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2` -for which the gain is given by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -For a given stride, the gain is given by the gain of the bilinear -filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength -:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, -as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. -The resulting filter is pass band between the poles, but since the -poles are spread at different integer values in k space, a wide band -low pass filter can be constructed by combining filters using different -strides. As shown in (Vay et al. 2011), the successive application -of 4-passes + compensation of filters with strides 1, 2 and 4 has -a nearly equivalent fall-off in gain as 80 passes + compensation of -a bilinear filter. Yet, the strided filter solution needs only 15 -passes of a three-point filter, compared to 81 passes for an equivalent -n-pass bilinear filter, yielding a gain of 5.4 in number of operations -in favor of the combination of filters with stride. The width of the -filter with stride 4 extends only on 9 points, compared to 81 points -for a single pass equivalent filter, hence giving a gain of 9 in compactness -for the stride filters combination in comparison to the single-pass -filter with large stencil, resulting in more favorable scaling with the number -of computational cores for parallel calculations. - -.. raw:: latex - - \IfFileExists{\jobname.bbl}{} {\typeout{} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{{*}{*} Please run \textquotedbl{}bibtex \jobname\textquotedbl{} - to optain} \typeout{{*}{*} the bibliography and then re-run LaTeX} - \typeout{{*}{*} twice to fix the references!} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{} } - -.. raw:: html - -
- -.. raw:: html - -
- -Abe, H, N Sakairi, R Itatani, and H Okuda. 1986. “High-Order Spline Interpolations in the Particle Simulation.” *Journal of Computational Physics* 63 (2): 247–67. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp. 1970. “Relativistic Plasma Simulation-Optimization of a Hybrid Code.” In *Proc. Fourth Conf. Num. Sim. Plasmas*, 3–67. Naval Res. Lab., Wash., D. C. - -.. raw:: html - -
- -.. raw:: html - -
- -Cole, J. B. 1997. “A High-Accuracy Realization of the Yee Algorithm Using Non-Standard Finite Differences.” *Ieee Transactions on Microwave Theory and Techniques* 45 (6): 991–96. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “High-Accuracy Yee Algorithm Based on Nonstandard Finite Differences: New Developments and Verifications.” *Ieee Transactions on Antennas and Propagation* 50 (9): 1185–91. https://doi.org/10.1109/Tap.2002.801268. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Dawson, J M. 1983. “Particle Simulation of Plasmas.” *Reviews of Modern Physics* 55 (2): 403–47. https://doi.org/10.1103/RevModPhys.55.403. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Karkkainen, M, E Gjonaj, T Lau, and T Weiland. 2006. “Low-Dispersionwake Field Calculation Tools.” In *Proc. Of International Computational Accelerator Physics Conference*, 35–40. Chamonix, France. - -.. raw:: html - -
- -.. raw:: html - -
- -Langdon, A B. 1992. “On Enforcing Gauss Law in Electromagnetic Particle-in-Cell Codes.” *Computer Physics Communications* 70 (3): 447–50. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R, A Lifschitz, C Thaury, V Malka, and X Davoine. 2013. “Numerical growth of emittance in simulations of laser-wakefield acceleration.” *Physical Review Special Topics-Accelerators and Beams* 16 (2). https://doi.org/10.1103/PhysRevSTAB.16.021301. - -.. raw:: html - -
- -.. raw:: html - -
- -Liu, Qh. 1997. “The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength.” *Microwave and Optical Technology Letters* 15 (3): 158–65. `https://doi.org/10.1002/(SICI)1098-2760(19970620)15\:3\<158\:\:AID-MOP11\>3.0.CO\;2-3 3.0.CO\;2-3>`_. - -.. raw:: html - -
- -.. raw:: html - -
- -Marder, B. 1987. “A Method for Incorporating Gauss Law into Electromagnetic Pic Codes.” *Journal of Computational Physics* 68 (1): 48–55. - -.. raw:: html - -
- -.. raw:: html - -
- -Morse, Rl, and Cw Nielson. 1971. “Numerical Simulation of Weibel Instability in One and 2 Dimensions.” *Phys. Fluids* 14 (4): 830 –&. https://doi.org/10.1063/1.1693518. - -.. raw:: html - -
- -.. raw:: html - -
- -Munz, Cd, P Omnes, R Schneider, E Sonnendrucker, and U Voss. 2000. “Divergence Correction Techniques for Maxwell Solvers Based on A Hyperbolic Model.” *Journal of Computational Physics* 161 (2): 484–511. https://doi.org/10.1006/Jcph.2000.6507. - -.. raw:: html - -
- -.. raw:: html - -
- -Ohmura, Y, and Y Okamura. 2010. “Staggered Grid Pseudo-Spectral Time-Domain Method for Light Scattering Analysis.” *Piers Online* 6 (7): 632–35. - -.. raw:: html - -
- -.. raw:: html - -
- -Pukhov, A. 1999. “Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab).” *Journal of Plasma Physics* 61 (3): 425–33. https://doi.org/10.1017/S0022377899007515. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean-Luc, Irving Haber, and Brendan B Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243 (June): 260–68. https://doi.org/10.1016/j.jcp.2013.03.010. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., and C Deutsch. 1998. “Charge Compensated Ion Beam Propagation in A Reactor Sized Chamber.” *Physics of Plasmas* 5 (4): 1190–7. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., D P Grote, R H Cohen, and A Friedman. 2012. “Novel methods in the particle-in-cell accelerator code-framework warp.” Journal Paper. *Computational Science and Discovery* 5 (1): 014019 (20 pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Villasenor, J, and O Buneman. 1992. “Rigorous Charge Conservation for Local Electromagnetic-Field Solvers.” *Computer Physics Communications* 69 (2-3): 306–16. - -.. raw:: html - -
- -.. raw:: html - -
- -Vincenti, H., and J.-L. Vay. 2016. “Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver.” *Computer Physics Communications* 200 (March). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 147–67. https://doi.org/10.1016/j.cpc.2015.11.009. - -.. raw:: html - -
- -.. raw:: html - -
- -Yee, Ks. 1966. “Numerical Solution of Initial Boundary Value Problems Involving Maxwells Equations in Isotropic Media.” *Ieee Transactions on Antennas and Propagation* Ap14 (3): 302–7. - -.. raw:: html - -
- -.. raw:: html - -
diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index eeee6ddf092..0492372b4e6 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -25,14 +25,6 @@ Plasma-Based Acceleration examples/pwfa/README.rst pwfa.rst -Coming soon: - -* LWFA: External injection in the boosted frame -* LWFA: Ionization injection in the lab frame using a LASY data file -* PWFA: External injection in the boosted frame -* PWFA: Self-injection in the lab frame -* MR case? - Laser-Plasma Interaction ------------------------ @@ -43,11 +35,6 @@ Laser-Plasma Interaction examples/laser_ion/README.rst examples/plasma_mirror/README.rst -Coming soon: - -* MVA (3D & RZ) -* MR for the planar example? - Particle Accelerator & Beam Physics ----------------------------------- @@ -56,12 +43,7 @@ Particle Accelerator & Beam Physics :maxdepth: 1 examples/gaussian_beam/README.rst - -Coming soon: - -* Beam-Beam Collision -* Beam Transport or Injector -* Cathode/source + examples/beam-beam_collision/README.rst High Energy Astrophysical Plasma Physics @@ -89,11 +71,6 @@ Nuclear Fusion TODO -Coming soon: - -* Microchannel -* Magnetically Confined Plasma with a Single Coil - Magnetic bottle: simple geometry with an external field - Fundamental Plasma Physics -------------------------- @@ -104,18 +81,22 @@ Fundamental Plasma Physics examples/langmuir/README.rst examples/capacitive_discharge/README.rst -Coming soon: - -* Expanding Sphere example .. _examples-hybrid-model: Kinetic-fluid Hybrid Models ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. -The first few examples are replications of the verification tests described in :cite:t:`ex-MUNOZ2018`. -The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures in the examples below were generated at that time. +WarpX includes a reduced plasma model in which electrons are treated as a massless +fluid while ions are kinetically evolved, and Ohm's law is used to calculate +the electric field. This model is appropriate for problems in which ion kinetics +dominate (ion cyclotron waves, for instance). See the +:ref:`theory section ` for more details. Several +examples and benchmarks of this kinetic-fluid hybrid model are provided below. +A few of the examples are replications of the verification tests described in +:cite:t:`ex-MUNOZ2018`. The hybrid-PIC model was added to WarpX in +`PR #3665 `_ - the figures in the +examples below were generated at that time. .. toctree:: :maxdepth: 1 @@ -142,7 +123,7 @@ Manipulating fields via Python .. note:: - TODO: The section needs to be sorted into either science cases (above) or later sections (workflows and Python API details). + TODO: The section needs to be sorted into either science cases (above) or later sections (:ref:`workflows and Python API details `). An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. diff --git a/Docs/source/usage/examples/beam-beam_collision b/Docs/source/usage/examples/beam-beam_collision new file mode 120000 index 00000000000..8c6ac6b30b1 --- /dev/null +++ b/Docs/source/usage/examples/beam-beam_collision @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/beam-beam_collision \ No newline at end of file diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 18c1e53ada2..674f86143e3 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -81,6 +81,43 @@ Overall simulation parameters one should not expect to obtain the same random numbers, even if a fixed ``warpx.random_seed`` is provided. +* ``algo.evolve_scheme`` (`string`, default: `explicit`) + Specifies the evolve scheme used by WarpX. + + * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD + + * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. + Note that this method is for demonstration only. It is inefficient and does not work well when + :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. + For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + + * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. + For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + +* ``algo.max_picard_iterations`` (`integer`, default: 10) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard + itearations that are done each time step. + +* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of + the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. + If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last + iteration (for a slight optimization). + +* ``algo.require_picard_convergence`` (`bool`, default: 1) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step + is required to converge. + If it is required, an abort is raised if it does not converge and the code then exits. + If not, then a warning is issued and the calculation continues. + * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating the fields at each iteration with the full Maxwell equations, the fields @@ -713,29 +750,48 @@ Particle initialization * ``SingleParticle``: Inject a single macroparticle. This requires the additional parameters: - ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) - ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) - ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) + + * ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) + + * ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) + + * ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) * ``MultipleParticles``: Inject multiple macroparticles. This requires the additional parameters: - ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) - ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) - ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) - ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) - ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) - ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) - ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) + + * ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) + + * ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) + + * ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) + + * ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) + + * ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) + + * ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) + + * ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) * ``gaussian_beam``: Inject particle beam with gaussian distribution in space in all directions. This requires additional parameters: - ``.q_tot`` (beam charge), - ``.npart`` (number of particles in the beam), - ``.x/y/z_m`` (average position in `x/y/z`), - ``.x/y/z_rms`` (standard deviation in `x/y/z`), - ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), - and optional arguments ``.do_symmetrize`` (whether to - symmetrize the beam) and ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + + * ``.q_tot`` (beam charge), + + * ``.npart`` (number of particles in the beam), + + * ``.x/y/z_m`` (average position in `x/y/z`), + + * ``.x/y/z_rms`` (standard deviation in `x/y/z`), + + There are additional optional parameters: + + * ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), + * ``.do_symmetrize`` (optional, whether to symmetrize the beam) + + * ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + If ``.do_symmetrize`` is 0, no symmetrization occurs. If ``.do_symmetrize`` is 1, then the beam is symmetrized according to the value of ``.symmetrization_order``. If set to 4, symmetrization is in the x and y direction, (x,y) (-x,y) (x,-y) (-x,-y). @@ -743,11 +799,17 @@ Particle initialization * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: - ``.injection_file`` (`string`) openPMD file name and - ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. - ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + + * ``.injection_file`` (`string`) openPMD file name and + + * ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. + + * ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + Warning: ``q_tot!=0`` is not supported with the ``external_file`` injection style. If a value is provided, it is ignored and no re-scaling is done. The external file must include the species ``openPMD::Record`` labeled ``position`` and ``momentum`` (`double` arrays), with dimensionality and units set via ``openPMD::setUnitDimension`` and ``setUnitSI``. If the external file also contains ``openPMD::Records`` for ``mass`` and ``charge`` (constant `double` scalars) then the species will use these, unless overwritten in the input file (see ``.mass``, ``.charge`` or ``.species_type``). @@ -756,13 +818,20 @@ Particle initialization * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a planar surface. This requires the additional parameters: - ``.flux_profile`` (see the description of this parameter further below) - ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) - ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) - ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) - ``.num_particles_per_cell`` (`double`) - ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) - ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) + + * ``.flux_profile`` (see the description of this parameter further below) + + * ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) + + * ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + + * ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) + + * ``.num_particles_per_cell`` (`double`) + + * ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) + + * ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) * ``none``: Do not inject macro-particles (for example, in a simulation that starts with neutral, ionizable atoms, one may want to create the electrons species -- where ionized electrons can be stored later on -- without injecting electron macro-particles). @@ -1098,6 +1167,11 @@ Particle initialization * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). +* ``.do_adk_correction`` (`0` or `1`) optional (default `0`) + Whether to apply the correction to the ADK theory proposed by Zhang, Lan and Lu in `Q. Zhang et al. (Phys. Rev. A 90, 043410, 2014) `__. + If so, the probability of ionization is modified using an empirical model that should be more accurate in the regime of high electric fields. + Currently, this is only implemented for Hydrogen, although Argon is also available in the same reference. + * ``.physical_element`` (`string`) Only read if `do_field_ionization = 1`. Symbol of chemical element for this species. Example: for Helium, use ``physical_element = He``. @@ -1243,7 +1317,7 @@ Laser initialization be parallel to ``warpx.boost_direction``, for now. * ``.e_max`` (`float` ; in V/m) - Peak amplitude of the laser field. + Peak amplitude of the laser field, in the focal plane. For a laser with a wavelength :math:`\lambda = 0.8\,\mu m`, the peak amplitude is related to :math:`a_0` by: @@ -1257,9 +1331,9 @@ Laser initialization perform the conversion to the boosted frame. * ``.a0`` (`float` ; dimensionless) - Peak normalized amplitude of the laser field (given in the lab frame, just as ``e_max`` above). + Peak normalized amplitude of the laser field, in the focal plane (given in the lab frame, just as ``e_max`` above). See the description of ``.e_max`` for the conversion between ``a0`` and ``e_max``. - Exactly one of ``a0`` and ``e_max`` must be specified. + Either ``a0`` or ``e_max`` must be specified. * ``.wavelength`` (`float`; in meters) The wavelength of the laser in vacuum. @@ -1337,12 +1411,15 @@ Laser initialization Note that :math:`\tau` relates to the full width at half maximum (FWHM) of *intensity*, which is closer to pulse length measurements in experiments, as :math:`\tau = \mathrm{FWHM}_I / \sqrt{2\ln(2)}` :math:`\approx \mathrm{FWHM}_I / 1.1774`. + For a chirped laser pulse (i.e. with a non-zero ``.phi2``), ``profile_duration`` is the Fourier-limited duration of the pulse, not the actual duration of the pulse. See the documentation for ``.phi2`` for more detail. + When running a **boosted-frame simulation**, provide the value of ``.profile_duration`` in the laboratory frame, and use ``warpx.gamma_boost`` to automatically perform the conversion to the boosted frame. * ``.profile_waist`` (`float` ; in meters) - The waist of the transverse Gaussian laser profile, defined as :math:`w_0` : + The waist of the transverse Gaussian :math:`w_0`, i.e. defined such that the electric field of the + laser pulse in the focal plane is of the form: .. math:: @@ -1375,8 +1452,23 @@ Laser initialization See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). * ``.phi2`` (`float`; in seconds**2) optional (default `0.`) - Temporal chirp at focus. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + The amount of temporal chirp :math:`\phi^{(2)}`, at focus (in the lab frame). Namely, a wave packet + centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity + at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`. Thus, a positive + :math:`\phi^{(2)}` corresponds to positive chirp, i.e. red part of the spectrum in the + front of the pulse and blue part of the spectrum in the back. More specifically, the electric + field in the focal plane is of the form: + + .. math:: + + E(\boldsymbol{x},t) \propto Re\left[ \exp\left( -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right] + + where :math:`\tau` is given by ``.profile_duration`` and represents the + Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is: + + .. math:: + + \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 } * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. @@ -1421,9 +1513,15 @@ Laser initialization External fields --------------- -Grid initialization +Applied to the grid ^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_grid_init_`` or ``warpx.E_ext_grid_init_`` +are applied to the grid directly. In particular, these fields can be seen in the diagnostics that output the fields on the grid. + + - When using an **electromagnetic** field solver, these fields are applied to the grid at the beginning of the simulation, and serve as initial condition for the Maxwell solver. + - When using an **electrostatic** or **magnetostatic** field solver, these fields are added to the fields computed by the Poisson solver, at each timestep. + * ``warpx.B_ext_grid_init_style`` (string) optional This parameter determines the type of initialization for the external magnetic field. By default, the @@ -1510,6 +1608,9 @@ Grid initialization Applied to Particles ^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_particle_init_`` or ``warpx.E_ext_particle_init_`` +are applied to the particles directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``particles.E_ext_particle_init_style`` & ``particles.B_ext_particle_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the particles at every timestep. @@ -1570,6 +1671,9 @@ Applied to Particles Applied to Cold Relativistic Fluids ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_init_`` or ``warpx.E_ext_init_`` +are applied to the fluids directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``.E_ext_init_style`` & ``.B_ext_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the cold relativistic fluids at every timestep. @@ -1663,6 +1767,7 @@ Collision models ---------------- WarpX provides several particle collision models, using varying degrees of approximation. +Details about the collision models can be found in the :ref:`theory section `. * ``collisions.collision_names`` (`strings`, separated by spaces) The name of each collision type. @@ -1672,29 +1777,29 @@ WarpX provides several particle collision models, using varying degrees of appro * ``.type`` (`string`) optional The type of collision. The types implemented are: - - ``pairwisecoulomb`` for pairwise Coulomb collisions, the default if unspecified. + - ``pairwisecoulomb`` for pair-wise Coulomb collisions, the default if unspecified. This provides a pair-wise relativistic elastic Monte Carlo binary Coulomb collision model, - following the algorithm given by `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + following the algorithm given by :cite:t:`param-Perez2012`. When the RZ mode is used, `warpx.n_rz_azimuthal_modes` must be set to 1 at the moment, since the current implementation of the collision module assumes axisymmetry. - ``nuclearfusion`` for fusion reactions. - This implements the pair-wise fusion model by `Higginson et al. (JCP 388, 439-453, 2019) `__. + This implements the pair-wise fusion model by :cite:t:`param-Higginson2019`. Currently, WarpX supports deuterium-deuterium, deuterium-tritium, deuterium-helium and proton-boron fusion. When initializing the reactant and product species, you need to use ``species_type`` (see the documentation for this parameter), so that WarpX can identify the type of reaction to use. (e.g. ``.species_type = 'deuterium'``) + - ``dsmc`` for pair-wise, non-Coulomb collisions between kinetic species. + This is a "direct simulation Monte Carlo" treatment of collisions between + kinetic species. See :ref:`DSMC section `. - ``background_mcc`` for collisions between particles and a neutral background. This is a relativistic Monte Carlo treatment for particles colliding - with a neutral background gas. The implementation follows the so-called - null collision strategy discussed for example in `Birdsall (IEEE Transactions on - Plasma Science, vol. 19, no. 2, pp. 65-85, 1991) `_. - See also :ref:`collisions section `. + with a neutral background gas. See :ref:`MCC section `. - ``background_stopping`` for slowing of ions due to collisions with electrons or ions. This implements the approximate formulae as derived in Introduction to Plasma Physics, from Goldston and Rutherford, section 14.2. * ``.species`` (`strings`) - If using ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, + If using ``dsmc``, ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, between which the collision will be considered. (Provide only one name for intra-species collisions.) If using ``background_mcc`` or ``background_stopping`` type this should be the name of the species for which collisions with a background will be included. @@ -1717,7 +1822,7 @@ WarpX provides several particle collision models, using varying degrees of appro :math:`A` is the mass number. If this is not provided, or if a non-positive value is provided, a Coulomb logarithm will be computed automatically according to the algorithm in - `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + :cite:t:`param-Perez2012`. * ``.fusion_multiplier`` (`float`) optional. Only for ``nuclearfusion``. @@ -1728,8 +1833,8 @@ WarpX provides several particle collision models, using varying degrees of appro More specifically, in a fusion reaction between two macroparticles with weight ``w_1`` and ``w_2``, the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multiplier``. (And the weights of the reactant macroparticles are reduced correspondingly after the reaction.) - See `Higginson et al. (JCP 388, 439-453, 2019) `__ - for more details. The default value of ``fusion_multiplier`` is 1. + See :cite:t:`param-Higginson2019` for more details. + The default value of ``fusion_multiplier`` is 1. * ``.fusion_probability_threshold`` (`float`) optional. Only for ``nuclearfusion``. @@ -1799,25 +1904,21 @@ WarpX provides several particle collision models, using varying degrees of appro where :math:`\beta` is the term on the r.h.s except :math:`W_b`. * ``.scattering_processes`` (`strings` separated by spaces) - Only for ``background_mcc``. The MCC scattering processes that should be + Only for ``dsmc`` and ``background_mcc``. The scattering processes that should be included. Available options are ``elastic``, ``back`` & ``charge_exchange`` for ions and ``elastic``, ``excitationX`` & ``ionization`` for electrons. - The ``elastic`` option uses hard-sphere scattering, with a differential - cross section that is independent of angle. - With ``charge_exchange``, the ion velocity is replaced with the neutral - velocity, chosen from a Maxwellian based on the value of - ``.background_temperature``. Multiple excitation events can be included for electrons corresponding to excitation to different levels, the ``X`` above can be changed to a unique identifier for each excitation process. For each scattering process specified - a path to a cross-section data file must also be given. We use + a path to a cross-section data file must also be given. We use ```` as a placeholder going forward. * ``._cross_section`` (`string`) - Only for ``background_mcc``. Path to the file containing cross-section data + Only for ``dsmc`` and ``background_mcc``. Path to the file containing cross-section data for the given scattering processes. The cross-section file must have exactly 2 columns of data, the first containing equally spaced energies in eV and the - second the corresponding cross-section in :math:`m^2`. + second the corresponding cross-section in :math:`m^2`. The energy column should + represent the kinetic energy of the colliding particles in the center-of-mass frame. * ``._energy`` (`float`) Only for ``background_mcc``. If the scattering process is either @@ -2501,8 +2602,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a Only works with ``.format = plotfile``. * ``.coarsening_ratio`` (list of `int`) optional (default `1 1 1`) - Reduce size of the field output by this ratio in each dimension. - (This is done by averaging the field over 1 or 2 points along each direction, depending on the staggering). + Reduce size of the selected diagnostic fields output by this ratio in each dimension. + (For a ratio of N, this is done by averaging the fields over N or (N+1) points depending on the staggering). If ``blocking_factor`` and ``max_grid_size`` are used for the domain decomposition, as detailed in the :ref:`domain decomposition ` section, ``coarsening_ratio`` should be an integer divisor of ``blocking_factor``. If ``warpx.numprocs`` is used instead, the total number of cells in a given @@ -2653,6 +2754,8 @@ This can be important if a large number of particles are lost, avoiding filling In addition to their usual attributes, the saved particles have an integer attribute ``timestamp``, which indicates the PIC iteration at which each particle was absorbed at the boundary. +``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. + .. _running-cpp-parameters-diagnostics-reduced: Reduced Diagnostics @@ -2784,8 +2887,9 @@ Reduced Diagnostics defaulting to ``1``. In RZ geometry, this only saves the 0'th azimuthal mode component of the fields. - Integrated electric and magnetic field components can instead be obtained by specifying + Time integrated electric and magnetic field components can instead be obtained by specifying ``.integrate = true``. + The integration is done every time step even when the data is written out less often. In a *moving window* simulation, the FieldProbe can be set to follow the moving frame by specifying ``.do_moving_window_FP = 1`` (default 0). .. warning:: diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index fc26f5db31a..5119184089c 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -44,7 +44,7 @@ Listed below are the key arguments and best-practices relevant for choosing the Finite Difference Time Domain ----------------------------- - For standard plasma wakefield configurations, it is possible to model the physics correctly using the Particle-In-Cell (PIC) Finite Difference Time Domain (FDTD) algorithms (:doc:`../theory/picsar_theory`). + For standard plasma wakefield configurations, it is possible to model the physics correctly using the :ref:`Particle-In-Cell (PIC) ` Finite Difference Time Domain (FDTD) algorithms. If the simulation contains localised extremely high intensity fields, however, numerical instabilities might arise, such as the numerical Cherenkov instability (:doc:`../theory/boosted_frame`). In that case, it is recommended to use the Pseudo Spectral Analytical Time Domain (PSATD) or the Pseudo-Spectral Time-Domain (PSTD) algorithms. In the example we are describing, it is sufficient to use FDTD. @@ -98,7 +98,7 @@ Time step The time step (:math:`dt`) is used to iterated over the main PIC loop and is computed by WarpX differently depending on the Maxwell field FDTD solvers used: - * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :doc:`../theory/picsar_theory` + * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :ref:`theory-pic` * **For CKC** is equal to CFL times the minimum between the boosted frame cell dimensions where CFL is chosen to be below unity and set an optimal trade-off between making the simulation faster and avoiding NCI and other spurious effects. diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index c9d846b41e7..448800f0d4d 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -13,7 +13,7 @@ In the input file, instances of classes are created defining the various aspects A variable of type :py:class:`pywarpx.picmi.Simulation` is the central object to which all other options are passed, defining the simulation time, field solver, registered species, etc. Once the simulation is fully configured, it can be used in one of two modes. -**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: +**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: .. tab-set:: @@ -25,6 +25,9 @@ Once the simulation is fully configured, it can be used in one of two modes. :py:meth:`~pywarpx.picmi.Simulation.write_input_file`: create an :ref:`inputs file for a WarpX executable ` +When run directly from Python, one can also extend WarpX with further custom user logic. +See the :ref:`detailed workflow page ` on how to extend WarpX from Python. + .. _usage-picmi-parameters: @@ -121,6 +124,8 @@ Other operations related to particles: .. autoclass:: pywarpx.picmi.CoulombCollisions +.. autoclass:: pywarpx.picmi.DSMCCollisions + .. autoclass:: pywarpx.picmi.MCCCollisions .. autoclass:: pywarpx.picmi.FieldIonization @@ -137,118 +142,3 @@ Laser profiles can be used to initialize laser pulses in the simulation. Laser injectors control where to initialize laser pulses on the simulation grid. .. autoclass:: pywarpx.picmi.LaserAntenna - - -.. _usage-picmi-extend: - -Extending a Simulation from Python ----------------------------------- - -When running WarpX directly from Python it is possible to interact with the simulation. - -For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: - -.. code-block:: python3 - - # Preparation: set up the simulation - # sim = picmi.Simulation(...) - # ... - - steps = 1000 - for _ in range(steps): - sim.step(nsteps=1) - - # do something custom with the sim object - -As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a -specific location in the WarpX simulation loop. - -.. automodule:: pywarpx.callbacks - :members: installcallback, uninstallcallback, isinstalled - -Data Access -^^^^^^^^^^^ - -While the simulation is running, callbacks can access the WarpX simulation data *in situ*. - -An important object for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. -This object is the Python equivalent to the C++ ``WarpX`` simulation class and provides access to field ``MultiFab`` and ``ParticleContainer`` data. - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.getistep() - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.gett_new() - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.evolve() - -.. autofunction:: pywarpx.picmi.Simulation.extension.finalize() - -These and other classes are provided through `pyAMReX `__. -After the simulation is initialized, pyAMReX can be accessed via - -.. code-block:: python - - from pywarpx import picmi, libwarpx - - # ... simulation definition ... - - # equivalent to - # import amrex.space3d as amr - # for a 3D simulation - amr = libwarpx.amr # picks the right 1d, 2d or 3d variant - -.. py:function:: amr.ParallelDescriptor.NProcs() - -.. py:function:: amr.ParallelDescriptor.MyProc() - -.. py:function:: amr.ParallelDescriptor.IOProcessor() - -.. py:function:: amr.ParallelDescriptor.IOProcessorNumber() - -Particles can be added to the simulation at specific positions and with specific attribute values: - -.. code-block:: python - - from pywarpx import particle_containers, picmi - - # ... - - electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_particles - -Properties of the particles already in the simulation can be obtained with various functions. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_count - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_structs - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_arrays - -The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called -by several utility functions of the form ``get_particle_{comp_name}`` where -``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, -``weight``, ``ux``, ``uy`` or ``uz``. - -New components can be added via Python. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_real_comp - -Various diagnostics are also accessible from Python. -This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. -See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_species_charge_sum - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.deposit_charge_density - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_size - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_structs - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.clear_buffer - -The embedded boundary conditions can be modified when using the electrostatic solver. - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb() diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 7a93d081c6b..e68ec9391be 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -8,11 +8,12 @@ This section collects typical user workflows and best practices for WarpX. .. toctree:: :maxdepth: 2 + workflows/python_extend workflows/domain_decomposition + workflows/plot_distribution_mapping workflows/debugging workflows/libensemble workflows/plot_timestep_duration - workflows/plot_distribution_mapping workflows/psatd_stencil workflows/archiving workflows/ml_dataset_training diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst index ace0577c763..6e60a318bee 100644 --- a/Docs/source/usage/workflows/ml_dataset_training.rst +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -15,7 +15,7 @@ For example, a simulation determined by the following input script :language: python In this section we walk through a workflow for data processing and model training. -This workflow was developed and first presented in Refs. :cite:t:`SandbergIPAC23` and :cite:t:`SandbergPASC24`. +This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. This assumes you have an up-to-date environment with PyTorch and openPMD. @@ -25,21 +25,26 @@ Data Cleaning It is important to inspect the data for artifacts to check that input/output data make sense. If we plot the final phase space for beams 1-8, -the particle data is distributed in a single blob. +the particle data is distributed in a single blob, +as shown by :numref:`fig_phase_space_beam_1` for beam 1. +This is as we expect and what is optimal for training neural networks. + +.. _fig_phase_space_beam_1: .. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png - :alt: Plot comparing model prediction with simulation output. + :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. - Plot showing the final phase space projections of the training data for stage 1. + The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. -This is as we expect and what is optimal for training neural networks. -On the other hand, the final phase space for beam 0 has a halo of outlying particles. +.. _fig_phase_space_beam_0: .. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png - :alt: Plot comparing model prediction with simulation output. + :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. - Plot showing the final phase space projections of the training data for stage 1. + The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 +On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, +has a halo of outlying particles. Looking closer at the z-pz space, we see that some particles got caught in a decelerating region of the wake, have slipped back and are much slower than the rest of the beam. To assist our neural network in learning dynamics of interest, we filter out these particles. @@ -126,7 +131,7 @@ use in training and inference. Neural Network Structure ------------------------ -It was found in Ref. :cite:p:`SandbergPASC24` that reasonable surrogate models are obtained with +It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with shallow feedforward neural networks consisting of fewer than 10 hidden layers and just under 1000 nodes per layer. The example shown here uses 3 hidden layers and 20 nodes per layer @@ -200,7 +205,7 @@ Save Neural Network Parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The model weights are saved after training to record the updates to the model parameters. -Addtionally, we save some model metainformation with the model for convenience, +Additionally, we save some model metainformation with the model for convenience, including the model hyperparameters, the training and testing losses, and how long the training took. .. literalinclude:: ml_materials/train.py @@ -212,7 +217,7 @@ Evaluate -------- In this section we show two ways to diagnose how well the neural network is learning the data. -First we consider the train-test loss curves, shown in Fig. `[fig:train_test_loss] <#fig:train-test>`__ . +First we consider the train-test loss curves, shown in :numref:`fig_train_test_loss`. This figure shows the model error on the training data (in blue) and testing data (in green) as a function of the number of epochs seen. The training data is used to update the model parameters, so training error should be lower than testing error. A key feature to look for in the train-test loss curve is the inflection point in the test loss trend. @@ -221,22 +226,25 @@ The testing error serves as a metric of model generalizability, indicating how w on data it hasn't seen yet. When the test-loss starts to trend flat or even upward, the neural network is no longer improving its ability to generalize to new data. +.. _fig_train_test_loss: + .. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png - :alt: Plot of training and testing loss curves + :alt: Plot of training and testing loss curves versus number of training epochs. - Plot of training (in blue) and testing (in green) loss curves versus number of training epochs. + Training (in blue) and testing (in green) loss curves versus number of training epochs. -A visual inspection of the model prediction can be seen in Fig. `[fig:train_evaluation]` . -This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. +.. _fig_train_evaluation: .. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png :alt: Plot comparing model prediction with simulation output. - Plot comparing model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). + A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). +A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. +This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, -as discussed in :cite:t:`SandbergPASC24`. +as discussed in :cite:t:`ml-SandbergPASC24`. These figures can be generated with the following Python script. @@ -254,3 +262,6 @@ Surrogate Usage in Accelerator Physics A neural network such as the one we trained here can be incorporated in other BLAST codes. `Consider the example using neural networks in ImpactX `__. + +.. bibliography:: + :keyprefix: ml- diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst new file mode 100644 index 00000000000..6c9286c02ce --- /dev/null +++ b/Docs/source/usage/workflows/python_extend.rst @@ -0,0 +1,279 @@ +.. _usage-python-extend: + +Extend a Simulation with Python +=============================== + +When running WarpX directly :ref:`from Python ` it is possible to interact with the simulation. + +For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: + +.. code-block:: python3 + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + steps = 1000 + for _ in range(steps): + sim.step(nsteps=1) + + # do something custom with the sim object + +As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a +specific location in the WarpX simulation loop. + +.. automodule:: pywarpx.callbacks + :members: installcallback, uninstallcallback, isinstalled + + +pyAMReX +------- + +Many of the following classes are provided through `pyAMReX `__. +After the simulation is initialized, the pyAMReX module can be accessed via + +.. code-block:: python + + from pywarpx import picmi, libwarpx + + # ... simulation definition ... + + # equivalent to + # import amrex.space3d as amr + # for a 3D simulation + amr = libwarpx.amr # picks the right 1d, 2d or 3d variant + + +Full details for pyAMReX APIs are `documented here `__. +Important APIs include: + +* `amr.ParallelDescriptor `__: MPI-parallel rank information +* `amr.MultiFab `__: MPI-parallel field data +* `amr.ParticleContainer_* `__: MPI-parallel particle data for a particle species + + +Data Access +----------- + +While the simulation is running, callbacks can have read and write access the WarpX simulation data *in situ*. + +An important object in the ``pywarpx.picmi`` module for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. +This object is the Python equivalent to the C++ ``WarpX`` simulation class. + +.. py:class:: WarpX + + .. py:method:: getistep(lev: int) + + Get the current step on mesh-refinement level ``lev``. + + .. py:method:: gett_new(lev: int) + + Get the current physical time on mesh-refinement level ``lev``. + + .. py:method:: getdt(lev: int) + + Get the current physical time step size on mesh-refinement level ``lev``. + + .. py:method:: multifab(multifab_name: str) + + Return MultiFabs by name, e.g., ``"Efield_aux[x][level=0]"``, ``"Efield_cp[x][level=0]"``, ... + + The physical fields in WarpX have the following naming: + + - ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level + - ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. + - ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher). + + .. py:method:: multi_particle_container + + .. py:method:: get_particle_boundary_buffer + + .. py:method:: set_potential_on_eb(potential: str) + + The embedded boundary (EB) conditions can be modified when using the electrostatic solver. + This set the EB potential string and updates the function parser. + + .. py:method:: evolve(numsteps=-1) + + Evolve the simulation the specified number of steps. + + .. autofunction:: pywarpx.picmi.Simulation.extension.finalize + +.. py::def:: pywarpx.picmi.Simulation.extension.get_instance + + Return a reference to the WarpX object. + + +The :py:class:`WarpX` also provides read and write access to field ``MultiFab`` and ``ParticleContainer`` data, shown in the following examples. + +Fields +^^^^^^ + +This example accesses the :math:`E_x(x,y,z)` field at level 0 after every time step and calculate the largest value in it. + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + + @callfromafterstep + def set_E_x(): + warpx = sim.extension.warpx + + # data access + E_x_mf = warpx.multifab(f"Efield_fp[x][level=0]") + + # compute + # iterate over mesh-refinement levels + for lev in range(warpx.finest_level + 1): + # grow (aka guard/ghost/halo) regions + ngv = E_x_mf.n_grow_vect + + # get every local block of the field + for mfi in E_x_mf: + # global index space box, including guards + bx = mfi.tilebox().grow(ngv) + print(bx) # note: global index space of this block + + # numpy representation: non-copying view, including the + # guard/ghost region; .to_cupy() for GPU! + E_x_np = E_x_mf.array(mfi).to_numpy() + + # notes on indexing in E_x_np: + # - numpy uses locally zero-based indexing + # - layout is F_CONTIGUOUS by default, just like AMReX + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + E_x_np[()] = 42.0 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``E_x``, please see the `pyAMReX documentation `__. + + +High-Level Field Wrapper +"""""""""""""""""""""""" + +.. note:: + + TODO + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + + +Particles +^^^^^^^^^ + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + @callfromafterstep + def my_after_step_callback(): + warpx = sim.extension.warpx + + # data access + multi_pc = warpx.multi_particle_container() + pc = multi_pc.get_particle_container_from_name("electrons") + + # compute + # iterate over mesh-refinement levels + for lvl in range(pc.finest_level + 1): + # get every local chunk of particles + for pti in pc.iterator(pc, level=lvl): + # default layout: AoS with positions and cpuid + aos = pti.aos().to_numpy() + + # additional compile-time and runtime attributes in SoA format + soa = pti.soa().to_numpy() + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + + # write to all particles in the chunk + # note: careful, if you change particle positions, you need to + # redistribute particles before continuing the simulation step + # aos[()]["x"] = 0.30 + # aos[()]["y"] = 0.35 + # aos[()]["z"] = 0.40 + + for soa_real in soa.real: + soa_real[()] = 42.0 + + for soa_int in soa.int: + soa_int[()] = 12 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``electrons``, please see the `pyAMReX documentation `__. + + +High-Level Particle Wrapper +""""""""""""""""""""""""""" + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + +Particles can be added to the simulation at specific positions and with specific attribute values: + +.. code-block:: python + + from pywarpx import particle_containers, picmi + + # ... + + electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") + + +.. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper + :members: + +The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called +by several utility functions of the form ``get_particle_{comp_name}`` where +``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, +``weight``, ``ux``, ``uy`` or ``uz``. + + +Diagnostics +----------- + +Various diagnostics are also accessible from Python. +This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. +See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. + + +.. autoclass:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper + :members: + + +Modify Solvers +-------------- + +From Python, one can also replace numerical solvers in the PIC loop or add new physical processes into the time step loop. +Examples: + +* :ref:`Capacitive Discharge `: replaces the Poisson solver of an electrostatic simulation (default: MLMG) with a python function that uses `superLU `__ to directly solve the Poisson equation. diff --git a/Docs/spack.yaml b/Docs/spack.yaml index db1add0018a..f17c36e5d97 100644 --- a/Docs/spack.yaml +++ b/Docs/spack.yaml @@ -19,11 +19,13 @@ spack: - doxygen - graphviz - python + - py-openpmd-viewer - py-breathe - - py-recommonmark - py-pybtex - py-pygments + - py-recommonmark - py-sphinx - py-sphinx-copybutton - py-sphinx-design - py-sphinx-rtd-theme + - py-yt diff --git a/Examples/Physics_applications/beam-beam_collision/README.rst b/Examples/Physics_applications/beam-beam_collision/README.rst new file mode 100644 index 00000000000..559a81277db --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/README.rst @@ -0,0 +1,70 @@ +.. _examples-beam-beam_collision: + +Beam-beam collision +==================== + +This example shows how to simulate the collision between two ultra-relativistic particle beams. +This is representative of what happens at the interaction point of a linear collider. +We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. + +We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and +the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). + +To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. + + +This example is based on the following paper :cite:t:`ex-Yakimenko2019`. + + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs``. + + +Visualize +--------- + +The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). + +We compare different results: +* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; +* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; +* (black) literature results from :cite:t:`ex-Yakimenko2019`. + +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 128`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: + +.. code-block:: ini + + qed_qs.lookup_table_mode = generate + qed_bw.lookup_table_mode = generate + qed_qs.tab_dndt_chi_min=1e-3 + qed_qs.tab_dndt_chi_max=2e3 + qed_qs.tab_dndt_how_many=512 + qed_qs.tab_em_chi_min=1e-3 + qed_qs.tab_em_chi_max=2e3 + qed_qs.tab_em_chi_how_many=512 + qed_qs.tab_em_frac_how_many=512 + qed_qs.tab_em_frac_min=1e-12 + qed_qs.save_table_in=my_qs_table.txt + qed_bw.tab_dndt_chi_min=1e-2 + qed_bw.tab_dndt_chi_max=2e3 + qed_bw.tab_dndt_how_many=512 + qed_bw.tab_pair_chi_min=1e-2 + qed_bw.tab_pair_chi_max=2e3 + qed_bw.tab_pair_chi_how_many=512 + qed_bw.tab_pair_frac_how_many=512 + qed_bw.save_table_in=my_bw_table.txt + +.. figure:: https://user-images.githubusercontent.com/17280419/291749626-aa61fff2-e6d2-45a3-80ee-84b2851ea0bf.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDMwMzQzNTEsIm5iZiI6MTcwMzAzNDA1MSwicGF0aCI6Ii8xNzI4MDQxOS8yOTE3NDk2MjYtYWE2MWZmZjItZTZkMi00NWEzLTgwZWUtODRiMjg1MWVhMGJmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMjAlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjIwVDAxMDA1MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFiYzY2MGQyYzIyZGIzYzUxOWI3MzNjZTk5ZDM1YzgyNmY4ZDYxOGRlZjAyZTIwNTAyMTc3NTgwN2Q0YjEwNGMmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.I96LQpjqmFXirPDVnBlFQIkCuenR6IuOSY0OIIQvtCo + :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + :width: 100% + + Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs new file mode 100644 index 00000000000..488e997f895 --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/inputs @@ -0,0 +1,231 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.beam_energy = 125.*GeV +my_constants.beam_uz = beam_energy/(mc2) +my_constants.beam_charge = 0.14*nano +my_constants.sigmax = 10*nano +my_constants.sigmay = 10*nano +my_constants.sigmaz = 10*nano +my_constants.beam_uth = 0.1/100.*beam_uz +my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) +my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = -0.5*Lz+3.2*sigmaz + +# BOX +my_constants.Lx = 100.0*clight/omegab +my_constants.Ly = 100.0*clight/omegab +my_constants.Lz = 180.0*clight/omegab + +# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 128 + + +# TIME +my_constants.T = 0.7*Lz/clight +my_constants.dt = sigmaz/clight/10. + +# DIAGS +my_constants.every_red = 1. +warpx.used_inputs_file = warpx_used_inputs.txt + +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 pho1 pho2 ele1 pos1 ele2 pos2 +particles.photon_species = pho1 pho2 + +beam1.species_type = electron +beam1.injection_style = NUniformPerCell +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" +beam1.density_min = n0 / 1e3 +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_uz +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = beam_uth +beam1.uy_th = beam_uth +beam1.uz_th = beam_uth +beam1.initialize_self_fields = 1 +beam1.self_fields_required_precision = 5e-10 +beam1.self_fields_max_iters = 10000 +beam1.do_qed_quantum_sync = 1 +beam1.qed_quantum_sync_phot_product_species = pho1 +beam1.do_classical_radiation_reaction = 0 + +beam2.species_type = positron +beam2.injection_style = NUniformPerCell +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" +beam2.density_min = n0 / 1e3 +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_uz +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = beam_uth +beam2.uy_th = beam_uth +beam2.uz_th = beam_uth +beam2.initialize_self_fields = 1 +beam2.self_fields_required_precision = 5e-10 +beam2.self_fields_max_iters = 10000 +beam2.do_qed_quantum_sync = 1 +beam2.qed_quantum_sync_phot_product_species = pho2 +beam2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +ele1.species_type = electron +ele1.injection_style = none +ele1.self_fields_required_precision = 1e-11 +ele1.self_fields_max_iters = 10000 +ele1.do_qed_quantum_sync = 1 +ele1.qed_quantum_sync_phot_product_species = pho1 +ele1.do_classical_radiation_reaction = 0 + +pos1.species_type = positron +pos1.injection_style = none +pos1.self_fields_required_precision = 1e-11 +pos1.self_fields_max_iters = 10000 +pos1.do_qed_quantum_sync = 1 +pos1.qed_quantum_sync_phot_product_species = pho1 +pos1.do_classical_radiation_reaction = 0 + +ele2.species_type = electron +ele2.injection_style = none +ele2.self_fields_required_precision = 1e-11 +ele2.self_fields_max_iters = 10000 +ele2.do_qed_quantum_sync = 1 +ele2.qed_quantum_sync_phot_product_species = pho2 +ele2.do_classical_radiation_reaction = 0 + +pos2.species_type = positron +pos2.injection_style = none +pos2.self_fields_required_precision = 1e-11 +pos2.self_fields_max_iters = 10000 +pos2.do_qed_quantum_sync = 1 +pos2.qed_quantum_sync_phot_product_species = pho2 +pos2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. + +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 + +qed_bw.lookup_table_mode = builtin +qed_bw.chi_min = 1.e-2 + +# for accurate results use the generated tables with +# the following parameters +# note: must compile with -DWarpX_QED_TABLE_GEN=ON +#qed_qs.lookup_table_mode = generate +#qed_bw.lookup_table_mode = generate +#qed_qs.tab_dndt_chi_min=1e-3 +#qed_qs.tab_dndt_chi_max=2e3 +#qed_qs.tab_dndt_how_many=512 +#qed_qs.tab_em_chi_min=1e-3 +#qed_qs.tab_em_chi_max=2e3 +#qed_qs.tab_em_chi_how_many=512 +#qed_qs.tab_em_frac_how_many=512 +#qed_qs.tab_em_frac_min=1e-12 +#qed_qs.save_table_in=my_qs_table.txt +#qed_bw.tab_dndt_chi_min=1e-2 +#qed_bw.tab_dndt_chi_max=2e3 +#qed_bw.tab_dndt_how_many=512 +#qed_bw.tab_pair_chi_min=1e-2 +#qed_bw.tab_pair_chi_max=2e3 +#qed_bw.tab_pair_chi_how_many=512 +#qed_bw.tab_pair_frac_how_many=512 +#qed_bw.save_table_in=my_bw_table.txt + +warpx.do_qed_schwinger = 0. + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 0 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 +diag1.format = openpmd +diag1.dump_last_timestep = 1 +diag1.species = pho1 pho2 ele1 pos1 ele2 pos2 beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 + +ColliderRelevant_beam1_beam2.type = ColliderRelevant +ColliderRelevant_beam1_beam2.intervals = every_red +ColliderRelevant_beam1_beam2.species = beam1 beam2 + +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = every_red diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 0c6f9828543..38b83e46948 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# --- Copyright 2021 Modern Electron +# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) # --- Monte-Carlo Collision script to reproduce the benchmark tests from # --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 @@ -46,9 +46,7 @@ def initialize_inputs(self): WarpX parser. """ # grab the boundary potentials from the grid object - self.right_voltage = ( - self.grid.potential_zmax.replace('sin', 'np.sin').replace('pi', 'np.pi') - ) + self.right_voltage = self.grid.potential_zmax # set WarpX boundary potentials to None since we will handle it # ourselves in this solver @@ -108,8 +106,12 @@ def solve(self): calculating phi from rho.""" left_voltage = 0.0 - t = self.sim.extension.warpx.gett_new(0) - right_voltage = eval(self.right_voltage) + right_voltage = eval( + self.right_voltage, { + 't': self.sim.extension.warpx.gett_new(0), + 'sin': np.sin, 'pi': np.pi + } + ) # Construct b vector rho = -self.rho_data / constants.ep0 @@ -158,11 +160,12 @@ class CapacitiveDischargeExample(object): # Time (in seconds) between diagnostic evaluations diag_interval = 32 / freq - def __init__(self, n=0, test=False, pythonsolver=False): + def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): """Get input parameters for the specific case (n) desired.""" self.n = n self.test = test self.pythonsolver = pythonsolver + self.dsmc = dsmc # Case specific input parameters self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" @@ -184,6 +187,9 @@ def __init__(self, n=0, test=False, pythonsolver=False): else: self.mcc_subcycling_steps = None + if self.dsmc: + self.rng = np.random.default_rng(23094290) + self.ion_density_array = np.zeros(self.nz + 1) self.setup_run() @@ -236,13 +242,26 @@ def setup_run(self): rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, ) ) + if self.dsmc: + self.neutrals = picmi.Species( + particle_type='He', name='neutrals', + charge=0, mass=self.m_ion, + warpx_reflection_model_zlo=1.0, + warpx_reflection_model_zhi=1.0, + warpx_do_resampling=True, + warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc*1.5), + initial_distribution=picmi.UniformDistribution( + density=self.gas_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, + ) + ) ####################################################################### # Collision initialization # ####################################################################### cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' - mcc_electrons = picmi.MCCCollisions( + electron_colls = picmi.MCCCollisions( name='coll_elec', species=self.electrons, background_density=self.gas_density, @@ -269,24 +288,26 @@ def setup_run(self): } ) - mcc_ions = picmi.MCCCollisions( - name='coll_ion', - species=self.ions, - background_density=self.gas_density, - background_temperature=self.gas_temp, - ndt=self.mcc_subcycling_steps, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'ion_scattering.dat' - }, - 'back' : { - 'cross_section' : cross_sec_direc+'ion_back_scatter.dat' - }, - # 'charge_exchange' : { - # 'cross_section' : cross_sec_direc+'charge_exchange.dat' - # } - } - ) + ion_scattering_processes={ + 'elastic': {'cross_section': cross_sec_direc+'ion_scattering.dat'}, + 'back': {'cross_section': cross_sec_direc+'ion_back_scatter.dat'}, + # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} + } + if self.dsmc: + ion_colls = picmi.DSMCCollisions( + name='coll_ion', + species=[self.ions, self.neutrals], + ndt=5, scattering_processes=ion_scattering_processes + ) + else: + ion_colls = picmi.MCCCollisions( + name='coll_ion', + species=self.ions, + background_density=self.gas_density, + background_temperature=self.gas_temp, + ndt=self.mcc_subcycling_steps, + scattering_processes=ion_scattering_processes + ) ####################################################################### # Initialize simulation # @@ -296,8 +317,7 @@ def setup_run(self): solver=self.solver, time_step_size=self.dt, max_steps=self.max_steps, - warpx_collisions=[mcc_electrons, mcc_ions], - warpx_load_balance_intervals=self.max_steps//5000, + warpx_collisions=[electron_colls, ion_colls], verbose=self.test ) self.solver.sim = self.sim @@ -314,18 +334,36 @@ def setup_run(self): n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid ) ) + if self.dsmc: + self.sim.add_species( + self.neutrals, + layout = picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc//2], grid=self.grid + ) + ) + self.solver.sim_ext = self.sim.extension + + if self.dsmc: + # Periodically reset neutral density to starting temperature + callbacks.installbeforecollisions(self.rethermalize_neutrals) ####################################################################### # Add diagnostics for the CI test to be happy # ####################################################################### - if self.pythonsolver: - file_prefix = 'Python_background_mcc_1d_plt' + if self.dsmc: + file_prefix = 'Python_dsmc_1d_plt' else: - file_prefix = 'Python_background_mcc_1d_tridiag_plt' - + if self.pythonsolver: + file_prefix = 'Python_background_mcc_1d_plt' + else: + file_prefix = 'Python_background_mcc_1d_tridiag_plt' + + species = [self.electrons, self.ions] + if self.dsmc: + species.append(self.neutrals) particle_diag = picmi.ParticleDiagnostic( - species=[self.electrons, self.ions], + species=species, name='diag1', period=0, write_dir='.', @@ -342,6 +380,30 @@ def setup_run(self): self.sim.add_diagnostic(particle_diag) self.sim.add_diagnostic(field_diag) + def rethermalize_neutrals(self): + # When using DSMC the neutral temperature will change due to collisions + # with the ions. This is not captured in the original MCC test. + # Re-thermalize the neutrals every 1000 steps + step = self.sim.extension.warpx.getistep(lev=0) + if step % 1000 != 10: + return + + if not hasattr(self, 'neutral_cont'): + self.neutral_cont = particle_containers.ParticleContainerWrapper( + self.neutrals.name + ) + + ux_arrays = self.neutral_cont.uxp + uy_arrays = self.neutral_cont.uyp + uz_arrays = self.neutral_cont.uzp + + vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) + for ii in range(len(ux_arrays)): + nps = len(ux_arrays[ii]) + ux_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uy_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uz_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + def _get_rho_ions(self): # deposit the ion density in rho_fp he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') @@ -389,11 +451,17 @@ def run_sim(self): '--pythonsolver', help='toggle whether to use the Python level solver', action='store_true' ) +parser.add_argument( + '--dsmc', help='toggle whether to use DSMC for ions in place of MCC', + action='store_true' +) args, left = parser.parse_known_args() sys.argv = sys.argv[:1]+left if args.n < 1 or args.n > 4: raise AttributeError('Test number must be an integer from 1 to 4.') -run = CapacitiveDischargeExample(n=args.n-1, test=args.test, pythonsolver=args.pythonsolver) +run = CapacitiveDischargeExample( + n=args.n-1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc +) run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py new file mode 100755 index 00000000000..a7a76be46ad --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# 2023 TAE Technologies + +import os +import sys + +import numpy as np + +sys.path.append('../../../../warpx/Regression/Checksum/') + +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] + +my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) + +ref_density = np.array([ + 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, + 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, + 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, + 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, + 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, + 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, + 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, + 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, + 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, + 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, + 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, + 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, + 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, + 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, + 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, + 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, + 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, + 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, + 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, + 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, + 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, + 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, + 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, + 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, + 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, + 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, + 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, + 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, + 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, + 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, + 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, + 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, + 1.27720853e+14 +]) + +density_data = np.load( 'ion_density_case_1.npy' ) +print(repr(density_data)) +assert np.allclose(density_data, ref_density) diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst index 737cb1fe677..0ba3b5382f2 100644 --- a/Examples/Physics_applications/laser_acceleration/README.rst +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -5,8 +5,9 @@ Laser-Wakefield Acceleration of Electrons This example shows how to model a laser-wakefield accelerator (LWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. -Laser-wakefield acceleration is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. -RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). +Laser-wakefield acceleration is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (laser diffraction, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round laser driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. For LWFA scenarios with long propagation lengths, use the :ref:`boosted frame method `. An example can be seen in the :ref:`PWFA example `. diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index 27ecc00117b..bdcfd7676a4 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -77,6 +77,7 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho +diag1.format = openpmd # Reduced Diagnostics warpx.reduced_diags_names = FP diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst index d5605dd93ef..d5775e93aa8 100644 --- a/Examples/Physics_applications/plasma_acceleration/README.rst +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -5,8 +5,9 @@ Beam-Driven Wakefield Acceleration of Electrons This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. -PWFA is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. -RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). +PWFA is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (structure of the space-charge fields, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. Additionally, to speed up computation, this example uses the :ref:`boosted frame method ` to effectively model long acceleration lengths. diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py new file mode 100755 index 00000000000..0f00010a505 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright 2023 David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. +import os +import re +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +if re.match('SemiImplicitPicard_1d', fn): + tolerance_rel = 2.5e-5 +elif re.match('ImplicitPicard_1d', fn): + # This case should have near machine precision conservation of energy + tolerance_rel = 1.e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel}") + +assert( max_delta_E < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d new file mode 100644 index 00000000000..50d28a2db75 --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = implicit_picard +algo.max_picard_iterations = 31 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit new file mode 100644 index 00000000000..2271a0bb1bc --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = semi_implicit_picard +algo.max_picard_iterations = 5 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz index 050118d036f..2e22ca299ea 100644 --- a/Examples/Tests/LoadExternalField/inputs_rz +++ b/Examples/Tests/LoadExternalField/inputs_rz @@ -46,7 +46,7 @@ algo.particle_shape = 1 ################################# ############ PLASMA ############# ################################# -particles.species_names = proton #electron +particles.species_names = proton proton.injection_style = "SingleParticle" proton.single_particle_pos = 0.0 0.2 2.5 proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 @@ -55,14 +55,11 @@ proton.do_not_deposit = 1 proton.mass = m_p proton.charge = q_e -#electron.injection_style = "SingleParticle" -#electron.single_particle_pos = 0.0 0.2 2.5 -#electron.single_particle_u = 0.0 0.0 0.0 -#electron.single_particle_weight = 1.0 -#electron.mass = 1.0 -#electron.charge = -q_e*1.0e-20 - # Diagnostics -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 chk diag1.intervals = 300 diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c6c089f9807..c0b03f4a20b 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,6 +21,7 @@ import numpy as np import openpmd_api as io +from openpmd_viewer import OpenPMDTimeSeries import yt yt.funcs.mylog.setLevel(0) @@ -48,9 +49,13 @@ Ez_openpmd = ds_openpmd.meshes['E']['z'].load_chunk() Ez_openpmd = Ez_openpmd.transpose() series.flush() - # Compare arrays to check consistency between new BTD formats (plotfile and openPMD) assert(np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol)) +# Check that particle random sub-selection has been applied +ts = OpenPMDTimeSeries('./diags/diag2/') +w, = ts.get_particle(['w'], species='beam', iteration=3) +assert (400 < len(w)) & (len(w) < 600) + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/boosted_diags/inputs_3d b/Examples/Tests/boosted_diags/inputs_3d index ba98558be47..1b6b3448f26 100644 --- a/Examples/Tests/boosted_diags/inputs_3d +++ b/Examples/Tests/boosted_diags/inputs_3d @@ -122,3 +122,4 @@ diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diag2.format = openpmd diag2.buffer_size = 32 diag2.openpmd_backend = h5 +diag2.beam.random_fraction = 0.5 diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index e5d61fc0c0a..95732b03e36 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -93,5 +93,13 @@ assert( error_rel < tolerance_rel ) +# Check that the user runtime component (if it exists) worked as expected +try: + orig_z = ad['electrons', 'particle_orig_z'].v + assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print('particle_orig_z has reasonable values') +except yt.utilities.exceptions.YTFieldNotFound: + pass # Some of the tested script to not have the quantity orig_z + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_rt b/Examples/Tests/ionization/inputs_2d_rt index 130eb1cc46a..f7035c567ac 100644 --- a/Examples/Tests/ionization/inputs_2d_rt +++ b/Examples/Tests/ionization/inputs_2d_rt @@ -36,6 +36,8 @@ ions.physical_element = N electrons.mass = m_e electrons.charge = -q_e electrons.injection_style = none +electrons.addRealAttributes = orig_z +electrons.attribute.orig_z(x,y,z,ux,uy,uz,t) = z lasers.names = laser1 laser1.profile = Gaussian diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst index a45cecb3c88..034ee5815f0 100644 --- a/Examples/Tests/ohm_solver_EM_modes/README.rst +++ b/Examples/Tests/ohm_solver_EM_modes/README.rst @@ -1,11 +1,56 @@ .. _examples-ohm-solver-em-modes: -Ohm Solver: Electromagnetic modes +Ohm solver: Electromagnetic modes ================================= In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the :math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -analyzed for mode excitations. +Fourier analyzed for Alfvén mode excitations. + +Run +--- + +The same input script can be used for 1d, 2d or 3d Cartesian simulations as well +as replicating either the parallel propagating or ion-Bernstein modes as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Parallel propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir z + + .. tab-item:: Perpendicular propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and compares the calculated spectrum +to the theoretical dispersions. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except in a region of thermal resonances as indicated on the plot below. @@ -14,32 +59,61 @@ in a region of thermal resonances as indicated on the plot below. :alt: Parallel EM modes in thermal ion plasma :width: 70% + Calculated Alvén waves spectrum with the theoretical dispersions overlaid. + Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. .. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular EM modes in thermal ion plasma + :alt: Perpendicular modes in thermal ion plasma :width: 50% -The input file for these examples and the corresponding analysis can be found at: + Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. + +Ohm solver: Cylindrical normal modes +==================================== + +A RZ-geometry example case for normal modes propagating along an applied magnetic +field in a cylinder is also available. The analytical solution for these modes +are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +Run +--- -* :download:`EM modes input ` -* :download:`Analysis script ` +The following script initializes a thermal plasma in a metallic cylinder with +periodic boundaries at the cylinder ends. -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or -ion-Bernstein modes as indicated below. +.. dropdown:: Script ``PICMI_inputs_rz.py`` - .. code-block:: bash + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py``. - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} +The example can be executed using: -A RZ-geometry example case for normal modes propagating along an applied magnetic field in a cylinder is also available. -The analytical solution for these modes are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. +.. code-block:: bash + + python3 PICMI_inputs_rz.py + +Analyze +------- + +After the simulation completes the following script can be used to analyze the +field evolution and extract the normal mode dispersion relation. It performs a +standard Fourier transform along the cylinder axis and a Hankel transform in the +radial direction. + +.. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. + +The following figure was produced with the above analysis script, showing excellent +agreement between the calculated and theoretical dispersion relations. .. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png :alt: Normal EM modes in a metallic cylinder :width: 90% -The input file for this example and corresponding analysis can be found at: - -* :download:`Cylindrical modes input ` -* :download:`Analysis script ` + Cylindrical normal mode dispersion comparing the calculated spectrum with the + theoretical one. diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst index 7e1fc21d935..dd4f94b4edf 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -1,25 +1,50 @@ .. _examples-ohm-solver-ion-landau-damping: -Ohm Solver: Ion Landau Damping +Ohm solver: Ion Landau Damping ============================== -Landau damping is a well known process in which electrostatic (acoustic) waves are damped by transferring energy to particles satisfying a resonance condition. -The process can be simulated by seeding a plasma with a specific acoustic mode (density perturbation) and tracking the strength of the mode as a function of time. -The figure below shows a set of such simulations with parameters matching those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. -The straight lines show the theoretical damping rate for the given temperature ratios. - -.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png - :alt: Ion Landau damping - :width: 70% +Landau damping is a well known process in which electrostatic (acoustic) waves are +damped by transferring energy to particles satisfying a resonance condition. +The process can be simulated by seeding a plasma with a specific acoustic mode +(density perturbation) and tracking the strength of the mode as a function of time. -The input file for these examples and the corresponding analysis can be found at: - -* :download:`Ion Landau damping input ` -* :download:`Analysis script ` +Run +--- The same input script can be used for 1d, 2d or 3d simulations and to sweep different temperature ratios. +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + .. code-block:: bash python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + +Analyze +------- + +The following script extracts the amplitude of the seeded mode as a function +of time and compares it to the theoretical damping rate. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py``. + +The figure below shows a set of such simulations with parameters matching those +described in section 4.5 of :cite:t:`ex-MUNOZ2018`. +The straight lines show the theoretical damping rate for the given temperature ratios. + +.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png + :alt: Ion Landau damping + :width: 70% + + Decay of seeded modes as a function of time for different electron-ion temperature ratios. + The theoretical damping of the given modes are shown in dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst index ae98787b799..59469cf4aa9 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -1,11 +1,57 @@ .. _examples-ohm-solver-ion-beam-instability: -Ohm Solver: Ion Beam R Instability +Ohm solver: Ion Beam R Instability ================================== In this example a low density ion beam interacts with a "core" plasma population which induces an instability. Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can -be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and +be accessed. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations as well as +replicating either the resonant or non-resonant condition as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --resonant + + .. tab-item:: Non-resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and outputs the figures shown below. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/analysis.py``. + +The figures below show the evolution of the y-component of the magnetic field as the beam and core plasma interact. .. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png @@ -16,6 +62,8 @@ core plasma interact. :alt: Non-resonant ion beam R instability :width: 70% + Evolution of :math:`B_y` for resonant (top) and non-resonant (bottom) conditions. + The growth rates of the strongest growing modes for the resonant case are compared to theory (dashed lines) in the figure below. @@ -23,14 +71,5 @@ to theory (dashed lines) in the figure below. :alt: Resonant ion beam R instability growth rates :width: 50% -The input file for these examples and the corresponding analysis can be found at: - -* :download:`Ion beam R instability input ` -* :download:`Analysis script ` - -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant -condition as indicated below. - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --resonant + Time series of the mode amplitudes for m = 4, 5, 6 from simulation. The + theoretical growth for these modes are also shown as dashed lines. diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst index c31f9915abd..943b5bd0248 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -4,32 +4,33 @@ Ohm Solver: Magnetic Reconnection ================================= Hybrid-PIC codes are often used to simulate magnetic reconnection in space plasmas. -An example of magnetic reconnection from a force-free sheet is provided, based on the simulation described in :cite:t:`ex-Le2016`. +An example of magnetic reconnection from a force-free sheet is provided, based on +the simulation described in :cite:t:`ex-Le2016`. Run --- -This example can be run as a **Python** script: ``python3 PICMI_inputs.py``. +The following **Python** script configures and launches the simulation. -.. literalinclude:: PICMI_inputs.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. +.. dropdown:: Script ``PICMI_inputs.py`` + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. -Analyze -------- - -We run the following script to analyze correctness: - -.. note:: +Running the full simulation should take about 4 hours if executed on 1 V100 GPU. +For `MPI-parallel `__ runs, prefix these lines with +``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. - This section is TODO. + .. code-block:: bash + python3 PICMI_inputs.py -Visualize ---------- +Analyze +------- -You can run the following script to visualize the B-field evolution over time: +The following script extracts the reconnection rate as a function of time and +animates the evolution of the magnetic field (as shown below). .. dropdown:: Script ``analysis.py`` @@ -41,4 +42,4 @@ You can run the following script to visualize the B-field evolution over time: :alt: Magnetic reconnection. :width: 70% - Magnetic reconnection. + Magnetic reconnection from a force-free sheet. diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py new file mode 100755 index 00000000000..b97b6e0eb5a --- /dev/null +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2023 Kale Weichman +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the particle scraping for the embedded boundary in RZ. +# Particles are initialized between r=0.15 and r=0.2 +# having a negative radial velocity. +# A cylindrical embedded surface is placed at r=0.1. +# Upon reaching the surface, particles should be removed. +# At the end of the simulation, i.e., at time step 37, +# there should be 512 particles left. +# This test checks that plot_filter_fucntion works with the boundary scraping diagnostic +# by making sure that the particles removed from only half the domain (z>0) have been recorded. + +# Possible errors: 0 +# tolerance: 0 +# Possible running time: < 1 s + +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +tolerance = 0 + +fn = sys.argv[1] +ds = yt.load( fn ) +ad = ds.all_data() +x = ad['electron', 'particle_position_x'].v + +error = len(x)-512 +print('error = ', error) +print('tolerance = ', tolerance) +assert(error==tolerance) + +# Check that all the removed particles are properly recorded +# by making sure that, at each iteration, the sum of the number of +# remaining particles and scraped particles is equal to half the +# original number of particles +# also check that no particles with z <= 0 have been scraped +ts_full = OpenPMDTimeSeries('./diags/diag2/') +ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') + +def n_remaining_particles( iteration ): + w, = ts_full.get_particle(['w'], iteration=iteration) + return len(w) +def n_scraped_particles( iteration ): + timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) + return (timestamp <= iteration).sum() +def n_scraped_z_leq_zero( iteration ): + z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) + return (z_pos <= 0).sum() +n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) +n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) +n_z_leq_zero = np.array([ n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations ]) +n_total = n_remaining[0] +assert np.all( 2*n_scraped+n_remaining == n_total) +assert np.all( n_z_leq_zero == 0) diff --git a/Examples/Tests/scraping/inputs_rz_filter b/Examples/Tests/scraping/inputs_rz_filter new file mode 100644 index 00000000000..0d67fb96f6c --- /dev/null +++ b/Examples/Tests/scraping/inputs_rz_filter @@ -0,0 +1,58 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd +diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Examples/analysis_default_openpmd_regression.py b/Examples/analysis_default_openpmd_regression.py new file mode 100755 index 00000000000..3aadc49ac51 --- /dev/null +++ b/Examples/analysis_default_openpmd_regression.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import re +import sys + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd', rtol=2.e-6) +else: + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd') diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 1c191e607f4..8f3c1dc5e2c 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL """Callback Locations -================== +------------------ These are the functions which allow installing user created functions so that they are called at various places along the time step. @@ -23,9 +23,6 @@ instance method as an argument. Note that if an instance method is used, an extra reference to the method's object is saved. -The install can be done using a decorator, which has the prefix ``callfrom``. See -example below. - Functions can be called at the following times: * ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) @@ -46,27 +43,37 @@ * ``particleinjection``: called when particle injection happens, after the position advance and before deposition is called, allowing a user defined particle distribution to be injected each time step -* ``appliedfields``: allows directly specifying any fields to be applied to the particles - during the advance -To use a decorator, the syntax is as follows. This will install the function -``myplots`` to be called after each step. +Example that calls the Python function ``myplots`` after each step: .. code-block:: python3 - @callfromafterstep + from pywarpx.callbacks import installcallback + def myplots(): - ppzx() + # do something here + + installcallback('afterstep', myplots) -This is equivalent to the following: + # run simulation + sim.step(nsteps=100) + +The install can also be done using a `Python decorator `__, which has the prefix ``callfrom``. +To use a decorator, the syntax is as follows. This will install the function ``myplots`` to be called after each step. +The above example is quivalent to the following: .. code-block:: python3 + from pywarpx.callbacks import callfromafterstep + + @callfromafterstep def myplots(): - ppzx() + # do something here - installcallback('afterstep', myplots) + # run simulation + sim.step(nsteps=100) """ + import copy import sys import time diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 168ad8508f6..7d7e150f111 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1159,10 +1159,21 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.Jy_external_function = Jy_external_function self.Jz_external_function = Jz_external_function + # Handle keyword arguments used in expressions + self.user_defined_kw = {} + for k in list(kw.keys()): + self.user_defined_kw[k] = kw[k] + del kw[k] + self.handle_init(kw) def initialize_inputs(self): + # Add the user defined keywords to my_constants + # The keywords are mangled if there is a conflicting variable already + # defined in my_constants with the same name but different value. + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) + self.grid.initialize_inputs() pywarpx.algo.maxwell_solver = self.method @@ -1172,17 +1183,21 @@ def initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho)', self.plasma_resistivity + 'plasma_resistivity(rho)', + pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) pywarpx.hybridpicmodel.substeps = self.substeps pywarpx.hybridpicmodel.__setattr__( - 'Jx_external_grid_function(x,y,z,t)', self.Jx_external_function + 'Jx_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jx_external_function, self.mangle_dict) ) pywarpx.hybridpicmodel.__setattr__( - 'Jy_external_grid_function(x,y,z,t)', self.Jy_external_function + 'Jy_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jy_external_function, self.mangle_dict) ) pywarpx.hybridpicmodel.__setattr__( - 'Jz_external_grid_function(x,y,z,t)', self.Jz_external_function + 'Jz_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jz_external_function, self.mangle_dict) ) @@ -1530,6 +1545,49 @@ def initialize_inputs(self): collision.add_new_attr(process+'_'+key, val) +class DSMCCollisions(picmistandard.base._ClassWithInit): + """ + Custom class to handle setup of DSMC collisions in WarpX. If collision + initialization is added to picmistandard this can be changed to inherit + that functionality. + + Parameters + ---------- + name: string + Name of instance (used in the inputs file) + + species: species instance + The species involved in the collision + + scattering_processes: dictionary + The scattering process to use and any needed information + + ndt: integer, optional + The collisions will be applied every "ndt" steps. Must be 1 or larger. + """ + + def __init__(self, name, species, scattering_processes, ndt=None, **kw): + self.name = name + self.species = species + self.scattering_processes = scattering_processes + self.ndt = ndt + + self.handle_init(kw) + + def initialize_inputs(self): + collision = pywarpx.Collisions.newcollision(self.name) + collision.type = 'dsmc' + collision.species = [species.name for species in self.species] + collision.ndt = self.ndt + + collision.scattering_processes = self.scattering_processes.keys() + for process, kw in self.scattering_processes.items(): + for key, val in kw.items(): + if key == 'species': + val = val.name + collision.add_new_attr(process+'_'+key, val) + + class EmbeddedBoundary(picmistandard.base._ClassWithInit): """ Custom class to handle set up of embedded boundaries specific to WarpX. diff --git a/Python/setup.py b/Python/setup.py index 47028642cce..f82e6563a31 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '23.12', + version = '24.01', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 3110120b70a..01e4c687292 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552174367742e-08, - "Bt": 2380179.567792259, - "Bz": 2.4079075035536954e-08, + "Br": 1.8705552264208163e-08, + "Bt": 2380179.5677922587, + "Bz": 2.4079077116116535e-08, "Er": 497571119514841.25, - "Et": 7.048225282653208, - "Ez": 137058870936728.17, + "Et": 7.048225464720808, + "Ez": 137058870936728.28, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json new file mode 100644 index 00000000000..16c27055ab5 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -0,0 +1,96 @@ +{ + "lev=0": { + "Bx": 971135657171.612, + "By": 971078812454.5405, + "Bz": 20140193.235893946, + "Ex": 2.9111756162943966e+20, + "Ey": 2.9113725115697712e+20, + "Ez": 1.0213536367191107e+17, + "rho_beam1": 7.970337929028706e+16, + "rho_beam2": 7.969213804851568e+16, + "rho_ele1": 343600677331163.7, + "rho_ele2": 302746939366837.25, + "rho_pos1": 333855946230626.06, + "rho_pos2": 310879461124837.44 + }, + "beam1": { + "particle_opticalDepthQSR": 104909.59461909423, + "particle_position_x": 0.001500222221634118, + "particle_position_y": 0.0015002445303035634, + "particle_position_z": 0.0049656251227976015, + "particle_momentum_x": 6.205341799808723e-15, + "particle_momentum_y": 6.1592257603817594e-15, + "particle_momentum_z": 6.806886719670214e-12, + "particle_weight": 635949610.5135971 + }, + "beam2": { + "particle_opticalDepthQSR": 104164.848014815, + "particle_position_x": 0.0015001011957527532, + "particle_position_y": 0.001500139975740741, + "particle_position_z": 0.004965479176845744, + "particle_momentum_x": 6.200690794877584e-15, + "particle_momentum_y": 6.186048913459861e-15, + "particle_momentum_z": 6.7990490255176515e-12, + "particle_weight": 635863144.251134 + }, + "ele1": { + "particle_opticalDepthQSR": 435.73003117907257, + "particle_position_x": 4.882183530045367e-06, + "particle_position_y": 4.841391483672882e-06, + "particle_position_z": 1.8449175055560687e-05, + "particle_momentum_x": 5.6656608489971696e-18, + "particle_momentum_y": 5.724425295258085e-18, + "particle_momentum_z": 2.6277553331470036e-15, + "particle_weight": 2696555.9200674472 + }, + "ele2": { + "particle_opticalDepthQSR": 340.82229684726735, + "particle_position_x": 5.233059654483856e-06, + "particle_position_y": 4.781569085220371e-06, + "particle_position_z": 1.6293559425324337e-05, + "particle_momentum_x": 4.802611971470525e-18, + "particle_momentum_y": 4.3556825243407754e-18, + "particle_momentum_z": 2.41587659230925e-15, + "particle_weight": 2481137.860727036 + }, + "pho1": { + "particle_opticalDepthBW": 9894.64959724129, + "particle_position_x": 0.00014191369823397644, + "particle_position_y": 0.00014347545392717968, + "particle_position_z": 0.00047442826029322116, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 61063948.68610941 + }, + "pho2": { + "particle_opticalDepthBW": 10292.1955840901, + "particle_position_x": 0.00014731892710321073, + "particle_position_y": 0.00014515617182809124, + "particle_position_z": 0.00048756513452074315, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 62299636.622087024 + }, + "pos1": { + "particle_opticalDepthQSR": 387.7441392212553, + "particle_position_x": 5.1462880118803425e-06, + "particle_position_y": 5.2613832293016684e-06, + "particle_position_z": 1.7054223425917483e-05, + "particle_momentum_x": 4.6437665862693495e-18, + "particle_momentum_y": 4.761862836969051e-18, + "particle_momentum_z": 2.3776996599289627e-15, + "particle_weight": 2625121.7841375084 + }, + "pos2": { + "particle_opticalDepthQSR": 361.943365907597, + "particle_position_x": 4.969019565031149e-06, + "particle_position_y": 4.361394970806125e-06, + "particle_position_z": 1.7413304358612675e-05, + "particle_momentum_x": 5.6348322786528905e-18, + "particle_momentum_y": 4.8171439953214205e-18, + "particle_momentum_z": 2.1937254860708963e-15, + "particle_weight": 2529794.7740602638 + } +} diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json new file mode 100644 index 00000000000..cd4120002be --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3730.0029376363264, + "By": 1593.5906541698305, + "Bz": 0.0, + "Ex": 797541065253.7858, + "Ey": 981292393404.0359, + "Ez": 3528134993266.091, + "divE": 2.0829069134855788e+21, + "jx": 7.639291641209293e+17, + "jy": 1.4113587963237038e+18, + "jz": 1.3506587033985085e+18, + "rho": 18442449008.581665 + }, + "protons": { + "particle_momentum_x": 5.231105747759245e-19, + "particle_momentum_y": 5.367982834807453e-19, + "particle_momentum_z": 5.253213507906386e-19, + "particle_position_x": 0.00010628272743703996, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196379551301037e-20, + "particle_momentum_y": 1.2271443795645239e-20, + "particle_momentum_z": 1.2277752539495415e-20, + "particle_position_x": 0.00010649569055433632, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index ee19b84c7af..0efcdaeca3c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 3.535284585563231e-19, - "particle_momentum_y": 4.403094613346061e-19, - "particle_momentum_z": 5.658013779496569e-17, - "particle_position_x": 0.008317876520240174, - "particle_position_y": 1.1704335094514386, - "particle_weight": 62415090744.60765 + "lev=0": { + "Bx": 4818955.480792835, + "By": 1752.8025402207227, + "Bz": 14516.21278267981, + "Ex": 2366115496505.249, + "Ey": 1446112025634143.0, + "Ez": 21864189507353.19, + "jx": 1996366349839211.5, + "jy": 5.312583827165288e+16, + "jz": 2.049135262445976e+16, + "rho": 68443961.71835628 }, "electrons": { - "particle_momentum_x": 2.2135959939611405e-23, - "particle_momentum_y": 2.822519730011994e-22, - "particle_momentum_z": 5.260625039372931e-22, - "particle_position_x": 0.010800577787577741, - "particle_position_y": 0.21115060628258137, + "particle_momentum_x": 2.2135945391227107e-23, + "particle_momentum_y": 2.8224559499572622e-22, + "particle_momentum_z": 5.260626010211241e-22, + "particle_position_x": 0.010800577787628053, + "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472008953412e-23, - "particle_momentum_y": 4.449200926395666e-22, - "particle_momentum_z": 5.768167708374496e-22, - "particle_position_x": 0.010800001678510793, - "particle_position_y": 0.21114947608115497, + "particle_momentum_x": 6.248472277235318e-23, + "particle_momentum_y": 4.449097689427615e-22, + "particle_momentum_z": 5.768168724780326e-22, + "particle_position_x": 0.010800001678510512, + "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, - "lev=0": { - "Bx": 4818965.813977506, - "By": 1752.781451346485, - "Bz": 14516.29947347909, - "Ex": 2366115950699.717, - "Ey": 1446115205306590.0, - "Ez": 21864183756771.12, - "jx": 1996366451911408.0, - "jy": 5.312642483130767e+16, - "jz": 2.049134468340688e+16, - "rho": 68443935.06252655 + "beam": { + "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_y": 4.363391839372122e-19, + "particle_momentum_z": 5.658606416951657e-17, + "particle_position_x": 0.008314723025211447, + "particle_position_y": 1.1704335743854242, + "particle_weight": 62415090744.60765 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 0c6f5242b87..1602058dbdc 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,55 +1,55 @@ { "lev=0": { - "Br": 104965.61239547668, - "Br_0_real": 0.27473110662919303, - "Br_1_imag": 104.10451752777047, + "Br": 104965.61239547684, + "Br_0_real": 0.2747311066289638, + "Br_1_imag": 104.10451752777048, "Br_1_real": 104965.62478916143, - "Bt": 1296.2723279668432, - "Btheta_0_real": 1297.3296772773876, + "Bt": 1296.2723277779253, + "Btheta_0_real": 1297.3296772773874, "Btheta_1_imag": 105725.25398122355, - "Btheta_1_real": 141.255481019691, - "Bz": 5076.744762384442, - "Bz_0_real": 0.4603819957305941, - "Bz_1_imag": 1.007360812967319, + "Btheta_1_real": 141.2554810196911, + "Bz": 5076.74476238461, + "Bz_0_real": 0.4603819957306249, + "Bz_1_imag": 1.0073608129672305, "Bz_1_real": 5076.632351216658, - "Er": 273570494222.5625, + "Er": 273570493775.55588, "Er_0_real": 271974091013.7082, "Er_1_imag": 39530787242966.72, - "Er_1_real": 42616886403.53066, - "Et": 39016542462571.72, - "Etheta_0_real": 112249482.93967965, - "Etheta_1_imag": 33602799006.8748, + "Er_1_real": 42616886403.53069, + "Et": 39016542462571.68, + "Etheta_0_real": 112249482.93975669, + "Etheta_1_imag": 33602799006.874825, "Etheta_1_real": 39016517454881.91, - "Ez": 511653064866.5733, + "Ez": 511653064863.048, "Ez_0_real": 496845125310.57947, - "Ez_1_imag": 1245709520854.4875, - "Ez_1_real": 24849976994.356285, - "Jr_0_real": 1264417193501.7915, + "Ez_1_imag": 1245709520854.488, + "Ez_1_real": 24849976994.356266, + "Jr_0_real": 1264417193503.146, "Jr_1_imag": 2.3356633334993878e+17, - "Jr_1_real": 2272922066062.2944, + "Jr_1_real": 2272922066062.586, "Jtheta_0_real": 475304056513.7001, - "Jtheta_1_imag": 1028929701334.7467, + "Jtheta_1_imag": 1028929701334.8286, "Jtheta_1_real": 2.1766379427377802e+17, "Jz_0_real": 1832468408628522.8, "Jz_1_imag": 556484600934089.9, "Jz_1_real": 602703622358902.4, - "jr": 1749994884866.3667, - "jt": 2.176637940885753e+17, - "jz": 1954078685186198.5, - "rho": 39314210.88334631, + "jr": 1749985661974.403, + "jt": 2.1766379408857264e+17, + "jz": 1954078684852864.2, + "rho": 39314210.931097575, "rho_0_real": 38889615.839967206, - "rho_1_imag": 21546499.619336687, - "rho_1_real": 2012888.5658883716 + "rho_1_imag": 21546499.619336277, + "rho_1_real": 2012888.5658883736 }, "electrons": { - "particle_momentum_x": 1.3203417227476259e-24, + "particle_momentum_x": 1.3203417227476271e-24, "particle_momentum_y": 4.0070625287284664e-22, - "particle_momentum_z": 1.2493391415020164e-23, + "particle_momentum_z": 1.2493391415020162e-23, "particle_orig_x": 0.026508328457558912, "particle_orig_z": 0.04789125000000001, "particle_position_x": 0.04160250006680718, "particle_position_y": 0.04789125046517409, - "particle_theta": 7325.121266775259, + "particle_theta": 7325.160426984388, "particle_weight": 813672305.532158 }, "beam": { @@ -61,4 +61,4 @@ "particle_theta": 151.40797316586125, "particle_weight": 6241509.074460764 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index 6b30064365c..dd224516c5c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497087553.4743837, - "By": 252364344.72213668, - "Bz": 16986109.62608196, - "Ex": 7.304419665394173e+16, - "Ey": 1.4583459400150573e+17, - "Ez": 2.261309965998215e+16, - "jx": 8.273582439836952e+17, - "jy": 2.1044181577417728e+18, - "jz": 2.84208498868846e+19, - "rho": 91637594416.50476 + "Bx": 497756265.44724107, + "By": 252197580.117894, + "Bz": 16988475.5444833, + "Ex": 7.299911104263803e+16, + "Ey": 1.460116488178734e+17, + "Ez": 2.26033100286114e+16, + "jx": 8.27644503757345e+17, + "jy": 2.1062078409959675e+18, + "jz": 2.8491727096438305e+19, + "rho": 91863764144.41415 }, "electrons": { - "particle_momentum_x": 5.922677819206403e-20, - "particle_momentum_y": 2.7778096010261295e-19, - "particle_momentum_z": 4.183846407463124e-19, - "particle_position_x": 0.0025865980636989934, - "particle_position_y": 0.002652673793955211, - "particle_position_z": 0.18411922027879662, - "particle_weight": 1731649095408.5505 + "particle_momentum_x": 5.76165226700654e-20, + "particle_momentum_y": 2.7567389504898156e-19, + "particle_momentum_z": 4.134562048099117e-19, + "particle_position_x": 0.0025269863605945427, + "particle_position_y": 0.0024538321295153346, + "particle_position_z": 0.17818421763751244, + "particle_weight": 1675789447169.5652 }, "beam": { - "particle_momentum_x": 1.1925686969448298e-17, - "particle_momentum_y": 2.570810132551278e-17, - "particle_momentum_z": 5.156190278524037e-14, - "particle_position_x": 0.0005608477913384702, - "particle_position_y": 0.0008431191912437461, - "particle_position_z": 2.9800048756186395, + "particle_momentum_x": 1.1926313055043134e-17, + "particle_momentum_y": 2.7056205218547404e-17, + "particle_momentum_z": 5.1562131494813424e-14, + "particle_position_x": 0.000560821518739447, + "particle_position_y": 0.000800729816549036, + "particle_position_z": 2.9800048650570097, "particle_weight": 62415.090744607616 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 793d8c11cb3..30eef0cedb7 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -1,25 +1,25 @@ { + "lev=0": { + "Bx": 5863879.051452597, + "By": 2411.5124004699082, + "Bz": 116025.40178726945, + "Ex": 6267725953308.365, + "Ey": 1670763222566621.8, + "Ez": 104345977762699.77, + "jx": 555688154318027.4, + "jy": 1595897074125252.8, + "jz": 1045267363178542.9, + "rho": 2205684400.3678846 + }, "electrons": { "particle_initialenergy": 0.0, - "particle_momentum_x": 1.7921229236407606e-20, - "particle_momentum_y": 7.225825184653885e-20, - "particle_momentum_z": 4.231731488281653e-20, - "particle_position_x": 0.7139122621535502, - "particle_position_y": 0.7150340889556129, - "particle_position_z": 1.3175770602802896, "particle_regionofinterest": 1936.0, + "particle_position_x": 0.7139122620952513, + "particle_position_y": 0.7150340902640349, + "particle_position_z": 1.317577060220835, + "particle_momentum_x": 1.7921232385614662e-20, + "particle_momentum_y": 7.22582607277354e-20, + "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5863879.02030842, - "By": 2411.501904737579, - "Bz": 116025.42462998998, - "Ex": 6267728094111.663, - "Ey": 1670763233105822.0, - "Ez": 104345989831222.4, - "jx": 555687912108453.8, - "jy": 1595896363359136.0, - "jz": 1045267552192496.5, - "rho": 2205684400.307701 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index d98aabd84e9..242cf00f0c3 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { "Br": 100278645.72225758, - "Bz": 2509008.1146699777, - "Er": 3.258880159474263, + "Bz": 2509008.1146699786, + "Er": 3.263343388037509, "Et": 3.0045297157982412e+16, - "Ez": 0.24010979707502123, - "jr": 50.13668665903749, - "jt": 4.305139762894391e+17, + "Ez": 0.2422526411295923, + "jr": 49.67802097760541, + "jt": 4.3051397628943917e+17, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json index 36b2bc66aed..1678e794683 100644 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json @@ -1,34 +1,33 @@ { - "electrons": { - "particle_momentum_x": 3.856058278845215e-19, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.633111056849423e-18, - "particle_position_x": 0.008155341094488198, - "particle_position_y": 0.03087362289643631, - "particle_weight": 2.6462205036771795e+17 - }, - "hydrogen": { - "particle_momentum_x": 2.237944099185357e-18, - "particle_momentum_z": 1.0744000122946915e-18, - "particle_orig_x": 0.007768349609375002, - "particle_orig_z": 0.035127407226562504, - "particle_position_x": 0.007767975241766832, - "particle_position_y": 0.03512562609400349, - "particle_weight": 2.68584931046923e+17 - }, "lev=0": { "Bx": 0.0, - "By": 11399442.711372176, + "By": 11411047.898351602, "Bz": 0.0, - "Ex": 2033057399920464.5, + "Ex": 2031641076084948.5, "Ey": 0.0, - "Ez": 339244862686685.9, - "jx": 1.6477362358927364e+19, + "Ez": 334777106874158.8, + "jx": 1.6602050255725978e+19, "jy": 0.0, - "jz": 9.633815033523364e+18, - "rho": 68121082972.17873, - "rho_electrons": 17448735668294.348, - "rho_hydrogen": 17441797017887.941 + "jz": 9.545451466990307e+18, + "rho": 67237998613.86053, + "rho_electrons": 17449705826002.385, + "rho_hydrogen": 17441792654743.146 + }, + "hydrogen": { + "particle_momentum_x": 2.2575255298569512e-18, + "particle_momentum_z": 1.0773391029868316e-18, + "particle_orig_x": 0.007763427734375002, + "particle_orig_z": 0.0351975439453125, + "particle_position_x": 0.007763034484095496, + "particle_position_y": 0.035195761508116416, + "particle_weight": 2.6792730619156992e+17 + }, + "electrons": { + "particle_momentum_x": 3.8463518636930147e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.6287398567676136e-18, + "particle_position_x": 0.008139313907538123, + "particle_position_y": 0.0308605164193468, + "particle_weight": 2.647983436428149e+17 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/PlasmaMirror.json index 9a686e78d56..861f42f2d74 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaMirror.json +++ b/Regression/Checksum/benchmarks_json/PlasmaMirror.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 2.000457748960032e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.446670640741138e-17, - "particle_position_x": 0.37688793289418476, - "particle_position_y": 0.1714047288635171, - "particle_weight": 2.904173860599889e+18 - }, - "ions": { - "particle_momentum_x": 2.805690782833416e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 9.605283484426823e-17, - "particle_position_x": 0.3828171295926821, - "particle_position_y": 0.17237242725257168, - "particle_weight": 2.9212132379167017e+18 - }, "lev=0": { "Bx": 0.0, - "By": 75294856.83091721, + "By": 75293687.71417463, "Bz": 0.0, - "Ex": 2.081697252453305e+16, + "Ex": 2.0817302986029584e+16, "Ey": 0.0, - "Ez": 2.350511387513211e+16, - "jx": 5.496779405360711e+19, + "Ez": 2.350518662171605e+16, + "jx": 5.496864993586538e+19, "jy": 0.0, - "jz": 7.159348923050661e+19 + "jz": 7.159302467888838e+19 + }, + "ions": { + "particle_momentum_x": 2.8056907319288714e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 9.605283524169195e-17, + "particle_position_x": 0.382817129592504, + "particle_position_y": 0.1723724272525192, + "particle_weight": 2.9212132379167017e+18 + }, + "electrons": { + "particle_momentum_x": 2.000448978774197e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.446668081246458e-17, + "particle_position_x": 0.376887964713013, + "particle_position_y": 0.17140472970668535, + "particle_weight": 2.9041738605998894e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 0b0355b7cb6..08969db023e 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 5866866.518334419, + "By": 11175.108378013305, + "Bz": 116026.79919117832, + "Ex": 8178060925462.576, + "Ey": 1671615340558021.2, + "Ez": 106548534464553.72, + "jx": 555903248104584.56, + "jy": 1595980426561245.2, + "jz": 1366292265497407.5, + "rho": 2206755250.226452 + }, "beam": { - "particle_momentum_x": 4.707586336874016e-20, - "particle_momentum_y": 4.4850722108112576e-20, - "particle_momentum_z": 1.36054441043288e-17, - "particle_position_x": 4.058764306495361e-05, - "particle_position_y": 3.7888695722549883e-05, - "particle_position_z": 0.00019656701118308398, + "particle_momentum_x": 4.7075864374777216e-20, + "particle_momentum_y": 4.4875025179072e-20, + "particle_momentum_z": 1.3605444179112827e-17, + "particle_position_x": 4.058764328440382e-05, + "particle_position_y": 3.788866110052658e-05, + "particle_position_z": 0.00019656700944015084, "particle_weight": 6241509.074460764 }, "electrons": { - "particle_momentum_x": 1.7921232203945004e-20, - "particle_momentum_y": 7.225819894813053e-20, - "particle_momentum_z": 4.231725460173154e-20, - "particle_position_x": 0.7139122621161993, - "particle_position_y": 0.7150340887578637, - "particle_position_z": 1.3175770600690966, + "particle_momentum_x": 1.7921232210868553e-20, + "particle_momentum_y": 7.225819896136567e-20, + "particle_momentum_z": 4.2317254599358777e-20, + "particle_position_x": 0.713912262116188, + "particle_position_y": 0.7150340887578024, + "particle_position_z": 1.31757706006908, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5866866.85492377, - "By": 11177.920546471447, - "Bz": 116026.93444649166, - "Ex": 8178548880638.266, - "Ey": 1671614207789070.8, - "Ez": 106548168484665.61, - "jx": 555903247963958.4, - "jy": 1595974150308405.2, - "jz": 1366292284444382.5, - "rho": 2206755250.226321 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json index d4c8d2b0c4e..720148888e2 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json @@ -1,45 +1,54 @@ { "lev=0": { - "Br": 142258.01161290862, - "Br_0_real": 0.36356680791862006, - "Br_1_imag": 115.41915541351702, + "Br": 142258.01161290894, + "Br_0_real": 0.36356680791855334, + "Br_1_imag": 115.41915541351706, "Br_1_real": 142258.0180774376, - "Bt": 1301.5691011565148, + "Bt": 1301.5691003159159, "Btheta_0_real": 1299.8813681794945, "Btheta_1_imag": 143318.04184449295, "Btheta_1_real": 155.37799428725944, - "Bz": 5993.6411733668865, - "Bz_0_real": 0.47374186122724626, - "Bz_1_imag": 1.140891753831192, - "Bz_1_real": 5993.529100973487, - "Er": 278531478924.55994, + "Bz": 5993.641173367065, + "Bz_0_real": 0.4737418612272665, + "Bz_1_imag": 1.1408917538312353, + "Bz_1_real": 5993.529100973486, + "Er": 278531478379.8872, "Er_0_real": 276179480674.09875, - "Er_1_imag": 47911368149173.87, - "Er_1_real": 46900731766.21943, - "Et": 47328876349791.84, - "Etheta_0_real": 135868018.29309198, - "Etheta_1_imag": 36802947213.299355, + "Er_1_imag": 47911368149173.86, + "Er_1_real": 46900731766.219345, + "Et": 47328876349791.85, + "Etheta_0_real": 135868018.2930996, + "Etheta_1_imag": 36802947213.299324, "Etheta_1_real": 47328835574237.43, - "Ez": 514006688163.9632, + "Ez": 514006688162.3537, "Ez_0_real": 499008057521.7594, - "Ez_1_imag": 1565161964565.9727, - "Ez_1_real": 28898941539.846138, - "Jr_0_real": 1458783813352.3413, + "Ez_1_imag": 1565161964565.9724, + "Ez_1_real": 28898941539.846146, + "Jr_0_real": 1458783813352.7048, "Jr_1_imag": 2.335663323286376e+17, - "Jr_1_real": 2725720908169.8604, - "Jtheta_0_real": 499386783042.54987, - "Jtheta_1_imag": 1179203035377.7158, + "Jr_1_real": 2725720908168.6816, + "Jtheta_0_real": 499386783042.5499, + "Jtheta_1_imag": 1179203035377.2214, "Jtheta_1_real": 2.1766371791828432e+17, "Jz_0_real": 1832469778455967.8, "Jz_1_imag": 621923937700173.0, "Jz_1_real": 660910016274555.4, - "jr": 2108653492279.1746, - "jt": 2.1766371088503258e+17, - "jz": 1954236547127886.2, - "rho": 39480728.028152466, + "jr": 2108641839684.7007, + "jt": 2.1766371088502803e+17, + "jz": 1954236547059862.2, + "rho": 39480728.08547403, "rho_0_real": 39055923.162689775, - "rho_1_imag": 21660762.696674928, - "rho_1_real": 2131498.588887921 + "rho_1_imag": 21660762.696674515, + "rho_1_real": 2131498.5888879234 + }, + "electrons": { + "particle_momentum_x": 1.237777638089911e-24, + "particle_momentum_y": 3.9465600403988994e-22, + "particle_momentum_z": 1.0097949593654349e-23, + "particle_position_x": 0.04160250006989259, + "particle_position_y": 0.047891250488567585, + "particle_theta": 7325.162679147719, + "particle_weight": 813672305.532158 }, "beam": { "particle_momentum_x": 3.8798818966213747e-20, @@ -49,14 +58,5 @@ "particle_position_y": 0.0026764363296859625, "particle_theta": 151.40797595010355, "particle_weight": 6241509.074460764 - }, - "electrons": { - "particle_momentum_x": 1.2377776380899228e-24, - "particle_momentum_y": 3.9465600403988994e-22, - "particle_momentum_z": 1.009794959365435e-23, - "particle_position_x": 0.04160250006989259, - "particle_position_y": 0.047891250488567585, - "particle_theta": 7325.119014611929, - "particle_weight": 813672305.532158 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json new file mode 100644 index 00000000000..0b38f78e6f9 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "rho_electrons": 0.00443743609125863, + "rho_he_ions": 0.005198801518328451 + }, + "neutrals": { + "particle_momentum_x": 1.404700281648976e-19, + "particle_momentum_y": 1.4028127127618884e-19, + "particle_momentum_z": 1.4090901433394346e-19, + "particle_position_x": 1120.7727446759352, + "particle_weight": 6.4588e+19 + }, + "he_ions": { + "particle_momentum_x": 2.770386771117138e-19, + "particle_momentum_y": 2.7568040242914223e-19, + "particle_momentum_z": 3.619756966185903e-19, + "particle_position_x": 2200.683185473434, + "particle_weight": 17185500000000.002 + }, + "electrons": { + "particle_momentum_x": 3.5129762363657864e-20, + "particle_momentum_y": 3.5431134517510143e-20, + "particle_momentum_z": 1.2592093336142964e-19, + "particle_position_x": 2142.0662480700303, + "particle_weight": 14593699218750.002 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 35aa2c07a60..31f426aa362 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554466, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3483401471475687e-07 + }, "electrons": { - "particle_momentum_x": 4.410992240916769e-18, + "particle_momentum_x": 4.4206237143449475e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.637306716013194e-18, - "particle_position_x": 0.1095060155214851, - "particle_position_y": 0.6411875726789248, - "particle_weight": 3.44453125e-10 + "particle_momentum_z": 2.6361297302081026e-18, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324019342538e-18, + "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887006212893e-23, - "particle_position_x": 0.03199998810579664, - "particle_position_y": 0.12800000462171646, + "particle_momentum_z": 3.644887053263054e-23, + "particle_position_x": 0.03200001189420337, + "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.355517269126987e-07 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/RefinedInjection.json b/Regression/Checksum/benchmarks_json/RefinedInjection.json index 8be545f3887..43cc8e3072c 100644 --- a/Regression/Checksum/benchmarks_json/RefinedInjection.json +++ b/Regression/Checksum/benchmarks_json/RefinedInjection.json @@ -1,11 +1,27 @@ { - "beam": { - "particle_momentum_x": 4.5603842543853726e-20, - "particle_momentum_y": 4.191542263916011e-20, - "particle_momentum_z": 1.363947302075425e-17, - "particle_position_x": 4.672031050423284e-05, - "particle_position_y": 0.00024387147640533396, - "particle_weight": 12483018148921.525 + "lev=0": { + "Bx": 26338425.87401078, + "By": 160257.72293376247, + "Bz": 694883.1248007242, + "Ex": 56513906294889.77, + "Ey": 7806060318610865.0, + "Ez": 65816740142650.89, + "jx": 6873283794395257.0, + "jy": 4286847636945161.5, + "jz": 7506503405703726.0, + "rho": 278060758.9118884 + }, + "lev=1": { + "Bx": 41971351.93872842, + "By": 347834.73215718823, + "Bz": 1093501.9056767717, + "Ex": 136247073074384.92, + "Ey": 1.2435868091440666e+16, + "Ez": 92449081296414.34, + "jx": 1.6499002361434856e+16, + "jy": 1568934856430368.5, + "jz": 2.2190671920159212e+16, + "rho": 298322648.1883917 }, "electrons": { "particle_momentum_x": 4.599605609558194e-19, @@ -15,28 +31,12 @@ "particle_position_y": 0.3695766840020302, "particle_weight": 216500976562500.75 }, - "lev=0": { - "Bx": 26338374.3731115, - "By": 160257.72168459874, - "Bz": 694868.2285976049, - "Ex": 56513906233209.63, - "Ey": 7806074074131210.0, - "Ez": 65816739650512.98, - "jx": 6873283793354275.0, - "jy": 4287548168426209.0, - "jz": 7506499988368183.0, - "rho": 278060758.9118884 - }, - "lev=1": { - "Bx": 41971136.54330983, - "By": 347834.7278931723, - "Bz": 1093489.2683699469, - "Ex": 136247072743280.97, - "Ey": 1.2435931966910504e+16, - "Ez": 92449079327862.72, - "jx": 1.6499002357270928e+16, - "jy": 1571736982354559.0, - "jz": 2.219065825081704e+16, - "rho": 298322648.1883917 + "beam": { + "particle_momentum_x": 4.560382242970367e-20, + "particle_momentum_y": 4.2263074230751857e-20, + "particle_momentum_z": 1.3639472455787962e-17, + "particle_position_x": 4.672031008809431e-05, + "particle_position_y": 0.0002438714470005561, + "particle_weight": 12483018148921.525 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json new file mode 100644 index 00000000000..2c9859b037d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3559.0541122456157, + "By": 1685.942868827529, + "Bz": 0.0, + "Ex": 796541204346.5195, + "Ey": 961740397927.6577, + "Ez": 3528140764527.8877, + "divE": 2.0829159085076083e+21, + "jx": 7.683674095607745e+17, + "jy": 1.4132459141738875e+18, + "jz": 1.350650739310074e+18, + "rho": 18442528652.19583 + }, + "protons": { + "particle_momentum_x": 5.231109104020749e-19, + "particle_momentum_y": 5.367985047933474e-19, + "particle_momentum_z": 5.253213505843665e-19, + "particle_position_x": 0.00010628272743703473, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196357181461066e-20, + "particle_momentum_y": 1.2271903040162696e-20, + "particle_momentum_z": 1.2277743615209627e-20, + "particle_position_x": 0.0001064956905491333, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json index d62cd008670..8b03899369b 100644 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 6.875024052604206e-19, - "particle_momentum_y": 4.375444487966637e-19, - "particle_momentum_z": 6.432593176909762e-18, - "particle_position_x": 0.0012933598223665212, - "particle_position_y": 0.35872101659459354, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 7.058217306102507e-19, - "particle_momentum_y": 2.2042314720139012e-18, - "particle_momentum_z": 2.5305214299582585e-16, - "particle_position_x": 1.5006580327952221, - "particle_position_y": 16.454388306646475, - "particle_weight": 1.234867020725368e+18 + "lev=0": { + "Bx": 1118808.3686978193, + "By": 3248970.5506422943, + "Bz": 280612.7921641442, + "Ex": 975536649649286.1, + "Ey": 402861835403418.1, + "Ez": 159049265640492.28, + "jx": 2.9996888133195436e+16, + "jy": 8.866654944519546e+16, + "jz": 3.164008885453435e+17, + "rho": 1059988299.6088305 }, "ions": { - "particle_momentum_x": 1.6150569264030943e-18, - "particle_momentum_y": 2.2334239793537862e-18, - "particle_momentum_z": 4.279249530683602e-13, - "particle_position_x": 1.4883816864868449, - "particle_position_y": 16.452386504130835, + "particle_momentum_x": 1.6150513873065298e-18, + "particle_momentum_y": 2.233426695677123e-18, + "particle_momentum_z": 4.279249529993671e-13, + "particle_position_x": 1.4883816864183497, + "particle_position_y": 16.452386504127254, "particle_weight": 1.234867369440658e+18 }, - "lev=0": { - "Bx": 1118823.5061574595, - "By": 3248957.084212374, - "Bz": 280624.78102759377, - "Ex": 975532719061463.4, - "Ey": 402851207946673.1, - "Ez": 159049601047523.06, - "jx": 2.9997052529118484e+16, - "jy": 8.866616130905646e+16, - "jz": 3.163974567840701e+17, - "rho": 1059977703.1166165 + "electrons": { + "particle_momentum_x": 7.058167362825288e-19, + "particle_momentum_y": 2.204239326446281e-18, + "particle_momentum_z": 2.530521998715408e-16, + "particle_position_x": 1.5006581263609764, + "particle_position_y": 16.454388313398017, + "particle_weight": 1.234867020725368e+18 + }, + "beam": { + "particle_momentum_x": 6.869222298759882e-19, + "particle_momentum_y": 4.374719809060106e-19, + "particle_momentum_z": 6.4523206583503136e-18, + "particle_position_x": 0.001290816359726098, + "particle_position_y": 0.3586691102823157, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json index cfd4bc83a85..dd56f8170a9 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 7.005046981277183e-19, - "particle_momentum_y": 4.375209352729912e-19, - "particle_momentum_z": 6.175459223479959e-18, - "particle_position_x": 0.001602571749982411, - "particle_position_y": 0.35897924403061005, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 6.240989852249656e-19, - "particle_momentum_y": 1.5790301798509742e-18, - "particle_momentum_z": 2.5064352637575283e-16, - "particle_position_x": 1.501413662801212, - "particle_position_y": 16.523781706919216, - "particle_weight": 1.2372401466086835e+18 + "lev=0": { + "Bx": 1086729.9718613266, + "By": 2886554.482275311, + "Bz": 264259.55093734514, + "Ex": 867387781289915.2, + "Ey": 392666724461952.5, + "Ez": 146897592531660.03, + "jx": 2.702866174672266e+16, + "jy": 8.615938361747776e+16, + "jz": 2.7329155817806224e+17, + "rho": 915945723.7934376 }, "ions": { - "particle_momentum_x": 1.4394968631660078e-18, - "particle_momentum_y": 1.5967458174525868e-18, - "particle_momentum_z": 4.2873406586841774e-13, - "particle_position_x": 1.4911814217840753, - "particle_position_y": 16.52196497877435, + "particle_momentum_x": 1.4394902513923003e-18, + "particle_momentum_y": 1.5967629157922875e-18, + "particle_momentum_z": 4.287340658051679e-13, + "particle_position_x": 1.4911814217142487, + "particle_position_y": 16.521964978771, "particle_weight": 1.2372405194129536e+18 }, - "lev=0": { - "Bx": 1086731.2285090145, - "By": 2886537.258822082, - "Bz": 264280.80501631316, - "Ex": 867382744066315.0, - "Ey": 392669698675093.25, - "Ez": 146897949570681.72, - "jx": 2.7028852724044516e+16, - "jy": 8.615686606708202e+16, - "jz": 2.7328720244417827e+17, - "rho": 915931431.8889401 + "electrons": { + "particle_momentum_x": 6.240933687389075e-19, + "particle_momentum_y": 1.5790611427694247e-18, + "particle_momentum_z": 2.5064357834741096e-16, + "particle_position_x": 1.501413766926399, + "particle_position_y": 16.523781713952324, + "particle_weight": 1.2372401466086835e+18 + }, + "beam": { + "particle_momentum_x": 7.000932845220306e-19, + "particle_momentum_y": 4.374936866729326e-19, + "particle_momentum_z": 6.194468548032543e-18, + "particle_position_x": 0.0016030835496557787, + "particle_position_y": 0.3589262705964349, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index cb02eeba5b0..35b2db84a1a 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 18263123.342891, + "Bz": 0.0, + "Ex": 5472992180428804.0, + "Ey": 0.0, + "Ez": 922.5589707939612, + "jx": 12440856508004.96, + "jy": 0.0, + "jz": 78616.0000011086 + }, "electrons": { - "particle_momentum_x": 2.137334032699983e-17, + "particle_momentum_x": 2.1386770170623736e-17, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.729179869336611e-17, - "particle_position_x": 0.11036860836114047, - "particle_position_y": 1.7121959960266107, - "particle_weight": 3.075501431906104e-09 + "particle_momentum_z": 1.7287241262743654e-17, + "particle_position_x": 0.11064981928849067, + "particle_position_y": 1.7121826057017473, + "particle_weight": 3.0755014319061045e-09 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.63061972838759e-18, + "particle_momentum_x": 3.630619728387593e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, - "particle_position_x": 0.021440031727258925, + "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 - }, - "lev=0": { - "Bx": 0.0, - "By": 18263123.342890996, - "Bz": 0.0, - "Ex": 5472992180428804.0, - "Ey": 0.0, - "Ez": 922.6036749671132, - "jx": 12440856508004.96, - "jy": 0.0, - "jz": 78616.00000110848 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 3fafb968f29..82984141b59 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -1,30 +1,31 @@ { - "electrons": { - "particle_momentum_x": 4.407898469197755e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.642991174223682e-18, - "particle_position_x": 0.1095015206652257, - "particle_position_y": 0.6413864600981052, - "particle_weight": 3.443203125e-10 + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554467, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3575651149270143e-07 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324005206226e-18, + "particle_momentum_x": 1.7613240052056494e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.618696690270335e-23, - "particle_position_x": 0.03199998819289686, + "particle_momentum_z": 3.6186967375178554e-23, + "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.346772722412443e-07 + "electrons": { + "particle_momentum_x": 4.428135211584547e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6627518442038486e-18, + "particle_orig_z": 0.43043207622230534, + "particle_position_x": 0.11023346490802505, + "particle_position_y": 0.6417814148540429, + "particle_weight": 3.44453125e-10 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart.json b/Regression/Checksum/benchmarks_json/restart.json index 9ed02d0faad..d8aa0788997 100644 --- a/Regression/Checksum/benchmarks_json/restart.json +++ b/Regression/Checksum/benchmarks_json/restart.json @@ -1,12 +1,15 @@ { - "beam": { - "particle_momentum_x": 4.178482505909375e-19, - "particle_momentum_y": 4.56492260137707e-19, - "particle_momentum_z": 2.733972888170628e-17, - "particle_position_x": 0.0003995213395426269, - "particle_position_y": 0.0004148795632360405, - "particle_position_z": 1.9019426942919677, - "particle_weight": 3120754537.230381 + "lev=0": { + "Bx": 115361.74185283793, + "By": 242516.32397638055, + "Bz": 62078.1602361236, + "Ex": 119701543932824.69, + "Ey": 20420953176049.12, + "Ez": 53561498853753.336, + "jx": 2.1550943713704252e+16, + "jy": 328452566832977.2, + "jz": 4525238578330174.0, + "rho": 22112877.392750103 }, "driver": { "particle_momentum_x": 4.700436405078562e+21, @@ -26,34 +29,31 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 115361.85363382133, - "By": 242516.41036288123, - "Bz": 62078.12299476949, - "Ex": 119701563770527.33, - "Ey": 20420746160110.2, - "Ez": 53561518084714.88, - "jx": 2.155094272131813e+16, - "jy": 328438528912277.4, - "jz": 4525204093100645.0, - "rho": 22113102.212642767 + "beam": { + "particle_momentum_x": 4.178482505909375e-19, + "particle_momentum_y": 4.56492260137707e-19, + "particle_momentum_z": 2.733972888170628e-17, + "particle_position_x": 0.0003995213395426269, + "particle_position_y": 0.0004148795632360405, + "particle_position_z": 1.9019426942919677, + "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 2.6421618201433585e-19, - "particle_momentum_y": 1.3141433018232061e-20, - "particle_momentum_z": 2.6326692402728787e-17, - "particle_position_x": 0.49916042906025937, - "particle_position_y": 0.4991834641855549, - "particle_position_z": 0.45626372602154797, + "plasma_p": { + "particle_momentum_x": 2.6392309174575904e-19, + "particle_momentum_y": 5.301543151656233e-21, + "particle_momentum_z": 4.829600619848831e-14, + "particle_position_x": 0.4991250033821341, + "particle_position_y": 0.49912499879083855, + "particle_position_z": 0.4563845472726038, "particle_weight": 33067341227104.594 }, - "plasma_p": { - "particle_momentum_x": 2.639231047445633e-19, - "particle_momentum_y": 5.302522423284582e-21, - "particle_momentum_z": 4.829600619851657e-14, - "particle_position_x": 0.4991250033821394, - "particle_position_y": 0.49912499879083844, - "particle_position_z": 0.45638454727260425, + "plasma_e": { + "particle_momentum_x": 2.6421607111722265e-19, + "particle_momentum_y": 1.3141424232991868e-20, + "particle_momentum_z": 2.6326692443085376e-17, + "particle_position_x": 0.49916042919135184, + "particle_position_y": 0.49918346422905135, + "particle_position_z": 0.4562637258155577, "particle_weight": 33067341227104.594 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index 81df3c64c4c..d22fb5b57e7 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,4 +1,16 @@ { + "lev=0": { + "Bx": 116102.13010406512, + "By": 245467.5412026496, + "Bz": 65305.35287114741, + "Ex": 118565481099326.73, + "Ey": 18640100573642.96, + "Ez": 51777969821120.38, + "jx": 2.1177246490642464e+16, + "jy": 351544815339900.3, + "jz": 4573466562652713.0, + "rho": 22294533.587530266 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -26,34 +38,22 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 116102.28415732287, - "By": 245468.57438100342, - "Bz": 65305.38781838063, - "Ex": 118565257880521.4, - "Ey": 18640027127418.004, - "Ez": 51777964618631.914, - "jx": 2.1177240176169996e+16, - "jy": 351529204246424.9, - "jz": 4573419547907262.0, - "rho": 22294773.912022192 - }, "plasma_e": { - "particle_momentum_x": 2.0912370354448323e-19, - "particle_momentum_y": 1.3510736736168866e-20, - "particle_momentum_z": 2.633607024821822e-17, - "particle_position_x": 0.4991733347875822, - "particle_position_y": 0.49919828067421096, - "particle_position_z": 0.4562416755755165, + "particle_momentum_x": 2.0912364079892767e-19, + "particle_momentum_y": 1.3510804924903876e-20, + "particle_momentum_z": 2.6336070297356917e-17, + "particle_position_x": 0.49917333523960167, + "particle_position_y": 0.49919828074154127, + "particle_position_z": 0.45624167532314724, "particle_weight": 33067341227104.625 }, "plasma_p": { - "particle_momentum_x": 2.105303160755537e-19, - "particle_momentum_y": 3.3281768098792752e-21, - "particle_momentum_z": 4.829599953650214e-14, - "particle_position_x": 0.4991250029759827, - "particle_position_y": 0.49912499825258744, - "particle_position_z": 0.4563845470979585, + "particle_momentum_x": 2.1053025446547977e-19, + "particle_momentum_y": 3.3272838305044046e-21, + "particle_momentum_z": 4.829599953646669e-14, + "particle_position_x": 0.49912500297599893, + "particle_position_y": 0.49912499825258866, + "particle_position_z": 0.45638454709795806, "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 344495f8144..50971f0d45e 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,4 +1,25 @@ { + "lev=0": { + "Bx": 115622.1878941125, + "By": 243403.66715242947, + "Bz": 63602.66529020351, + "Ex": 117118934931289.56, + "Ey": 18448377440588.27, + "Ez": 50821967818817.24, + "jx": 2.1044522674691452e+16, + "jy": 329314843111847.94, + "jz": 4524787623275627.0, + "rho": 22128883.53068064 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -8,6 +29,24 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_e": { + "particle_momentum_x": 1.9905288525315664e-19, + "particle_momentum_y": 1.2685401564810352e-20, + "particle_momentum_z": 2.6334746597885943e-17, + "particle_position_x": 0.4991702885212871, + "particle_position_y": 0.4991949978513417, + "particle_position_z": 0.4562491611716817, + "particle_weight": 33067341227104.625 + }, + "plasma_p": { + "particle_momentum_x": 2.002897995377179e-19, + "particle_momentum_y": 2.87423099731012e-21, + "particle_momentum_z": 4.8296000849128737e-14, + "particle_position_x": 0.49912500259846493, + "particle_position_y": 0.4991249979476612, + "particle_position_z": 0.45638454712861487, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -16,44 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "lev=0": { - "Bx": 115622.38124168929, - "By": 243405.25199746038, - "Bz": 63602.713209078014, - "Ex": 117118723119070.72, - "Ey": 18448325716024.688, - "Ez": 50821980097172.29, - "jx": 2.104451599585796e+16, - "jy": 329301545706958.4, - "jz": 4524740919108259.0, - "rho": 22129124.901803844 - }, - "plasma_e": { - "particle_momentum_x": 1.9905296213155052e-19, - "particle_momentum_y": 1.2685326003720766e-20, - "particle_momentum_z": 2.6334746549552728e-17, - "particle_position_x": 0.4991702881158205, - "particle_position_y": 0.4991949977965001, - "particle_position_z": 0.45624916139143146, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.0028987523417718e-19, - "particle_momentum_y": 2.875037979316222e-21, - "particle_momentum_z": 4.8296000849162443e-14, - "particle_position_x": 0.49912500259844994, - "particle_position_y": 0.4991249979476608, - "particle_position_z": 0.4563845471286154, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index cc3c53a24c2..454edfc2606 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -11,6 +11,8 @@ from benchmark import Benchmark import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c import yt yt.funcs.mylog.setLevel(50) @@ -20,19 +22,22 @@ class Checksum: """Class for checksum comparison of one test. """ - def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): + def __init__(self, test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Checksum constructor. - Store test_name and plotfile name, and compute checksum - from plotfile and store it in self.data. + Store test_name, output file name and format, compute checksum + from output file and store it in self.data. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to compare fields in the checksum. @@ -42,83 +47,142 @@ def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): """ self.test_name = test_name - self.plotfile = plotfile - self.data = self.read_plotfile(do_fields=do_fields, - do_particles=do_particles) + self.output_file = output_file + self.output_format = output_format + self.data = self.read_output_file(do_fields=do_fields, + do_particles=do_particles) - def read_plotfile(self, do_fields=True, do_particles=True): + def read_output_file(self, do_fields=True, do_particles=True): """ - Get checksum from plotfile. - Read an AMReX plotfile with yt, compute 1 checksum per field and return - all checksums in a dictionary. + Get checksum from output file. + Read an AMReX plotfile with yt or an openPMD file with openPMD viewer, + compute 1 checksum per field and return all checksums in a dictionary. The checksum of quantity Q is max(abs(Q)). Parameters ---------- do_fields: bool, default=True - Whether to read fields from the plotfile. + Whether to read fields from the output file. do_particles: bool, default=True - Whether to read particles from the plotfile. + Whether to read particles from the output file. """ - ds = yt.load(self.plotfile) - # yt 4.0+ has rounding issues with our domain data: - # RuntimeError: yt attempted to read outside the boundaries - # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() - grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] - - # "fields"/"species" we remove: - # - nbody: added by yt by default, unused by us - species_list = set([item[0] for item in ds.field_list if - item[1][:9] == 'particle_' and - item[0] != 'all' and - item[0] != 'nbody']) - - data = {} - - # Compute checksum for field quantities - if do_fields: - for lev in range(ds.max_level+1): - data_lev = {} - lev_grids = [grid for grid in ds.index.grids - if grid.Level == lev] - # Warning: For now, we assume all levels are rectangular - LeftEdge = np.min( - np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) - all_data_level = ds.covering_grid( - level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) - for field in grid_fields: - Q = all_data_level[field].v.squeeze() - data_lev[field[1]] = np.sum(np.abs(Q)) - data['lev=' + str(lev)] = data_lev - - # Compute checksum for particle quantities - if do_particles: - ad = ds.all_data() - for species in species_list: - # properties we remove: - # - particle_cpu/id: they depend on the parallelism: MPI-ranks and - # on-node acceleration scheme, thus not portable - # and irrelevant for physics benchmarking - part_fields = [item[1] for item in ds.field_list - if item[0] == species and - item[1] != 'particle_cpu' and - item[1] != 'particle_id' - ] - data_species = {} - for field in part_fields: - Q = ad[(species, field)].v - data_species[field] = np.sum(np.abs(Q)) - data[species] = data_species + if self.output_format == 'plotfile': + ds = yt.load(self.output_file) + # yt 4.0+ has rounding issues with our domain data: + # RuntimeError: yt attempted to read outside the boundaries + # of a non-periodic domain along dimension 0. + if 'force_periodicity' in dir(ds): ds.force_periodicity() + grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] + + # "fields"/"species" we remove: + # - nbody: added by yt by default, unused by us + species_list = set([item[0] for item in ds.field_list if + item[1][:9] == 'particle_' and + item[0] != 'all' and + item[0] != 'nbody']) + + data = {} + + # Compute checksum for field quantities + if do_fields: + for lev in range(ds.max_level+1): + data_lev = {} + lev_grids = [grid for grid in ds.index.grids + if grid.Level == lev] + # Warning: For now, we assume all levels are rectangular + LeftEdge = np.min( + np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) + all_data_level = ds.covering_grid( + level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) + for field in grid_fields: + Q = all_data_level[field].v.squeeze() + data_lev[field[1]] = np.sum(np.abs(Q)) + data['lev=' + str(lev)] = data_lev + + # Compute checksum for particle quantities + if do_particles: + ad = ds.all_data() + for species in species_list: + # properties we remove: + # - particle_cpu/id: they depend on the parallelism: MPI-ranks and + # on-node acceleration scheme, thus not portable + # and irrelevant for physics benchmarking + part_fields = [item[1] for item in ds.field_list + if item[0] == species and + item[1] != 'particle_cpu' and + item[1] != 'particle_id' + ] + data_species = {} + for field in part_fields: + Q = ad[(species, field)].v + data_species[field] = np.sum(np.abs(Q)) + data[species] = data_species + + elif self.output_format == 'openpmd': + # Load time series + ts = OpenPMDTimeSeries(self.output_file) + data = {} + # Compute number of MR levels + # TODO This calculation of nlevels assumes that the last element + # of level_fields is by default on the highest MR level. + level_fields = [field for field in ts.avail_fields if 'lvl' in field] + nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) + # Compute checksum for field quantities + if do_fields: + for lev in range(nlevels+1): + # Create list of fields specific to level lev + grid_fields = [] + if lev == 0: + grid_fields = [field for field in ts.avail_fields if 'lvl' not in field] + else: + grid_fields = [field for field in ts.avail_fields if f'lvl{lev}' in field] + data_lev = {} + for field in grid_fields: + vector_components = ts.fields_metadata[field]['avail_components'] + if vector_components != []: + for coord in vector_components: + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1], coord=coord) + # key stores strings composed of field name and vector components + # (e.g., field='B' or field='B_lvl1' + coord='y' results in key='By') + key = field.replace(f'_lvl{lev}', '') + coord + data_lev[key] = np.sum(np.abs(Q)) + else: # scalar field + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1]) + data_lev[field] = np.sum(np.abs(Q)) + data[f'lev={lev}'] = data_lev + # Compute checksum for particle quantities + if do_particles: + species_list = [] + if ts.avail_record_components is not None: + species_list = ts.avail_record_components.keys() + for species in species_list: + data_species = {} + part_fields = [item for item in ts.avail_record_components[species] + if item != 'id' and item != 'charge' and item != 'mass'] + # Convert the field name to the name used in plotfiles + for field in part_fields: + Q = ts.get_particle(var_list=[field], species=species, iteration=ts.iterations[-1]) + if field in ['x', 'y', 'z']: + field_name = 'particle_position_' + field + elif field in ['ux', 'uy', 'uz']: + field_name = 'particle_momentum_' + field[-1] + m, = ts.get_particle(['mass'], species=species, iteration=ts.iterations[-1]) + Q *= m*c + elif field in ['w']: + field_name = 'particle_weight' + else: + field_name = 'particle_' + field + data_species[field_name] = np.sum(np.abs(Q)) + data[species] = data_species return data def evaluate(self, rtol=1.e-9, atol=1.e-40): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert that they are equal. Almost all the body of this functions is for user-readable print statements. @@ -136,10 +200,10 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same outer keys (levels, species)? if (self.data.keys() != ref_benchmark.data.keys()): - print("ERROR: Benchmark and plotfile checksum " + print("ERROR: Benchmark and output file checksum " "have different outer keys:") print("Benchmark: %s" % ref_benchmark.data.keys()) - print("Plotfile : %s" % self.data.keys()) + print("Test file: %s" % self.data.keys()) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) print("----------------") @@ -148,12 +212,12 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same inner keys (field and particle quantities)? for key1 in ref_benchmark.data.keys(): if (self.data[key1].keys() != ref_benchmark.data[key1].keys()): - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different inner keys:") print("Common outer keys: %s" % ref_benchmark.data.keys()) print("Benchmark inner keys in %s: %s" % (key1, ref_benchmark.data[key1].keys())) - print("Plotfile inner keys in %s: %s" + print("Test file inner keys in %s: %s" % (key1, self.data[key1].keys())) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) @@ -168,11 +232,11 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): ref_benchmark.data[key1][key2], rtol=rtol, atol=atol) if not passed: - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different value for key [%s,%s]" % (key1, key2)) print("Benchmark: [%s,%s] %.15e" % (key1, key2, ref_benchmark.data[key1][key2])) - print("Plotfile : [%s,%s] %.15e" + print("Test file: [%s,%s] %.15e" % (key1, key2, self.data[key1][key2])) checksums_differ = True # Print absolute and relative error for each failing key diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index c45b6e233f4..cc13ceefa28 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -23,23 +23,23 @@ Example: add these lines to a WarpX CI Python analysis script to run the checksum test. > import checksumAPI - > checksumAPI.evaluate_checksum(test_name, plotfile) + > checksumAPI.evaluate_checksum(test_name, output_file, output_format) - As a script, to evaluate or to reset a benchmark: * Evaluate a benchmark. From a bash terminal: - $ ./checksumAPI.py --evaluate --plotfile \ - --test-name + $ ./checksumAPI.py --evaluate --output-file \ + --output-format 'plotfile' --test-name * Reset a benchmark. From a bash terminal: - $ ./checksumAPI.py --reset-benchmark --plotfile \ - --test-name + $ ./checksumAPI.py --reset-benchmark --output-file \ + --output-format 'openpmd' --test-name """ -def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, +def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e-9, atol=1.e-40, do_fields=True, do_particles=True): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. Parameters @@ -47,8 +47,11 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, test_name: string Name of test, as found between [] in .ini file. - plotfile : string - Plotfile from which the checksum is computed. + output_file : string + Output file from which the checksum is computed. + + output_format : string + Format of the output file (plotfile, openpmd). rtol: float, default=1.e-9 Relative tolerance for the comparison. @@ -62,24 +65,27 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) test_checksum.evaluate(rtol=rtol, atol=atol) -def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): +def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Update the benchmark (overwrites reference json file). Overwrite value of benchmark corresponding to - test_name with checksum read from input plotfile. + test_name with checksum read from output file. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to write field checksums in the benchmark. @@ -87,34 +93,37 @@ def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): do_particles: bool, default=True Whether to write particles checksums in the benchmark. """ - ref_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + ref_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) ref_benchmark = Benchmark(test_name, ref_checksum.data) ref_benchmark.reset() -def reset_all_benchmarks(path_to_all_plotfiles): +def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): """ Update all benchmarks (overwrites reference json files) - found in path_to_all_plotfiles + found in path_to_all_output_files Parameters ---------- - path_to_all_plotfiles: string - Path to all plotfiles for which the benchmarks - are to be reset. The plotfiles should be named _plt, which is + path_to_all_output_files: string + Path to all output files for which the benchmarks + are to be reset. The output files should be named _plt, which is what regression_testing.regtests.py does, provided we're careful enough. + + output_format: string + Format of the output files (plotfile, openpmd). """ - # Get list of plotfiles in path_to_all_plotfiles - plotfile_list = glob.glob(path_to_all_plotfiles + '*_plt*[0-9]', - recursive=True) - plotfile_list.sort() + # Get list of output files in path_to_all_output_files + output_file_list = glob.glob(path_to_all_output_files + '*_plt*[0-9]', + recursive=True) + output_file_list.sort() - # Loop over plotfiles and reset the corresponding benchmark - for plotfile in plotfile_list: - test_name = os.path.split(plotfile)[1][:-9] - reset_benchmark(test_name, plotfile) + # Loop over output files and reset the corresponding benchmark + for output_file in output_file_list: + test_name = os.path.split(output_file)[1][:-9] + reset_benchmark(test_name, output_file, output_format) if __name__ == '__main__': @@ -131,11 +140,14 @@ def reset_all_benchmarks(path_to_all_plotfiles): required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, help='Name of the test (as in WarpX-tests.ini)') - parser.add_argument('--plotfile', dest='plotfile', type=str, default='', + parser.add_argument('--output-file', dest='output_file', type=str, default='', required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, - help='Name of WarpX plotfile') - + help='Name of WarpX output file') + parser.add_argument('--output-format', dest='output_format', type=str, default='plotfile', + required='--evaluate' in sys.argv or + '--reset-benchmark' in sys.argv, + help='Format of the output file (plotfile, openpmd)') parser.add_argument('--skip-fields', dest='do_fields', default=True, action='store_false', help='If used, do not read/write field checksums') @@ -143,7 +155,7 @@ def reset_all_benchmarks(path_to_all_plotfiles): default=True, action='store_false', help='If used, do not read/write particle checksums') - # Fields and/or particles are read from plotfile/written to benchmark? + # Fields and/or particles are read from output file/written to benchmark? parser.add_argument('--rtol', dest='rtol', type=float, default=1.e-9, help='relative tolerance for comparison') @@ -155,26 +167,27 @@ def reset_all_benchmarks(path_to_all_plotfiles): parser.add_argument('--reset-all-benchmarks', dest='reset_all_benchmarks', action='store_true', default=False, help='Reset all benchmarks.') - parser.add_argument('--path-to-all-plotfiles', - dest='path_to_all_plotfiles', type=str, default='', + parser.add_argument('--path-to-all-output-files', + dest='path_to_all_output_files', type=str, default='', required='--reset-all-benchmarks' in sys.argv, - help='Directory containing all benchmark plotfiles, \ + help='Directory containing all benchmark output files, \ typically WarpX-benchmarks generated by \ regression_testing/regtest.py') args = parser.parse_args() if args.reset_benchmark: - reset_benchmark(args.test_name, args.plotfile, + reset_benchmark(args.test_name, args.output_file, args.output_format, do_fields=args.do_fields, do_particles=args.do_particles) if args.evaluate: - evaluate_checksum(args.test_name, args.plotfile, rtol=args.rtol, - atol=args.atol, do_fields=args.do_fields, - do_particles=args.do_particles) + evaluate_checksum(args.test_name, args.output_file, args.output_format, + rtol=args.rtol, atol=args.atol, + do_fields=args.do_fields, do_particles=args.do_particles) if args.reset_all_benchmarks: - # WARNING: this mode does not support skip-fields/particles - # and tolerances - reset_all_benchmarks(args.path_to_all_plotfiles) + if args.output_format == 'openpmd': + sys.exit('Option --reset-all-benchmarks does not work with openPMD format') + # WARNING: this mode does not support skip-fields/particles and tolerances + reset_all_benchmarks(args.path_to_all_output_files, args.output_format) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index ceba435d251..2a98d7b3736 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d4f2e68d0d3..f5ef38e881b 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -1812,7 +1812,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserAcceleration_1d] buildDir = . @@ -1973,7 +1974,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_single_precision_comms_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserInjection] buildDir = . @@ -2212,11 +2214,12 @@ analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py [LoadExternalFieldRZ] buildDir = . inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 0 +restartTest = 1 +restartFileNum = 150 useMPI = 1 numprocs = 1 useOMP = 1 @@ -3076,6 +3079,25 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py + [Python_ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py @@ -4219,6 +4241,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py +[scraping_filter] +buildDir = . +inputFile = Examples/Tests/scraping/inputs_rz_filter +runtime_params = warpx.abort_on_warning_threshold = medium +dim = 2 +addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py + [silver_mueller_1d] buildDir = . inputFile = Examples/Tests/silver_mueller/inputs_1d @@ -4441,3 +4480,54 @@ compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py + +[BeamBeamCollision] +buildDir = . +inputFile = Examples/Physics_applications/beam-beam_collision/inputs +runtime_params = warpx.abort_on_warning_threshold=high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +outputFile = BeamBeamCollision_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py + +[ImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[SemiImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py diff --git a/Source/AcceleratorLattice/AcceleratorLattice.cpp b/Source/AcceleratorLattice/AcceleratorLattice.cpp index e05167c261d..8dc0983d622 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.cpp +++ b/Source/AcceleratorLattice/AcceleratorLattice.cpp @@ -88,8 +88,9 @@ AcceleratorLattice::InitElementFinder (int const lev, amrex::BoxArray const & ba } void -AcceleratorLattice::UpdateElementFinder (int const lev) -{ +AcceleratorLattice::UpdateElementFinder (int const lev) // NOLINT(readability-make-member-function-const) +{ // Techniquely clang-tidy is correct because + // m_element_finder is unique_ptr, not const*. if (m_lattice_defined) { for (amrex::MFIter mfi(*m_element_finder); mfi.isValid(); ++mfi) { diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index 24cdb691bb0..6773ed56a65 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -72,8 +72,8 @@ struct LatticeElementFinder * @param[in] a_offset particle index offset needed to access particle info * @param[in] accelerator_lattice a reference to the accelerator lattice at the refinement level */ - LatticeElementFinderDevice GetFinderDeviceInstance (WarpXParIter const& a_pti, int a_offset, - AcceleratorLattice const& accelerator_lattice); + [[nodiscard]] LatticeElementFinderDevice GetFinderDeviceInstance ( + WarpXParIter const& a_pti, int a_offset, AcceleratorLattice const& accelerator_lattice) const; /* The index lookup tables for each lattice element type */ amrex::Gpu::DeviceVector d_quad_indices; @@ -89,52 +89,7 @@ struct LatticeElementFinder */ void setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices) - { - - using namespace amrex::literals; - - const auto nelements = static_cast(zs.size()); - amrex::ParticleReal const * zs_arr = zs.data(); - amrex::ParticleReal const * ze_arr = ze.data(); - int * indices_arr = indices.data(); - - amrex::Real const zmin = m_zmin; - amrex::Real const dz = m_dz; - - amrex::ParticleReal const gamma_boost = m_gamma_boost; - amrex::ParticleReal const uz_boost = m_uz_boost; - amrex::Real const time = m_time; - - amrex::ParallelFor( m_nz, - [=] AMREX_GPU_DEVICE (int iz) { - - // Get the location of the grid node - amrex::Real z_node = zmin + iz*dz; - - if (gamma_boost > 1._prt) { - // Transform to lab frame - z_node = gamma_boost*z_node + uz_boost*time; - } - - // Find the index to the element that is closest to the grid cell. - // For now, this assumes that there is no overlap among elements of the same type. - for (int ie = 0 ; ie < nelements ; ie++) { - // Find the mid points between element ie and the ones before and after it. - // The first and last element need special handling. - const amrex::ParticleReal zcenter_left = (ie == 0)? - (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); - const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? - (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); - if (zcenter_left <= z_node && z_node < zcenter_right) { - indices_arr[iz] = ie; - } - - } - } - ); - } - + amrex::Gpu::DeviceVector & indices) const; }; /** diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index 62ebdcc1f4f..ba2e51715fb 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -78,14 +78,13 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, LatticeElementFinderDevice LatticeElementFinder::GetFinderDeviceInstance (WarpXParIter const& a_pti, int const a_offset, - AcceleratorLattice const& accelerator_lattice) + AcceleratorLattice const& accelerator_lattice) const { LatticeElementFinderDevice result; result.InitLatticeElementFinderDevice(a_pti, a_offset, accelerator_lattice, *this); return result; } - void LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& a_pti, int const a_offset, AcceleratorLattice const& accelerator_lattice, @@ -97,7 +96,7 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& int const lev = a_pti.GetLevel(); m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; @@ -121,3 +120,50 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& } } + +void +LatticeElementFinder::setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, + amrex::Gpu::DeviceVector const & ze, + amrex::Gpu::DeviceVector & indices) const +{ + + using namespace amrex::literals; + + const auto nelements = static_cast(zs.size()); + amrex::ParticleReal const * zs_arr = zs.data(); + amrex::ParticleReal const * ze_arr = ze.data(); + int * indices_arr = indices.data(); + + amrex::Real const zmin = m_zmin; + amrex::Real const dz = m_dz; + + amrex::ParticleReal const gamma_boost = m_gamma_boost; + amrex::ParticleReal const uz_boost = m_uz_boost; + amrex::Real const time = m_time; + + amrex::ParallelFor( m_nz, + [=] AMREX_GPU_DEVICE (int iz) { + + // Get the location of the grid node + amrex::Real z_node = zmin + iz*dz; + + if (gamma_boost > 1._prt) { + // Transform to lab frame + z_node = gamma_boost*z_node + uz_boost*time; + } + + // Find the index to the element that is closest to the grid cell. + // For now, this assumes that there is no overlap among elements of the same type. + for (int ie = 0 ; ie < nelements ; ie++) { + // Find the mid points between element ie and the ones before and after it. + // The first and last element need special handling. + const amrex::ParticleReal zcenter_left = (ie == 0)? + (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); + const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? + (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); + if (zcenter_left <= z_node && z_node < zcenter_right) { + indices_arr[iz] = ie; + } + } + }); +} diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp index 9c8726a9c4b..21b1f164777 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp @@ -58,7 +58,7 @@ HardEdgedPlasmaLensDevice::InitHardEdgedPlasmaLensDevice (HardEdgedPlasmaLens co nelements = h_plasmalens.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_plasmalens.d_zs.data(); d_ze_arr = h_plasmalens.d_ze.data(); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp index 51bcf7a2497..bcf50c55666 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp @@ -58,7 +58,7 @@ HardEdgedQuadrupoleDevice::InitHardEdgedQuadrupoleDevice (HardEdgedQuadrupole co nelements = h_quad.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_quad.d_zs.data(); d_ze_arr = h_quad.d_ze.data(); diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index b2016e65d2a..1f0b6f09a33 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -513,7 +513,7 @@ MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, void MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) { - if (dt == dt_B) return; + if (dt == dt_B) { return; } dt_B = dt; @@ -529,7 +529,7 @@ MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) void MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) { - if (dt == dt_E) return; + if (dt == dt_E) { return; } dt_E = dt; @@ -597,7 +597,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri auto nge = IntVect(AMREX_D_DECL(2, 2, 2)); auto ngb = IntVect(AMREX_D_DECL(2, 2, 2)); int ngf_int = 0; - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ngf_int = std::max( ngf_int, 1 ); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { + ngf_int = std::max( ngf_int, 1 ); + } auto ngf = IntVect(AMREX_D_DECL(ngf_int, ngf_int, ngf_int)); if (do_moving_window) { @@ -1074,9 +1076,9 @@ void PML::Exchange (const std::array& mf_pml, const int do_pml_in_domain) { const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); - if (mf_pml[1] && mf[1]) Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); - if (mf_pml[2] && mf[2]) Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } void diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 8f3c5a1b49b..af721d70b6d 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -51,13 +51,13 @@ void WarpX::DampPML (const int lev) { DampPML(lev, PatchType::fine); - if (lev > 0) DampPML(lev, PatchType::coarse); + if (lev > 0) { DampPML(lev, PatchType::coarse); } } void WarpX::DampPML (const int lev, PatchType patch_type) { - if (!do_pml) return; + if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -228,15 +228,15 @@ void WarpX::DampJPML (int lev) { DampJPML(lev, PatchType::fine); - if (lev > 0) DampJPML(lev, PatchType::coarse); + if (lev > 0) { DampJPML(lev, PatchType::coarse); } } void WarpX::DampJPML (int lev, PatchType patch_type) { - if (!do_pml) return; - if (!do_pml_j_damping) return; - if (!pml[lev]) return; + if (!do_pml) { return; } + if (!do_pml_j_damping) { return; } + if (!pml[lev]) { return; } WARPX_PROFILE("WarpX::DampJPML()"); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index f0a1d406f95..af7a385b729 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -81,11 +81,12 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d applySilverMueller = true; } } - if(applySilverMueller) m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], WarpX::field_boundary_lo, WarpX::field_boundary_hi); + } } } @@ -105,20 +106,20 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoRhofield(rho, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } } #ifdef WARPX_DIM_RZ // Applies the boundary conditions that are specific to the axis when in RZ. void -WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) +WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const { const amrex::IntVect ngE = get_ng_fieldgather(); @@ -139,7 +140,7 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex const amrex::Real rmin = xyzmin[0]; // Skip blocks that don't touch the axis - if (rmin > 0._rt) continue; + if (rmin > 0._rt) { continue; } amrex::Array4 const& Er_arr = Er->array(mfi); amrex::Array4 const& Et_arr = Et->array(mfi); diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index ef3d3dc2ad5..62b92ba94b5 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -191,7 +191,7 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) sign *= -1._rt; + if (is_tangent_to_PEC) { sign *= -1._rt; } #if (defined WARPX_DIM_RZ) if (icomp == 0 && idim == 0 && iside == 1) { // Add radial scale so that drEr/dr = 0. @@ -329,7 +329,7 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) sign *= -1._rt; + if (is_normal_to_PEC) { sign *= -1._rt; } #if (defined WARPX_DIM_RZ) if (icomp == 0 && idim == 0 && iside == 1) { // Add radial scale so that drBr/dr = 0. @@ -391,17 +391,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) field(ijk_vec, n) = 0._rt; + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; // otherwise update the internal cell if the mirror guard cell exists - else if (fabbox.contains(iv_mirror)) - { + } else if (fabbox.contains(iv_mirror)) { field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); } } @@ -412,16 +412,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) { - if (tangent_to_bndy[idim]) + if (tangent_to_bndy[idim]) { field(iv_mirror, n) = -field(ijk_vec, n); - else + } else { field(iv_mirror, n) = field(ijk_vec, n); + } } } } @@ -454,7 +455,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; @@ -464,7 +465,7 @@ using namespace amrex; // first value in the domain (nodal fields) if (ijk_vec == iv_mirror) { iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) field(ijk_vec, n) = field(iv_mirror, n); + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } } // otherwise set the mirror guard cell equal to the internal cell value else if (fabbox.contains(iv_mirror)) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0b8d1c0fcc8..4692d8e980f 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -18,8 +18,8 @@ using namespace amrex::literals; bool PEC::isAnyBoundaryPEC() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } } return false; } @@ -250,8 +250,8 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density @@ -276,7 +276,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& rho_array = rho->array(mfi); @@ -342,8 +342,8 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -398,7 +398,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jx_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jx_array = Jx->array(mfi); @@ -433,7 +433,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jy_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jy_array = Jy->array(mfi); @@ -468,7 +468,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jz_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jz_array = Jz->array(mfi); @@ -520,8 +520,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - Pe_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - Pe_nodal[idim]); @@ -538,7 +538,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Pe_array = Pefield->array(mfi); diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 209113dedeb..b87c54be9f0 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -171,11 +171,11 @@ class BTDMultiFabHeaderImpl void WriteMultiFabHeader (); /** Returns size, m_ba_size, of the Box Array, m_ba.*/ - int ba_size () {return m_ba_size;} + [[nodiscard]] int ba_size () const {return m_ba_size;} /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Returns prefix of the ith-fab on disk, i.e., ith fab of the MultiFab data. * \param[in] ifab index of the ith fab in the MultiFab data. */ @@ -316,9 +316,9 @@ public: /** Reads the particle header file at m_Header_path and stores its data*/ void ReadHeader (); /** Writes the meta-data of particle box array in header file, with path, m_Header_path*/ - void WriteHeader (); + void WriteHeader () const; /** Returns the size of the box array, m_ba_size */ - int ba_size () {return m_ba_size; } + [[nodiscard]] int ba_size () const {return m_ba_size; } /** Increases Box array size, m_ba_size, by add_size * \param[in] add_size */ @@ -326,7 +326,7 @@ public: /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Resize boxArray, m_ba, to size, m_ba_size. */ void ResizeBoxArray () { m_ba.resize(m_ba_size); } /** Set Box indices of the ith-box in Box Array, m_ba, to the new Box, ba_box. diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index 9f4976c20cb..da4a3a79547 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -118,7 +118,7 @@ BTDPlotfileHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -241,7 +241,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_minval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_minval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } ablastr::utils::text::goto_next_line(is); @@ -251,7 +251,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_maxval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_maxval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } @@ -266,9 +266,9 @@ BTDMultiFabHeaderImpl::WriteMultiFabHeader () } std::ofstream FabHeaderFile; FabHeaderFile.open(m_Header_path.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if ( !FabHeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + std::ofstream::trunc | + std::ofstream::binary); + if ( !FabHeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } FabHeaderFile.precision(17); @@ -421,7 +421,7 @@ BTDSpeciesHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -516,7 +516,7 @@ BTDParticleDataHeaderImpl::ReadHeader () } void -BTDParticleDataHeaderImpl::WriteHeader () +BTDParticleDataHeaderImpl::WriteHeader () const { if (amrex::FileExists(m_Header_path)) { amrex::FileSystem::Remove(m_Header_path); @@ -525,7 +525,7 @@ BTDParticleDataHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); m_ba.writeOn(HeaderFile); diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index f72e7aa8b07..ab894da69c2 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -275,7 +275,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_boost = (t_lab / m_gamma_boost - t_boost) * PhysConst::c / m_beta_boost; return current_z_boost; @@ -284,7 +284,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_lab = (t_lab - t_boost / m_gamma_boost ) * PhysConst::c / m_beta_boost; return current_z_lab; @@ -294,13 +294,13 @@ private: * \param[in] ref_ratio refinement ratio in the z-direction at level, lev-1. * The ref-ratio in the z-direction for single-level diagnostics is 1. */ - amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio); + [[nodiscard]] amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio) const; /** Compute k-index corresponding to current lab-frame z co-ordinate (m_current_z_lab) * for the ith buffer i_buffer, and at level, lev. * \param[in] i_buffer snapshot index * \param[in] lev mesh-refinement level at which the lab-frame z-index is computed */ - int k_index_zlab (int i_buffer, int lev); + [[nodiscard]] int k_index_zlab (int i_buffer, int lev) const; /** whether field buffer is full * \param[in] i_buffer buffer id for which the buffer size is checked. * returns bool = true is buffer is full, that is, @@ -380,8 +380,8 @@ private: /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader( std::string Buffer_FabHeaderFilename, - std::string snapshot_FabHeaderFilename, + void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, std::string newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index ce70e82108e..36179ee9d0c 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -112,8 +112,9 @@ void BTDiagnostics::DerivedInitData () const amrex::ParmParse pp_diag_name(m_diag_name); int write_species = 1; pp_diag_name.query("write_species", write_species); - if ((m_output_species_names.empty()) && (write_species == 1)) + if ((m_output_species_names.empty()) && (write_species == 1)) { m_output_species_names = mpc.GetSpeciesNames(); + } m_do_back_transformed_particles = ((!m_output_species_names.empty()) && (write_species == 1)); @@ -233,7 +234,7 @@ BTDiagnostics::ReadParameters () pp_diag_name.query("do_back_transformed_fields", m_do_back_transformed_fields); pp_diag_name.query("do_back_transformed_particles", m_do_back_transformed_particles); AMREX_ALWAYS_ASSERT(m_do_back_transformed_fields or m_do_back_transformed_particles); - if (!m_do_back_transformed_fields) m_varnames.clear(); + if (!m_do_back_transformed_fields) { m_varnames.clear(); } std::vector intervals_string_vec = {"0"}; @@ -424,11 +425,11 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // computed using the coarsened cell-size in the lab-frame obtained using // the ref_ratio at level, lev-1. auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } // Number of lab-frame cells in z-direction at level, lev const int num_zcells_lab = static_cast( std::floor ( ( zmax_buffer_lab - zmin_buffer_lab) - / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]) ) ); + / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]))); // Take the max of 0 and num_zcells_lab const int Nz_lab = std::max( 0, num_zcells_lab ); #if (AMREX_SPACEDIM >= 2) @@ -502,7 +503,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) void BTDiagnostics::DefineCellCenteredMultiFab(int lev) { - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } // Creating MultiFab to store cell-centered data in boosted-frame for the entire-domain // This MultiFab will store all the user-requested fields in the boosted-frame auto & warpx = WarpX::GetInstance(); @@ -524,7 +525,7 @@ void BTDiagnostics::InitializeFieldFunctors (int lev) { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } #ifdef WARPX_DIM_RZ // For RZ, initialize field functors RZ for openpmd @@ -611,16 +612,16 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () const auto m_varnames_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(m_cellcenter_varnames_fields.size()); for (int comp=0; comp 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } const int k_lab = static_cast(std::floor ( ( m_current_z_lab[i_buffer] - (prob_domain_zmin_lab ) ) @@ -884,17 +887,16 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) void BTDiagnostics::SetSnapshotFullStatus (const int i_buffer) { - if (m_snapshot_full[i_buffer] == 1) return; + if (m_snapshot_full[i_buffer] == 1) { return; } // if the last valid z-index of the snapshot, which is 0, is filled, then // set the snapshot full integer to 1 - if (m_lastValidZSlice[i_buffer] == 1) m_snapshot_full[i_buffer] = 1; - + if (m_lastValidZSlice[i_buffer] == 1) { m_snapshot_full[i_buffer] = 1; } } void BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) { - if (m_field_buffer_multifab_defined[i_buffer] == 1) return; + if (m_field_buffer_multifab_defined[i_buffer] == 1) { return; } auto & warpx = WarpX::GetInstance(); const int hi_k_lab = m_buffer_k_index_hi[i_buffer]; @@ -911,7 +913,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) m_mf_output[i_buffer][lev].setVal(0.); auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Real cellsize = (idim < WARPX_ZINDEX)? warpx.Geom(lev).CellSize(idim): @@ -954,7 +956,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) void BTDiagnostics::DefineSnapshotGeometry (const int i_buffer, const int lev) { - if (m_snapshot_geometry_defined[i_buffer] == 1) return; + if (m_snapshot_geometry_defined[i_buffer] == 1) { return; } if (lev == 0) { // Default non-periodic geometry for diags @@ -1151,17 +1153,20 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) // Create directory only when the first buffer is flushed out. if (m_buffer_flush_counter[i_snapshot] == 0 || m_first_flush_after_restart[i_snapshot] == 1) { // Create Level_0 directory to store all Cell_D and Cell_H files - if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) + if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) { amrex::CreateDirectoryFailed(snapshot_Level0_path); + } // Create directory for each species selected for diagnostic for (int i = 0; i < m_particles_buffer[i_snapshot].size(); ++i) { const std::string snapshot_species_path = snapshot_path + "/" + m_output_species_names[i]; - if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(snapshot_species_path); + } // Create Level_0 directory for particles to store Particle_H and DATA files const std::string species_Level0_path = snapshot_species_path + "/Level_0"; - if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(species_Level0_path); + } } const std::string buffer_WarpXHeader_path = recent_Buffer_filepath + "/WarpXHeader"; const std::string snapshot_WarpXHeader_path = snapshot_path + "/WarpXHeader"; @@ -1254,7 +1259,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_species_Header.c_str(), snapshot_species_Header.c_str()) == 0, std::string("Renaming ").append(recent_species_Header).append(" to ").append(snapshot_species_Header).append(" has failed")); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } // if finite number of particles in the output, copy ParticleHdr and Data file WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1265,7 +1270,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } else { InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } if (m_totalParticles_flushed_already[i_snapshot][i]==0) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1332,9 +1337,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader(std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, + std::string newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 9797b77ac1d..c85dbd6b226 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -142,7 +142,7 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer, bool /* force_flush */) // If the saving of the particles was not set up for any of the species for this boundary // or if no particles have been lost, then don't write anything out. - if (n_particles == 0) return; + if (n_particles == 0) { return; } // This is not a backtransform diagnostics bool const isBTD = false; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index f6f2bcd1205..22754f51a53 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -162,7 +162,7 @@ BackTransformFunctor::PrepareFunctorData (int i_buffer, m_current_z_boost[i_buffer] = current_z_boost; m_k_index_zlab[i_buffer] = k_index_zlab; m_perform_backtransform[i_buffer] = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform[i_buffer] = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform[i_buffer] = 1; } } void diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index ce463a11ca3..8a6f0765664 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -37,10 +37,10 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti { using namespace amrex::literals; - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_wpnew = attribs[PIdx::w].dataPtr(); m_uxpnew = attribs[PIdx::ux].dataPtr(); m_uypnew = attribs[PIdx::uy].dataPtr(); @@ -80,7 +80,7 @@ BackTransformParticleFunctor::BackTransformParticleFunctor ( void BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst, int &totalParticleCounter, int i_buffer) const { - if (m_perform_backtransform[i_buffer] == 0) return; + if (m_perform_backtransform[i_buffer] == 0) { return; } auto &warpx = WarpX::GetInstance(); // get particle slice const int nlevs = std::max(0, m_pc_src->finestLevel()+1); @@ -142,8 +142,9 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i) { - if (Flag[i] == 1) GetParticleLorentzTransform(dst_data, src_data, i, + if (Flag[i] == 1) { GetParticleLorentzTransform(dst_data, src_data, i, old_size + IndexLocation[i]); + } }); amrex::Gpu::synchronize(); } @@ -171,5 +172,5 @@ BackTransformParticleFunctor::PrepareFunctorData ( int i_buffer, bool z_slice_in m_current_z_boost.at(i_buffer) = current_z_boost; m_t_lab.at(i_buffer) = t_lab; m_perform_backtransform.at(i_buffer) = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform.at(i_buffer) = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform.at(i_buffer) = 1; } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 4663a5e1a70..b21bbded011 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -36,16 +36,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 60b62181c4b..feb25c11183 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -135,8 +135,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal uy = p.rdata(PIdx::uy) / PhysConst::c; const amrex::ParticleReal uz = p.rdata(PIdx::uz) / PhysConst::c; amrex::Real filter; - if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz) == 0._rt)) filter = 0._rt; - else filter = 1._rt; + if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz) == 0._rt)) { filter = 0._rt; + } else { filter = 1._rt; } amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(p.rdata(PIdx::w) * filter)); }); // Divide value by number of particles for average. Set average to zero if there are no particles @@ -147,8 +147,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, amrex::Array4 const& a_ppc = ppc_mf.const_array(mfi); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - if (a_ppc(i,j,k,0) == 0) a_red(i,j,k,0) = 0; - else a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); + if (a_ppc(i,j,k,0) == 0) { a_red(i,j,k,0) = 0; + } else { a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); } }); } } diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 21d5ff699d0..53ce319d747 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -114,7 +114,7 @@ public: /** Whether the last timestep is always dumped */ [[nodiscard]] bool DoDumpLastTimestep () const {return m_dump_last_timestep;} /** Returns the number of snapshots used in BTD. For Full-Diagnostics, the value is 1*/ - int getnumbuffers() {return m_num_buffers;} + [[nodiscard]] int getnumbuffers() const {return m_num_buffers;} /** Time in lab-frame associated with the ith snapshot * \param[in] i_buffer index of the buffer */ diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 0bae30689df..5de4b6a8c46 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -586,9 +586,7 @@ Diagnostics::FilterComputePackFlush (int step, bool force_flush) } for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - if ( !DoDump (step, i_buffer, force_flush) ) continue; + if ( !DoDump (step, i_buffer, force_flush) ) { continue; } Flush(i_buffer, force_flush); } - - } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 2247a7eaadf..5f59cd723da 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -172,7 +172,7 @@ FlushFormatCheckpoint::CheckpointParticles ( const std::string& dir, const amrex::Vector& particle_diags) const { - for (auto& part_diag: particle_diags) { + for (const auto& part_diag: particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); Vector real_names; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index e15a7065f3a..3b7006243e7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -27,8 +27,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) std::string openpmd_backend {"default"}; pp_diag_name.query("openpmd_backend", openpmd_backend); // pick first available backend if default is chosen - if( openpmd_backend == "default" ) + if( openpmd_backend == "default" ) { openpmd_backend = WarpXOpenPMDFileType(); + } pp_diag_name.add("openpmd_backend", openpmd_backend); @@ -37,12 +38,13 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) const bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; - if ( openpmd_encoding == "v" ) + if ( openpmd_encoding == "v" ) { encoding = openPMD::IterationEncoding::variableBased; - else if ( openpmd_encoding == "g" ) + } else if ( openpmd_encoding == "g" ) { encoding = openPMD::IterationEncoding::groupBased; - else if ( openpmd_encoding == "f" ) + } else if ( openpmd_encoding == "f" ) { encoding = openPMD::IterationEncoding::fileBased; + } std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); @@ -65,8 +67,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) { bool openpmd_tspf = false; const bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); - if ( tspfDefined && openpmd_tspf ) + if ( tspfDefined && openpmd_tspf ) { encoding = openPMD::IterationEncoding::fileBased; + } } // ADIOS2 operator type & parameters @@ -148,8 +151,9 @@ FlushFormatOpenPMD::WriteToFile ( int output_iteration = iteration[0]; // in backtransformed diagnostics (BTD), we dump into a series of labframe // snapshots - if( isBTD ) + if( isBTD ) { output_iteration = snapshotID; + } // Set step and output directory name. m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, file_min_digits, isBTD); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 5e7d9b7b8e5..486dcc3b5ee 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -46,12 +46,12 @@ public: const std::string& plotfilename, bool plot_raw_fields_guards) const; /** \brief Write particles data to file. - * \param[in] filename name of output directory + * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level * \param[in] isBTD whether this is a back-transformed diagnostic */ - void WriteParticles(const std::string& filename, + void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, bool isBTD = false) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index c2e372b1fb8..df73ed34c94 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -86,7 +86,7 @@ FlushFormatPlotfile::WriteToFile ( Vector rfs; const VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::Version_v1); - if (plot_raw_fields) rfs.emplace_back("raw_fields"); + if (plot_raw_fields) { rfs.emplace_back("raw_fields"); } amrex::WriteMultiLevelPlotfile(filename, nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, @@ -245,8 +245,9 @@ FlushFormatPlotfile::WriteWarpXHeader( HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if( ! HeaderFile.good()) + if( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); + } HeaderFile.precision(17); @@ -342,7 +343,7 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Real time, bool isBTD) const { - for (auto& part_diag : particle_diags) { + for (const auto& part_diag : particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? @@ -484,7 +485,7 @@ WriteCoarseVector( const std::string field_name, const int lev, const bool plot_guards ) { IntVect ng(0); - if (plot_guards) ng = Fx_fp->nGrowVect(); + if (plot_guards) { ng = Fx_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -521,7 +522,7 @@ WriteCoarseScalar( const std::string field_name, const int icomp ) { IntVect ng(0); - if (plot_guards) ng = F_fp->nGrowVect(); + if (plot_guards) { ng = F_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -544,7 +545,7 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { - if (!plot_raw_fields) return; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 1dfbffad525..85a459126b7 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -147,7 +147,7 @@ FullDiagnostics::FlushRaw () {} bool FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) { - if (m_already_done) return false; + if (m_already_done) { return false; } if ( force_flush || (m_intervals.contains(step+1)) ){ m_already_done = true; return true; @@ -360,7 +360,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ - if (!m_dump_rz_modes) return; + if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); @@ -507,11 +507,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); if ( std::fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } if ( std::fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } // User-defined value for coarsening should be an integer divisor of // blocking factor at level, lev. This assert is not relevant and thus @@ -573,7 +575,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { ba.coarsen(m_crse_ratio); // Generate a new distribution map if the physical m_lo and m_hi for the output // is different from the lo and hi physical co-ordinates of the simulation domain. - if (!use_warpxba) dmap = amrex::DistributionMapping{ba}; + if (!use_warpxba) { dmap = amrex::DistributionMapping{ba}; } // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. const int ngrow = (m_format == "sensei" || m_format == "ascent") ? 1 : 0; int const ncomp = static_cast(m_varnames.size()); diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index f9907b8bdc6..d220396ed12 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -38,7 +38,7 @@ public: /** Start a new iteration, i.e., dump has not been done yet. */ void NewIteration (); Diagnostics& GetDiag(int idiag) {return *alldiags[idiag]; } - int GetTotalDiags() {return ndiags;} + [[nodiscard]] int GetTotalDiags() const {return ndiags;} DiagTypes diagstypes(int idiag) {return diags_types[idiag];} private: /** Vector of pointers to all diagnostics */ diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 5a50dfb6565..ea14919c713 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -70,9 +70,9 @@ MultiDiagnostics::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", ".diag_type must be Full or BackTransformed or BoundaryScraping"); - if (diag_type_str == "Full") diags_types[i] = DiagTypes::Full; - if (diag_type_str == "BackTransformed") diags_types[i] = DiagTypes::BackTransformed; - if (diag_type_str == "BoundaryScraping") diags_types[i] = DiagTypes::BoundaryScraping; + if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } + if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } } @@ -82,11 +82,13 @@ MultiDiagnostics::FilterComputePackFlush (int step, bool force_flush, bool BackT int i = 0; for (auto& diag : alldiags){ if (BackTransform) { - if (diags_types[i] == DiagTypes::BackTransformed) + if (diags_types[i] == DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } else { - if (diags_types[i] != DiagTypes::BackTransformed) + if (diags_types[i] != DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } ++i; } diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index 2a29748b3e1..fbe67ba6fc5 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -214,14 +214,21 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_RZ) +#if defined(WARPX_DIM_3D) + const ParticleReal p_pos1 = p.pos(1); + const ParticleReal p_x_mean = p_pos0*p_w; + const ParticleReal p_y_mean = p_pos1*p_w; +#elif defined(WARPX_DIM_RZ) const ParticleReal p_theta = p.rdata(PIdx::theta); const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#else - const ParticleReal p_pos1 = p.pos(1); +#elif defined(WARPX_DIM_XZ) const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; + const ParticleReal p_y_mean = 0; +#elif defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(p_pos0); + const ParticleReal p_x_mean = 0; + const ParticleReal p_y_mean = 0; #endif const ParticleReal p_z_mean = p.pos(index_z)*p_w; @@ -263,7 +270,7 @@ void BeamRelevant::ComputeDiags (int step) if (w_sum < std::numeric_limits::min() ) { - for (auto& item: m_data) item = 0.0_rt; + for (auto& item: m_data) { item = 0.0_rt; } return; } diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index b9698d5ffd2..fe2040e50d2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -94,7 +94,7 @@ FieldMomentum::FieldMomentum (std::string rd_name) void FieldMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index c041a70aaef..9f45392bb0a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -340,8 +340,8 @@ bool FieldProbe::ProbeInDomain () const auto & warpx = WarpX::GetInstance(); int const lev = 0; const amrex::Geometry& gm = warpx.Geom(lev); - const auto prob_lo = gm.ProbLo(); - const auto prob_hi = gm.ProbHi(); + const auto *const prob_lo = gm.ProbLo(); + const auto *const prob_hi = gm.ProbHi(); /* * Determine if probe exists within simulation boundaries. During 2D simulations, @@ -628,14 +628,15 @@ void FieldProbe::ComputeDiags (int step) void FieldProbe::WriteToFile (int step) const { - if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) return; + if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) { return; } // loop over num valid particles to find the lowest particle ID for later sorting auto first_id = static_cast(m_data_out[0]); for (long int i = 0; i < m_valid_particles; i++) { - if (m_data_out[i*noutputs] < first_id) + if (m_data_out[i*noutputs] < first_id) { first_id = static_cast(m_data_out[i*noutputs]); + } } // Create a new array to store probe data ordered by id, which will be printed to file. diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 7843527cbf5..893b00a5f00 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -87,7 +87,7 @@ void LoadBalanceCosts::ComputeDiags (int step) int nBoxes = 0; for (int lev = 0; lev < nLevels; ++lev) { - const auto cost = WarpX::getCosts(lev); + auto *const cost = WarpX::getCosts(lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( cost, "ERROR: costs are not initialized on level " + std::to_string(lev) + " !"); nBoxes += cost->size(); @@ -282,7 +282,7 @@ void LoadBalanceCosts::WriteToFile (int step) const // get a reference to WarpX instance auto& warpx = WarpX::GetInstance(); - if (!ParallelDescriptor::IOProcessor()) return; + if (!ParallelDescriptor::IOProcessor()) { return; } // final step is a special case, fill jagged array with NaN if (m_intervals.nextContains(step+1) > warpx.maxStep()) @@ -347,7 +347,7 @@ void LoadBalanceCosts::WriteToFile (int step) const while (std::getline(ss, token, m_sep[0])) { cnt += 1; - if (ss.peek() == m_sep[0]) ss.ignore(); + if (ss.peek() == m_sep[0]) { ss.ignore(); } } // 2 columns for step, time; then nBoxes*nDatafields columns for data; diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index a132c135471..157b4bfb368 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -88,7 +88,7 @@ ParticleEnergy::ParticleEnergy (std::string rd_name) void ParticleEnergy::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index ecefd08eab7..512e6f1394a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -158,7 +158,7 @@ ParticleHistogram::ParticleHistogram (std::string rd_name): void ParticleHistogram::ComputeDiags (int step) { // Judge if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -223,14 +223,16 @@ void ParticleHistogram::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if (fun_filterparser(t, x, y, z, ux, uy, uz) == 0._rt) + if (do_parser_filter) { + if (fun_filterparser(t, x, y, z, ux, uy, uz) == 0._rt) { return; + } + } // continue function if particle is not filtered out auto const f = fun_partparser(t, x, y, z, ux, uy, uz); // determine particle bin int const bin = int(Math::floor((f-bin_min)/bin_size)); - if ( bin<0 || bin>=num_bins ) return; // discard if out-of-range + if ( bin<0 || bin>=num_bins ) { return; } // discard if out-of-range // add particle to histogram bin //! @todo performance: on CPU, we are probably faster by @@ -260,11 +262,11 @@ void ParticleHistogram::ComputeDiags (int step) Real f_max = 0.0_rt; for ( int i = 0; i < m_bin_num; ++i ) { - if ( m_data[i] > f_max ) f_max = m_data[i]; + if ( m_data[i] > f_max ) { f_max = m_data[i]; } } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_max > std::numeric_limits::min() ) m_data[i] /= f_max; + if ( f_max > std::numeric_limits::min() ) { m_data[i] /= f_max; } } return; } @@ -279,7 +281,7 @@ void ParticleHistogram::ComputeDiags (int step) } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_area > std::numeric_limits::min() ) m_data[i] /= f_area; + if ( f_area > std::numeric_limits::min() ) { m_data[i] /= f_area; } } return; } diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index dfb53b7e2c3..7cac712b4fc 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -61,8 +61,9 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) pp_rd_name.query("openpmd_backend", m_openpmd_backend); // pick first available backend if default is chosen - if( m_openpmd_backend == "default" ) + if( m_openpmd_backend == "default" ) { m_openpmd_backend = WarpXOpenPMDFileType(); + } pp_rd_name.add("openpmd_backend", m_openpmd_backend); // read species @@ -130,7 +131,7 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) void ParticleHistogram2D::ComputeDiags (int step) { // Judge if the diags should be done at this step - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // resize data array Array tlo{0,0}; // lower bounds @@ -215,9 +216,11 @@ void ParticleHistogram2D::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) + if (do_parser_filter) { + if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) { return; + } + } // continue function if particle is not filtered out auto const f_abs = fun_partparser_abs(t, x, y, z, ux, uy, uz, w); @@ -226,10 +229,10 @@ void ParticleHistogram2D::ComputeDiags (int step) // determine particle bin int const bin_abs = int(Math::floor((f_abs-bin_min_abs)/bin_size_abs)); - if ( bin_abs<0 || bin_abs>=num_bins_abs ) return; // discard if out-of-range + if ( bin_abs<0 || bin_abs>=num_bins_abs ) { return; } // discard if out-of-range int const bin_ord = int(Math::floor((f_ord-bin_min_ord)/bin_size_ord)); - if ( bin_ord<0 || bin_ord>=num_bins_ord ) return; // discard if out-of-range + if ( bin_ord<0 || bin_ord>=num_bins_ord ) { return; } // discard if out-of-range amrex::Real &data = d_table(bin_abs, bin_ord); amrex::HostDevice::Atomic::Add(&data, weight); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5b370d29979..5f278d48f14 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -116,7 +116,7 @@ ParticleMomentum::ParticleMomentum (std::string rd_name) void ParticleMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 01847a1ff12..09aa85586be 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -94,7 +94,7 @@ public: * This function queries deprecated input parameters and aborts * the run if one of them is specified. */ - void BackwardCompatibility (); + void BackwardCompatibility () const; }; diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 758b48b7262..ae6dae25a71 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -86,7 +86,7 @@ void ReducedDiags::LoadBalance () // load balancing operations } -void ReducedDiags::BackwardCompatibility () +void ReducedDiags::BackwardCompatibility () const { const amrex::ParmParse pp_rd_name(m_rd_name); std::vector backward_strings; @@ -116,7 +116,7 @@ void ReducedDiags::WriteToFile (int step) const ofs << WarpX::GetInstance().gett_new(0); // loop over data size and write - for (const auto& item : m_data) ofs << m_sep << item; + for (const auto& item : m_data) { ofs << m_sep << item; } // end loop over data size diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 70471eb3910..71a585aea35 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -309,7 +309,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - if ( slice_cr_ratio[idim] > 1) slice_cr_ratio[idim] = 1; + if ( slice_cr_ratio[idim] > 1) { slice_cr_ratio[idim] = 1; } // check for interpolation -- compute index lo with floor and ceil if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 2ae990851e2..91409bc294d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -66,7 +66,7 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in ParallelDescriptor::ReadAndBcastFile(DMFileName, fileCharPtr); const std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream DMFile(fileCharPtrString, std::istringstream::in); - if ( ! DMFile.good()) amrex::FileOpenFailed(DMFileName); + if ( ! DMFile.good()) { amrex::FileOpenFailed(DMFileName); } DMFile.exceptions(std::ios_base::failbit | std::ios_base::badbit); int nprocs_in_checkpoint; @@ -382,11 +382,13 @@ WarpX::InitFromCheckpoint () if (do_pml) { for (int lev = 0; lev < nlevs; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) - if (pml_rz[lev]) + if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + } #endif } } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4675e219fdd..e3b7b893d0a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -44,7 +44,7 @@ public: using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; WarpXParticleCounter (ParticleContainer* pc); - unsigned long GetTotalNumParticles () {return m_Total;} + [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} std::vector m_ParticleOffsetAtRank; std::vector m_ParticleSizeAtRank; @@ -205,15 +205,15 @@ private: * @param[in] varname name from WarpX * @param[out] field_name field name for openPMD-api output * @param[in] comp_name comp name for openPMD-api output - * @param[in] is_theta_mode indicate if this field will be output with theta - * modes (instead of a reconstructed 2D slice) + * @param[in] var_in_theta_mode indicate if this field will be output with theta + * modes (instead of a reconstructed 2D slice) */ void GetMeshCompNames ( int meshLevel, const std::string& varname, std::string& field_name, std::string& comp_name, - bool is_theta_mode + bool var_in_theta_mode ) const; /** This function sets up the entries for storing the particle positions and global IDs diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index a375266b1d3..71d96a47927 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -100,8 +100,9 @@ namespace detail std::string const & engine_type, std::map< std::string, std::string > const & engine_parameters) { - if (operator_type.empty() && engine_type.empty()) + if (operator_type.empty() && engine_type.empty()) { return "{}"; + } std::string options; std::string top_block; @@ -111,7 +112,7 @@ namespace detail std::string op_parameters; for (const auto& kv : operator_parameters) { - if (!op_parameters.empty()) op_parameters.append(",\n"); + if (!op_parameters.empty()) { op_parameters.append(",\n"); } op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -119,7 +120,7 @@ namespace detail std::string en_parameters; for (const auto& kv : engine_parameters) { - if (!en_parameters.empty()) en_parameters.append(",\n"); + if (!en_parameters.empty()) { en_parameters.append(",\n"); } en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -154,8 +155,9 @@ namespace detail } ] })END"; - if (!engine_type.empty() || !en_parameters.empty()) + if (!engine_type.empty() || !en_parameters.empty()) { op_block += ","; + } } // end operator string block // add the engine string block @@ -170,8 +172,9 @@ namespace detail "type": ")END"; en_block += engine_type + "\""; - if(!en_parameters.empty()) + if(!en_parameters.empty()) { en_block += ","; + } } // non-default engine parameters @@ -304,33 +307,29 @@ namespace detail getUnitDimension ( std::string const & record_name ) { - if( (record_name == "position") || (record_name == "positionOffset") ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "momentum" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -1.} - }; - else if( record_name == "charge" ) return { - {openPMD::UnitDimension::T, 1.}, - {openPMD::UnitDimension::I, 1.} - }; - else if( record_name == "mass" ) return { - {openPMD::UnitDimension::M, 1.} - }; - else if( record_name == "E" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -3.}, - {openPMD::UnitDimension::I, -1.}, - }; - else if( record_name == "B" ) return { - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::I, -1.}, - {openPMD::UnitDimension::T, -2.} - }; - else return {}; + if( (record_name == "position") || (record_name == "positionOffset") ) { + return {{openPMD::UnitDimension::L, 1.}}; + } else if( record_name == "momentum" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -1.}}; + } else if( record_name == "charge" ) { + return {{openPMD::UnitDimension::T, 1.}, + {openPMD::UnitDimension::I, 1.}}; + } else if( record_name == "mass" ) { + return {{openPMD::UnitDimension::M, 1.}}; + } else if( record_name == "E" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -3.}, + {openPMD::UnitDimension::I, -1.}}; + } else if( record_name == "B" ) { + return {{openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::I, -1.}, + {openPMD::UnitDimension::T, -2.}}; + } else { + return {}; + } } /** \brief For a given field that is to be written to an openPMD file, @@ -449,7 +448,7 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // default close is true bool callClose = true; // close BTD file only when isLastBTDFlush is true - if (isBTD and !isLastBTDFlush) callClose = false; + if (isBTD and !isLastBTDFlush) { callClose = false; } if (callClose) { if (m_Series) { GetIteration(m_CurrentStep, isBTD).close(); @@ -472,8 +471,9 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) void WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) { - if( isBTD && m_Series != nullptr ) + if( isBTD && m_Series != nullptr ) { return; // already open for this snapshot (aka timestep in lab frame) + } // either for the next ts file, // or init a single file for all ts @@ -482,10 +482,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - if ( m_Encoding == openPMD::IterationEncoding::fileBased ) + if ( m_Encoding == openPMD::IterationEncoding::fileBased ) { m_Series = nullptr; - else if ( m_Series != nullptr ) + } else if ( m_Series != nullptr ) { return; + } if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) @@ -504,8 +505,9 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) m_Series->setIterationEncoding( m_Encoding ); // input file / simulation setup author - if( !m_authors.empty()) + if( !m_authors.empty()) { m_Series->setAuthor( m_authors ); + } // more natural naming for PIC m_Series->setMeshesPath( "fields" ); // conform to ED-PIC extension of openPMD @@ -527,9 +529,11 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); - if (isBTD || use_pinned_pc) - if (!pinned_pc->isDefined()) + if (isBTD || use_pinned_pc) { + if (!pinned_pc->isDefined()) { continue; // Skip to the next particle container + } + } PinnedMemoryParticleContainer tmp = (isBTD || use_pinned_pc) ? pinned_pc->make_alike() : @@ -585,20 +589,30 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { particle_diags[i].m_diag_domain); if (isBTD || use_pinned_pc) { - tmp.copyParticles(*pinned_pc, true); - particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass); + particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + tmp.copyParticles(*pinned_pc, + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); + particlesConvertUnits(ConvertDirection::SI_to_WarpX, pinned_pc, mass); } else { particlesConvertUnits(ConvertDirection::WarpX_to_SI, pc, mass); using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; tmp.copyParticles(*pc, - [random_filter,uniform_filter,parser_filter,geometry_filter] - AMREX_GPU_HOST_DEVICE - (const SrcData& src, int ip, const amrex::RandomEngine& engine) - { - const SuperParticleType& p = src.getSuperParticle(ip); - return random_filter(p, engine) * uniform_filter(p, engine) - * parser_filter(p, engine) * geometry_filter(p, engine); - }, true); + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } @@ -682,8 +696,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // we will set up empty particles unless it's BTD, where we might add some in a following buffer dump // during this setup, we mark some particle properties as constant and potentially zero-sized bool doParticleSetup = true; - if (isBTD) + if (isBTD) { doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; + } // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 @@ -705,7 +720,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { auto offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed - if (isBTD) offset += ParticleFlushOffset; + if (isBTD) { offset += ParticleFlushOffset; } for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { auto const numParticleOnTile = pti.numParticles(); auto const numParticleOnTile64 = static_cast( numParticleOnTile ); @@ -713,7 +728,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - if (numParticleOnTile == 0) continue; + if (numParticleOnTile == 0) { continue; } contributed_particles = true; @@ -729,8 +744,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, new amrex::ParticleReal[numParticleOnTile], [](amrex::ParticleReal const *p) { delete[] p; } ); - for (auto i = 0; i < numParticleOnTile; i++) + for (auto i = 0; i < numParticleOnTile; i++) { z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} + } std::string const positionComponent = "z"; currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); } @@ -782,7 +798,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, for (auto i = 0; i < numParticleOnTile; i++) { ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); } @@ -810,7 +826,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { for( auto & [record_name, record] : currSpecies ) { for( auto & [comp_name, comp] : record ) { - if (comp.constant()) continue; + if (comp.constant()) { continue; } auto dtype = comp.getDatatype(); switch (dtype) { @@ -860,7 +876,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, const unsigned long long np, bool const isBTD) const { std::string options = "{}"; - if (isBTD) options = "{ \"resizable\": true }"; + if (isBTD) { options = "{ \"resizable\": true }"; } auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto dtype_int = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); // @@ -897,14 +913,16 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, [[maybe_unused]] const auto [_, newRecord] = addedRecords.insert(record_name); if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); - if( record_name == "weighting" ) + if( record_name == "weighting" ) { currRecord.setAttribute( "macroWeighted", 1u ); - else + } else { currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + } + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -920,10 +938,11 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -958,8 +977,9 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, [](amrex::ParticleReal const *p){ delete[] p; } ); - for( auto kk=0; kk(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); @@ -1014,7 +1034,7 @@ WarpXOpenPMDPlot::SetupPos ( currSpecies["position"][comp].resetDataset( realType ); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].resetDataset( idType ); } @@ -1026,7 +1046,7 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( amrex::ParticleReal const mass) { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}); - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles auto const positionComponents = detail::getParticlePositionComponentLabels(true); @@ -1147,17 +1167,20 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, const auto HalfFieldBoundarySize = static_cast(fieldBoundary.size() / 2u); - for (auto i = 0; i < HalfFieldBoundarySize; ++i) - if (m_fieldPMLdirections.at(i)) + for (auto i = 0; i < HalfFieldBoundarySize; ++i) { + if (m_fieldPMLdirections.at(i)) { fieldBoundary.at(i) = "open"; + } + } - for (int i = 0; i < HalfFieldBoundarySize; ++i) + for (int i = 0; i < HalfFieldBoundarySize; ++i) { if (period.isPeriodic(i)) { fieldBoundary.at(2u * i) = "periodic"; fieldBoundary.at(2u * i + 1u) = "periodic"; particleBoundary.at(2u * i) = "periodic"; particleBoundary.at(2u * i + 1u) = "periodic"; } + } meshes.setAttribute("fieldSolver", []() { switch (WarpX::electromagnetic_solver_id) { @@ -1174,10 +1197,10 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, meshes.setAttribute("fieldBoundary", fieldBoundary); meshes.setAttribute("particleBoundary", particleBoundary); meshes.setAttribute("currentSmoothing", []() { - if (WarpX::use_filter) return "Binomial"; - else return "none"; + if (WarpX::use_filter) { return "Binomial"; } + else { return "none"; } }()); - if (WarpX::use_filter) + if (WarpX::use_filter) { meshes.setAttribute("currentSmoothingParameters", []() { std::stringstream ss; ss << "period=1;compensator=false"; @@ -1195,12 +1218,14 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, std::string currentSmoothingParameters = ss.str(); return currentSmoothingParameters; }()); + } meshes.setAttribute("chargeCorrection", []() { - if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check - else return "none"; + if (WarpX::do_dive_cleaning) { return "hyperbolic"; // TODO or "spectral" or something? double-check + } else { return "none"; } }()); - if (WarpX::do_dive_cleaning) + if (WarpX::do_dive_cleaning) { meshes.setAttribute("chargeCorrectionParameters", "period=1"); + } } @@ -1279,8 +1304,9 @@ WarpXOpenPMDPlot::GetMeshCompNames (int meshLevel, } } - if ( 0 == meshLevel ) + if ( 0 == meshLevel ) { return; + } field_name += std::string("_lvl").append(std::to_string(meshLevel)); } @@ -1363,19 +1389,21 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, } // If there are no fields to be written, interrupt the function here - if ( varnames.empty() ) return; + if ( varnames.empty() ) { return; } // loop over levels up to output_levels // note: this is usually the finestLevel, not the maxLevel for (int lev=0; lev < output_levels; lev++) { amrex::Geometry full_geom = geom[lev]; - if( isBTD ) + if( isBTD ) { full_geom = full_BTD_snapshot; + } // setup is called once. So it uses property "period" from first // geometry for field levels. - if ( (0 == lev) && first_write_to_iteration ) + if ( (0 == lev) && first_write_to_iteration ) { SetupFields(meshes, full_geom); + } amrex::Box const & global_box = full_geom.Domain(); @@ -1507,8 +1535,9 @@ WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc): m_ParticleSizeAtRank[currentLevel] = numParticles; // adjust offset, it should be numbered after particles from previous levels - for (auto lv=0; lv(result.size()); for (int i=0; iPushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - if (fft_do_time_averaging) + if (evolve_scheme == EvolveScheme::Explicit) { + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + if (is_synchronized) { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) - FillBoundaryAux(guard_cells.ng_UpdateAux); + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) HybridPICDepositInitialRhoAndJ(); + ) { HybridPICDepositInitialRhoAndJ(); } // Run multi-physics modules: // ionization, Coulomb collisions, QED @@ -182,11 +185,16 @@ WarpX::Evolve (int numsteps) // gather fields, push particles, deposit sources, update fields ExecutePythonCallback("particleinjection"); - // Electrostatic or hybrid-PIC case: only gather fields and push - // particles, deposition and calculation of fields done further below - if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + OneStep_ImplicitPicard(cur_time); + } + else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) { + // Electrostatic or hybrid-PIC case: only gather fields and push + // particles, deposition and calculation of fields done further below const bool skip_deposition = true; PushParticlesandDeposit(cur_time, skip_deposition); } @@ -220,31 +228,33 @@ WarpX::Evolve (int numsteps) // value of step in code (first step is 0) mypc->doResampling(istep[0]+1, verbose); - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } - - if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + if (evolve_scheme == EvolveScheme::Explicit) { + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + + if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { + // At the end of last step, push p by 0.5*dt to synchronize + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP(lev, 0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); + } + is_synchronized = true; } - is_synchronized = true; } for (int lev = 0; lev <= max_level; ++lev) { @@ -404,7 +414,7 @@ WarpX::Evolve (int numsteps) if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) || exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) ExecutePythonCallback("onbreaksignal"); + if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -439,8 +449,8 @@ WarpX::OneStep_nosub (Real cur_time) // solve. // For extended PML: copy J from regular grid to PML, and damp J in PML - if (do_pml && pml_has_particles) CopyJPML(); - if (do_pml && do_pml_j_damping) DampJPML(); + if (do_pml && pml_has_particles) { CopyJPML(); } + if (do_pml && do_pml_j_damping) { DampJPML(); } ExecutePythonCallback("beforeEsolve"); @@ -467,10 +477,12 @@ WarpX::OneStep_nosub (Real cur_time) else { FillBoundaryE(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } } } else { EvolveF(0.5_rt * dt[0], DtType::FirstHalf); @@ -506,8 +518,9 @@ WarpX::OneStep_nosub (Real cur_time) // E and B are up-to-date in the domain, but all guard cells are // outdated. - if (safe_guard_cells) + if (safe_guard_cells) { FillBoundaryB(guard_cells.ng_alloc_EB); + } } // !PSATD ExecutePythonCallback("afterEsolve"); @@ -549,7 +562,7 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) ApplyFilterJ(current_fp_vay, lev); + if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } } } } @@ -603,11 +616,11 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // 1) Prepare E,B,F,G fields in spectral space PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // 2) Set the averaged fields to zero - if (WarpX::fft_do_time_averaging) PSATDEraseAverageFields(); + if (WarpX::fft_do_time_averaging) { PSATDEraseAverageFields(); } // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) @@ -650,7 +663,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) for (int i_deposit = 0; i_deposit < n_loop; i_deposit++) { // Move J from new to old if J is linear in time - if (J_in_time == JInTime::Linear) PSATDMoveJNewToJOld(); + if (J_in_time == JInTime::Linear) { PSATDMoveJNewToJOld(); } const amrex::Real t_deposit_current = (J_in_time == JInTime::Linear) ? (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; @@ -676,7 +689,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (rho_fp[0]) { // Move rho from new to old if rho is linear in time - if (rho_in_time == RhoInTime::Linear) PSATDMoveRhoNewToRhoOld(); + if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } // Deposit rho at relative time t_deposit_charge mypc->DepositCharge(rho_fp, t_deposit_charge); @@ -701,8 +714,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (i_deposit == n_deposit-1) { PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } } @@ -722,9 +735,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } // Damp fields in PML before exchanging guard cells @@ -736,10 +749,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Exchange guard cells and synchronize nodal points FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } #else amrex::ignore_unused(cur_time); @@ -764,7 +779,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) * */ void -WarpX::OneStep_sub1 (Real curtime) +WarpX::OneStep_sub1 (Real cur_time) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, @@ -778,10 +793,10 @@ WarpX::OneStep_sub1 (Real curtime) const int coarse_lev = 0; // i) Push particles and fields on the fine patch (first fine step) - PushParticlesandDeposit(fine_lev, curtime, DtType::FirstHalf); + PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); @@ -809,7 +824,7 @@ WarpX::OneStep_sub1 (Real curtime) // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid // by only half a coarse step (first half) - PushParticlesandDeposit(coarse_lev, curtime, DtType::Full); + PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); @@ -839,10 +854,10 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryAux(guard_cells.ng_UpdateAux); // iv) Push particles and fields on the fine patch (second fine step) - PushParticlesandDeposit(fine_lev, curtime + dt[fine_lev], DtType::SecondHalf); + PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); @@ -863,8 +878,9 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid @@ -910,13 +926,15 @@ WarpX::OneStep_sub1 (Real curtime) WarpX::sync_nodal_points); } DampPML(coarse_lev, PatchType::fine); - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } void @@ -954,17 +972,18 @@ WarpX::doQEDEvents (int lev) #endif void -WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current) +WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current, PushType push_type) { // Evolve particles to p^{n+1/2} and x^{n+1} // Deposit current, j^{n+1/2} for (int lev = 0; lev <= finest_level; ++lev) { - PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current); + PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current, push_type); } } void -WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current) +WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, + PushType push_type) { amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; @@ -998,7 +1017,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, rho_fp[lev].get(), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_current); + cur_time, dt[lev], a_dt_type, skip_current, push_type); if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. @@ -1075,8 +1094,8 @@ WarpX::applyMirrors(Real time) NullifyMF(Bz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) NullifyMF(*F_fp[lev], lev, z_min, z_max); - if (G_fp[lev]) NullifyMF(*G_fp[lev], lev, z_min, z_max); + if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } + if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } if (lev>0) { @@ -1097,8 +1116,8 @@ WarpX::applyMirrors(Real time) NullifyMF(cBz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) NullifyMF(*F_cp[lev], lev, z_min, z_max); - if (G_cp[lev]) NullifyMF(*G_cp[lev], lev, z_min, z_max); + if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } + if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } } } } diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp new file mode 100644 index 00000000000..56ed26db1e2 --- /dev/null +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -0,0 +1,423 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::EvolveImplicitPicardInit (int lev) +{ + + if (lev == 0) { + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + } + + // Initialize MultiFabs to hold the E and B fields at the start of the time steps + // Only one refinement level is supported + const int nlevs_max = maxLevel() + 1; + Efield_n.resize(nlevs_max); + Efield_save.resize(nlevs_max); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_n.resize(nlevs_max); + Bfield_save.resize(nlevs_max); + } + + // The Efield_n and Bfield_n will hold the fields at the start of the time step. + // This is needed since in each iteration the fields are advanced from the values + // at the start of the step. + // The Efield_save and Bfield_save will hold the fields from the previous iteration, + // to check the change in the fields after the iterations to check for convergence. + // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then + // advance to the n+1 time level after the iterations complete. + AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); + AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); + AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); + AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); + AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); + AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); + AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); + AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); + AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); + AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); + AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); + } + +} + +void +WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) +{ + using namespace amrex::literals; + + // We have E^{n}. + // Particles have p^{n} and x^{n}. + // With full implicit, B^{n} + // With semi-implicit, B^{n-1/2} + + // Save the values at the start of the time step, + // copying particle data to x_n etc. + for (auto const& pc : *mypc) { + SaveParticlesAtImplicitStepStart (*pc, 0); + } + + // Save the fields at the start of the step + amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); + } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // Start the iterations + amrex::Real deltaE = 1._rt; + amrex::Real deltaB = 1._rt; + int iteration_count = 0; + while (iteration_count < max_picard_iterations && + (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { + iteration_count++; + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + bool skip_current = false; + PushType push_type = PushType::Implicit; + PushParticlesandDeposit(cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the E at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place + amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); + + // Updates Efield_fp so it holds the new E at n+1/2 + EvolveE(0.5_rt*dt[0]); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the B at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place + amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); + + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(0.5_rt*dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // The B field update needs + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date from the mirrors + // B : guard cells are NOT up-to-date from the mirrors + } + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Calculate the change in E and B from this iteration + // deltaE = abs(Enew - Eold)/max(abs(Enew)) + Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); + Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); + Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); + amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); + amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); + amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; + amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; + amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; + deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); + Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); + Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); + amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); + amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); + amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; + amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; + amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; + deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); + } else { + deltaB = 0.; + } + amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; + } + + // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold + // the new values at n+1/2 + } + + amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; + if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { + std::stringstream convergenceMsg; + convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; + if (require_picard_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + + // Advance particles to step n+1 + for (auto const& pc : *mypc) { + FinishImplicitParticleUpdate(*pc, 0); + } + + // Advance fields to step n+1 + // WarpX::sync_nodal_points is used to avoid instability + FinishImplicitFieldUpdate(Efield_fp, Efield_n); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + } + +} + +void +WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } +} + +void +WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) +{ + using namespace amrex::literals; + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } +} + +void +WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, + amrex::Vector, 3 > >& Field_n) +{ + using namespace amrex::literals; + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); + amrex::Box tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); + amrex::Box tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); + }); + } + } +} diff --git a/Source/Evolve/WarpXPushType.H b/Source/Evolve/WarpXPushType.H new file mode 100644 index 00000000000..dbca64a398c --- /dev/null +++ b/Source/Evolve/WarpXPushType.H @@ -0,0 +1,18 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PUSHTYPE_H_ +#define WARPX_PUSHTYPE_H_ + +// Specify which scheme to use for the particle advance +enum struct PushType : int +{ + Explicit = 0, // Use the standard leap-frog scheme + Implicit // Use the Crank-Nicolson scheme. + // See for example Eqs. 15-18 in Chen, JCP 407 (2020) 109228 +}; + +#endif // WARPX_PUSHTYPE_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 7bd982bb4e0..7fc6c9bf6da 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -52,7 +52,7 @@ namespace ElectrostaticSolver { void buildParsers (); void buildParsersEB (); - /* \brief Sets the EB potential string and updates the parsers + /* \brief Sets the EB potential string and updates the function parser * * \param [in] potential The string value of the potential */ diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 6b20d7772d9..bd3086c0438 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -105,8 +105,9 @@ WarpX::AddBoundaryField () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } // Allocate fields for charge and potential const int num_levels = max_level + 1; @@ -146,8 +147,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -181,7 +183,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); std::array beta; - for (int i=0 ; i < static_cast(beta.size()) ; i++) beta[i] = beta_pr[i]/PhysConst::c; // Normalize + for (int i=0 ; i < static_cast(beta.size()) ; i++) { + beta[i] = beta_pr[i]/PhysConst::c; // Normalize + } // Compute the potential phi, by solving the Poisson equation computePhi( rho, phi, beta, pc.self_fields_required_precision, @@ -201,8 +205,9 @@ WarpX::AddSpaceChargeFieldLabFrame () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -381,7 +386,7 @@ void WarpX::setPhiBC ( amrex::Vector>& phi ) const { // check if any dimension has non-periodic boundary conditions - if (!m_poisson_boundary_handler.has_non_periodic) return; + if (!m_poisson_boundary_handler.has_non_periodic) { return; } // get the boundary potentials at the current time amrex::Array phi_bc_values_lo; @@ -417,7 +422,7 @@ WarpX::setPhiBC ( amrex::Vector>& phi ) const // loop over dimensions for (int idim=0; idim, 3> > std::array const beta ) const { // return early if beta is 0 since there will be no B-field - if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) return; + if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } for (int lev = 0; lev <= max_level; lev++) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index b447d4e06a3..36be09696fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -106,13 +106,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) - coef2_z*Et(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) + coef2_z*Et(i,j+1,0,m); + } } }, @@ -120,13 +122,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) + coef2_z*Er(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) - coef2_z*Er(i,j+1,0,m); + } } // At the +r boundary (innermost guard cell) if ( apply_hi_r && (i==domain_box.bigEnd(0)+1) ){ @@ -233,31 +237,39 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) + coef2_y * Ez(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) - coef2_y * Ez(i,j+1,k); + } // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i+1,j,k); + } #endif }, @@ -266,33 +278,41 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) - coef2_x * Ez(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) + coef2_x * Ez(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i+1,j,k); + } #endif }, @@ -301,19 +321,23 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) + coef2_x * Ey(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) - coef2_x * Ey(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) - coef2_y * Ex(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) + coef2_y * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z ignore_unused(i,j,k); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 612267b50a3..861b2648c1e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -145,7 +145,7 @@ class FiniteDifferenceSolver * \param[in] Pefield scalar electron pressure MultiFab at a given level * \param[in] edge_lengths length of edges along embedded boundaries * \param[in] lev level number for the calculation - * \param[in] hybrid_pic_model instance of the hybrid-PIC model + * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] include_resistivity_term boolean flag for whether to include resistivity */ void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, @@ -156,7 +156,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); /** @@ -240,7 +240,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template @@ -345,7 +345,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 8923ec1f6a5..1bbc6c9b337 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -36,8 +36,9 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( m_grid_type{grid_type} { // return if not FDTD - if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) + if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) { return; + } // Calculate coefficients of finite-difference stencil #ifdef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 22f6e130486..750a5e06933 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -121,12 +121,12 @@ public: * charge density (and assumption of quasi-neutrality) using the user * specified electron equation of state. * - * \param[out] Pe scalar electron pressure MultiFab at a given level - * \param[in] rhofield scalar ion charge density Multifab at a given level + * \param[out] Pe_filed scalar electron pressure MultiFab at a given level + * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe, - amrex::MultiFab* const& rhofield ); + std::unique_ptr const& Pe_field, + amrex::MultiFab* const& rho_field ) const; // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 57999335ae6..2cc6a8ec437 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -242,7 +242,7 @@ void HybridPICModel::InitData () void HybridPICModel::GetCurrentExternal ( amrex::Vector, 3>> const& edge_lengths) { - if (!m_external_field_has_time_dependence) return; + if (!m_external_field_has_time_dependence) { return; } auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) @@ -414,7 +414,7 @@ void HybridPICModel::CalculateCurrentAmpere ( // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); + for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } } void HybridPICModel::HybridPICSolveE ( @@ -508,7 +508,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) void HybridPICModel::FillElectronPressureMF ( std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) + amrex::MultiFab* const& rho_field ) const { const auto n0_ref = m_n0_ref; const auto elec_temp = m_elec_temp; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 84d5b773097..1a72fee53c2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -590,7 +590,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0); @@ -601,7 +601,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); + if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); } }, // Et calculation @@ -623,7 +623,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure // -> d/dt = 0 for m = 0 @@ -635,20 +635,20 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); + if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,0) <= 0) return; + if (lz(i,j,0) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); @@ -659,7 +659,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } } ); @@ -854,7 +854,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k); @@ -865,7 +865,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); + if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); } }, // Ey calculation @@ -873,18 +873,18 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; + if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } #endif #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k); @@ -895,20 +895,20 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); + if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; + if (lz(i,j,k) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k); @@ -919,7 +919,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } } ); diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index bc51d65b536..192017656ce 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -206,7 +206,7 @@ void WarpX::setVectorPotentialBC ( amrex::Vector,3>>& A ) const { // check if any dimension has non-periodic boundary conditions - if (!m_vector_poisson_boundary_handler.has_non_periodic) return; + if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } auto dirichlet_flag = m_vector_poisson_boundary_handler.dirichlet_flag; @@ -228,7 +228,7 @@ WarpX::setVectorPotentialBC ( amrex::Vector= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; @@ -390,15 +390,15 @@ SpectralFieldData::BackwardTransform (const int lev, Complex spectral_field_value = field_arr(i,j,k,field_index); // Apply proper shift in each dimension #if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into temporary array tmp_arr(i,j,k) = spectral_field_value; @@ -447,7 +447,7 @@ SpectralFieldData::BackwardTransform (const int lev, { for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - if ((fill_guards[dir]) == 0) mf_box.grow(dir, -mf_ng[dir]); + if ((fill_guards[dir]) == 0) { mf_box.grow(dir, -mf_ng[dir]); } } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index 1268d277e63..c4a693bf25a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -331,7 +331,7 @@ SpectralFieldDataRZ::FABZForwardTransform (amrex::MFIter const & mfi, amrex::Box [=] AMREX_GPU_DEVICE(int i, int j, int k, int mode) noexcept { Complex spectral_field_value = tmp_arr(i,j,k,mode); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the correct index. int const ic = field_index + mode*n_fields; fields_arr(i,j,k,ic) = spectral_field_value*inv_nz; @@ -369,7 +369,7 @@ SpectralFieldDataRZ::FABZBackwardTransform (amrex::MFIter const & mfi, amrex::Bo int const ic = field_index + mode*n_fields; Complex spectral_field_value = fields_arr(i,j,k,ic); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the right index. tmp_arr(i,j,k,mode) = spectral_field_value; }); diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp index c2309512596..39782408eb0 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp @@ -58,7 +58,7 @@ namespace{ q0 = static_cast(jn(n, p0)); q1 = static_cast(jn(n, p1)); for (int it=1; it <= nitmx; it++) { - if (q1 == q0) break; + if (q1 == q0) { break; } p = p1 - q1*(p1 - p0)/(q1 - q0); dp = p - p1; if (it > 1 && std::abs(dp) < tol) { @@ -142,7 +142,7 @@ void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vec const auto errj = static_cast(std::abs(jn(n, zeroj))); // improve solution using procedure SecantRootFinder - if (errj > tol) ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + if (errj > tol) { ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); } roots[ik] = zeroj; ier[ik] = ierror; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fa9d1860053..bb9ec04f266 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -183,11 +183,11 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } } } } @@ -200,17 +200,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } #else - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } #else - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } #endif } } @@ -229,11 +229,11 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } } } } @@ -246,17 +246,17 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } #else - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } #else - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } #endif } } @@ -370,15 +370,15 @@ void WarpX::PSATDForwardTransformRho ( const amrex::Vector>& charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) return; + if (charge_fp[0] == nullptr) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); + if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); + if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } } } @@ -759,22 +759,23 @@ WarpX::PushPSATD () PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); #ifdef WARPX_DIM_RZ - if (pml_rz[0]) pml_rz[0]->PushPSATD(0); + if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } #endif // FFT of F and G - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // Update E, B, F, and G in k-space PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::fft_do_time_averaging) + if (WarpX::fft_do_time_averaging) { PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + } + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } // Evolve the fields in the PML boxes for (int lev = 0; lev <= finest_level; ++lev) @@ -784,9 +785,9 @@ WarpX::PushPSATD () pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } #endif } @@ -916,7 +917,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) void WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -927,16 +928,16 @@ WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveF (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } EvolveF(lev, PatchType::fine, a_dt, a_dt_type); - if (lev > 0) EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); + if (lev > 0) { EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); } } void WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveF()"); @@ -966,7 +967,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ void WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -977,7 +978,7 @@ WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } EvolveG(lev, PatchType::fine, a_dt, a_dt_type); @@ -990,7 +991,7 @@ WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_type*/) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveG()"); @@ -1076,8 +1077,8 @@ WarpX::DampFieldsInGuards(const int lev, { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for ( amrex::MFIter mfi(*Efield[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -1170,8 +1171,8 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& for (int iside = 0; iside < 2; iside++) { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 00c2ddab73a..9252671d1ff 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -47,9 +47,11 @@ void WarpX::HybridPICEvolveFields () // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to // a nodal grid - for (int lev = 0; lev <= finest_level; ++lev) - for (int idim = 0; idim < 3; ++idim) + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + } + } // Get requested number of substeps to use int sub_steps = m_hybrid_pic_model->m_substeps / 2; diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index ecfee2cd7c9..a095bce6ae3 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -35,7 +35,7 @@ namespace { for(int ipass=1; ipass< lastpass; ipass++){ // element 0 has to be treated in its own way new_s.at(0) = 0.5_rt * old_s.at(0); - if (1(el); + } stencil_length_each_dir += 1.; #if defined(WARPX_DIM_3D) // npass_each_dir = npass_x npass_y npass_z diff --git a/Source/Fluids/MusclHancockUtils.H b/Source/Fluids/MusclHancockUtils.H index 764143a7220..2765f28f3b3 100644 --- a/Source/Fluids/MusclHancockUtils.H +++ b/Source/Fluids/MusclHancockUtils.H @@ -44,12 +44,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::min(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::max(a, b); - else + } else { return 0.0_rt; + } } // Min of 3 inputs AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -68,24 +69,26 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod3 (amrex::Real a, amrex::Real b , amrex::Real c) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) { return min3(a,b,c); - else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) { return max3(a,b,c); - else + } else { return 0.0; + } } //maxmod AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real maxmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::max(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::min(a, b); - else + } else { return 0.0_rt; + } } // Rusanov Flux (density) AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -132,30 +135,33 @@ amrex::Real ave_adjustable_diff (amrex::Real a, amrex::Real b) { using namespace amrex::literals; constexpr auto sigma = static_cast(2.0*0.732050807568877); - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, sigma*a, sigma*b ); - else + } else { return 0.0_rt; + } } // ave_minmod Low diffusivity AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, 2.0_rt*a, 2.0_rt*b ); - else + } else { return 0.0_rt; + } } // ave_superbee AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave_superbee (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod( maxmod(a,b), minmod(2.0_rt*a,2.0_rt*b)); - else + } else { return 0.0_rt; + } } // stage2 slope limiting AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -379,7 +385,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -396,7 +402,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -414,7 +420,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -431,7 +437,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -450,11 +456,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k-1) > 0) U_m = NU(i,j,k-1)/N(i,j,k-1); + if (N(i,j,k-1) > 0) { U_m = NU(i,j,k-1)/N(i,j,k-1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } #else - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } #endif // Return the difference @@ -471,11 +477,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k+1) > 0) U_p = NU(i,j,k+1)/N(i,j,k+1); + if (N(i,j,k+1) > 0) { U_p = NU(i,j,k+1)/N(i,j,k+1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } #else - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } #endif // Return the difference diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 17d541f180f..04ec4d9e80d 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -99,12 +99,12 @@ public: * \param[in] Bx Yee magnetic field (x) * \param[in] By Yee magnetic field (y) * \param[in] Bz Yee magnetic field (z) - * \param[in] cur_time Current time + * \param[in] t Current time */ void GatherAndPush (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::Real cur_time); + amrex::Real t); /** * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 32edcad23e7..1915e4bf772 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -821,10 +821,12 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Radial Surfaces amrex::Real S_Ar_plus = 2.0_rt*MathConst::pi*(r + dr/2.0_rt)*dz; amrex::Real S_Ar_minus = 2.0_rt*MathConst::pi*(r - dr/2.0_rt)*dz; - if (i == domain.smallEnd(0)) + if (i == domain.smallEnd(0)) { S_Ar_minus = 0.0_rt; - if (i == domain.bigEnd(0)+1) + } + if (i == domain.bigEnd(0)+1) { S_Ar_plus = 2.0_rt*MathConst::pi*(r)*dz; + } // Impose "none" boundaries // Condition: Vx(r) = 0 at boundaries @@ -1222,7 +1224,7 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if ( owner_mask_rho_arr(i,j,k) ) rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); + if ( owner_mask_rho_arr(i,j,k) ) { rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); } } ); } @@ -1333,19 +1335,19 @@ void WarpXFluidContainer::DepositCurrent( { amrex::Real jx_tmp = ablastr::coarsen::sample::Interp(tmp_jx_fluid_arr, j_nodal_type, jx_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_x_arr(i,j,k) ) jx_arr(i, j, k) += jx_tmp; + if ( owner_mask_x_arr(i,j,k) ) { jx_arr(i, j, k) += jx_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jy_tmp = ablastr::coarsen::sample::Interp(tmp_jy_fluid_arr, j_nodal_type, jy_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_y_arr(i,j,k) ) jy_arr(i, j, k) += jy_tmp; + if ( owner_mask_y_arr(i,j,k) ) { jy_arr(i, j, k) += jy_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jz_tmp = ablastr::coarsen::sample::Interp(tmp_jz_fluid_arr, j_nodal_type, jz_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_z_arr(i,j,k) ) jz_arr(i, j, k) += jz_tmp; + if ( owner_mask_z_arr(i,j,k) ) { jz_arr(i, j, k) += jz_tmp; } } ); } diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index 04d34c59a32..909e5c01977 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -76,15 +76,17 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) // if the input string is "constant", the values for the // external grid must be provided in the input. auto v_B = std::vector(3); - if (B_ext_grid_type == ExternalFieldType::constant) + if (B_ext_grid_type == ExternalFieldType::constant) { utils::parser::getArrWithParser(pp_warpx, "B_external_grid", v_B); + } std::copy(v_B.begin(), v_B.end(), B_external_grid.begin()); // if the input string is "constant", the values for the // external grid must be provided in the input. auto v_E = std::vector(3); - if (E_ext_grid_type == ExternalFieldType::constant) + if (E_ext_grid_type == ExternalFieldType::constant) { utils::parser::getArrWithParser(pp_warpx, "E_external_grid", v_E); + } std::copy(v_E.begin(), v_E.end(), E_external_grid.begin()); //___________________________________________________________________________ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 98caae1bb23..f7ee6cff138 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -130,7 +130,7 @@ namespace { u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); // Rejection method xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) reject = false; + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } } } else { // Mean velocity magnitude is greater than thermal velocity @@ -151,7 +151,7 @@ namespace { } // Rejection method const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) reject = false; + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } } } @@ -197,7 +197,7 @@ struct InjectorMomentumGaussianFlux u_th = m_uz_th; } amrex::Real u = generateGaussianFluxDist(u_m, u_th, engine); - if (m_flux_direction < 0) u = -u; + if (m_flux_direction < 0) { u = -u; } // Note: Here, in RZ geometry, the variables `ux` and `uy` actually // correspond to the radial and azimuthal component of the momentum @@ -338,7 +338,7 @@ struct InjectorMomentumBoltzmann { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } const amrex::Real beta = velocity(x,y,z); int const dir = velocity.direction(); const auto gamma = 1._rt/std::sqrt(1._rt-beta*beta); @@ -441,7 +441,7 @@ struct InjectorMomentumJuttner { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } amrex::Real const beta = velocity(x,y,z); int const dir = velocity.direction(); auto const gamma = 1._rt/std::sqrt(1._rt-beta*beta); diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp index 35513260827..4642d3a0cc6 100644 --- a/Source/Initialization/InjectorMomentum.cpp +++ b/Source/Initialization/InjectorMomentum.cpp @@ -9,7 +9,7 @@ using namespace amrex; -void InjectorMomentum::clear () +void InjectorMomentum::clear () // NOLINT(readability-make-member-function-const) { switch (type) { diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 64cc1a2a620..c86ed432bd1 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -43,19 +43,19 @@ struct InjectorPositionRandomPlane using namespace amrex::literals; #if ((defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) // In RZ, the 3 components of the `XDim3` vector below correspond to r, theta, z respectively - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; - else return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; } + else { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } #elif (defined(WARPX_DIM_XZ)) // In 2D, the 2 first components of the `XDim3` vector below correspond to x and z - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; - else return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } + else { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt }; } #elif (defined(WARPX_DIM_1D_Z)) // In 2D, the first components of the `XDim3` vector below correspond to z - if (dir == 0) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - else return amrex::XDim3{0._rt, 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + else { return amrex::XDim3{0._rt, 0._rt, 0._rt}; } #endif } private: diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 3b2ad76513a..008709abe48 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -501,18 +501,15 @@ void PlasmaInjector::setupExternalFile (amrex::ParmParse const& pp_species) } // IOProcessor // Broadcast charge and mass to non-IO processors if read in from the file - if (!charge_is_specified && !species_is_specified) { - // Use ReduceBoolOr since Bcast(bool) doesn't compile - amrex::ParallelDescriptor::ReduceBoolOr(charge_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); - if (charge_from_source) { - amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); - } + std::array flags{charge_from_source, mass_from_source}; + amrex::ParallelDescriptor::Bcast(flags.data(), flags.size(), amrex::ParallelDescriptor::IOProcessorNumber()); + charge_from_source = flags[0]; + mass_from_source = flags[1]; + if (charge_from_source) { + amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); } - if (!mass_is_specified && !species_is_specified) { - amrex::ParallelDescriptor::ReduceBoolOr(mass_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); - if (mass_from_source) { - amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); - } + if (mass_from_source) { + amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); } #else WARPX_ABORT_WITH_MESSAGE( diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 50c90d40ae6..613250b5c2f 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -7,8 +7,6 @@ #ifndef WARPX_AMREX_INIT_H_ #define WARPX_AMREX_INIT_H_ -#include - #include namespace warpx::initialization diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index c153595604f..d1dd0bbe90b 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -8,6 +8,7 @@ #include "Initialization/WarpXAMReXInit.H" #include +#include #include #include diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bd8e6334f48..169453a6e99 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -476,8 +476,9 @@ WarpX::InitData () ExecutePythonCallback("beforeInitEsolve"); ComputeSpaceChargeField(reset_fields); ExecutePythonCallback("afterInitEsolve"); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); + } // Set up an invariant condition through the rest of // execution, that any code besides the field solver that @@ -552,7 +553,7 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) do_pml = 1; + if (finest_level > 0) { do_pml = 1; } if (do_pml) { #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -583,10 +584,12 @@ WarpX::InitPML () // Domain box at level, lev const amrex::Box DomainBox = Geom(lev).Domain(); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) + if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) { do_pml_Lo[lev][idim] = do_pml_Lo[0][idim]; - if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) + } + if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) { do_pml_Hi[lev][idim] = do_pml_Hi[0][idim]; + } } #ifdef WARPX_DIM_RZ @@ -619,8 +622,9 @@ WarpX::ComputePMLFactors () { for (int lev = 0; lev <= finest_level; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); + } } } } @@ -724,6 +728,9 @@ void WarpX::PostRestart () { mypc->PostRestart(); + for (int lev = 0; lev <= maxLevel(); ++lev) { + LoadExternalFieldsFromFile(lev); + } } @@ -745,14 +752,16 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } if (lev > 0) { Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } } } @@ -764,14 +773,16 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } if (lev > 0) { Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } } } } @@ -893,34 +904,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - // Reading external fields from data file - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#endif - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); -#else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); -#endif - } + LoadExternalFieldsFromFile(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1316,6 +1300,37 @@ void WarpX::CheckKnownIssues() #endif } +void +WarpX::LoadExternalFieldsFromFile (int const lev) +{ + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#endif + } + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } +} + #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( @@ -1391,12 +1406,12 @@ WarpX::ReadExternalFieldFromFile ( auto FC_chunk_data = FC.loadChunk(chunk_offset,chunk_extent); series.flush(); - auto FC_data_host = FC_chunk_data.get(); + auto *FC_data_host = FC_chunk_data.get(); // Load data to GPU const size_t total_extent = size_t(extent[0]) * extent[1] * extent[2]; amrex::Gpu::DeviceVector FC_data_gpu(total_extent); - auto FC_data = FC_data_gpu.data(); + auto *FC_data = FC_data_gpu.data(); amrex::Gpu::copy(amrex::Gpu::hostToDevice, FC_data_host, FC_data_host + total_extent, FC_data); // Loop over boxes diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index e29439c4bc0..0fdb45c64f8 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -114,8 +114,9 @@ void WarpXLaserProfiles::FromFileLaserProfile::update (amrex::Real t) { t += m_params.t_min - m_params.t_delay; - if(t >= m_params.t_max) + if(t >= m_params.t_max) { return; + } const auto idx_times = find_left_right_time_indices(t); const auto idx_t_left = idx_times.first; const auto idx_t_right = idx_times.second; @@ -186,8 +187,8 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ m_params.n_rz_azimuthal_components = static_cast(extent[0]); m_params.nt = static_cast(extent[1]); m_params.nr = static_cast(extent[2]); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); - if(m_params.nr <= 1) WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); } + if(m_params.nr <= 1) { WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); } // Calculate the min and max of the grid m_params.t_min = static_cast(offset[0] + position[0]*spacing[0]); m_params.t_max = static_cast(m_params.t_min + (m_params.nt-1)*spacing[0]); @@ -240,24 +241,24 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); //Uniform grid flag char flag; inp.read(&flag, 1); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flag, "Binary files with non uniform grid are no longer supported"); //Grid points along t, x and y inp.read(reinterpret_cast(&m_params.nt), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.nx), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.ny), sizeof(uint32_t)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); - if(m_params.nx <= 1) WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); } + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); } + if(m_params.nx <= 1) { WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); } #if (defined(WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) - if(m_params.ny <= 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); + if(m_params.ny <= 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); } #elif defined(WARPX_DIM_XZ) - if(m_params.ny != 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); + if(m_params.ny != 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); } #endif //Coordinates Vector dbuf_t, dbuf_x, dbuf_y; @@ -274,7 +275,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ static_cast(dbuf_x.size()*sizeof(double))); inp.read(reinterpret_cast(dbuf_y.dataPtr()), static_cast(dbuf_y.size()*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); } m_params.t_min = static_cast(dbuf_t[0]); m_params.t_max = static_cast(dbuf_t[1]); @@ -383,7 +384,7 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, if(ParallelDescriptor::IOProcessor()){ //Read data chunk std::ifstream inp(m_params.binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); #if (defined(WARPX_DIM_3D)) auto skip_amount = 1 + @@ -401,12 +402,12 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, sizeof(double)*t_begin*m_params.nx*m_params.ny; #endif inp.seekg(static_cast(skip_amount)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } const int read_size = (i_last - i_first + 1)* m_params.nx*m_params.ny; Vector buf_e(read_size); inp.read(reinterpret_cast(buf_e.dataPtr()), static_cast(read_size*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } std::transform(buf_e.begin(), buf_e.end(), h_E_binary_data.begin(), [](auto x) {return static_cast(x);} ); } @@ -440,7 +441,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cartes const auto tmp_y_max = m_params.y_max; const auto tmp_nx = m_params.nx; const auto tmp_ny = m_params.ny; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -523,7 +524,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cylind const auto tmp_r_max = m_params.r_max; const auto tmp_nr = m_params.nr; const auto tmp_n_rz_azimuthal_components = m_params.n_rz_azimuthal_components; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -625,7 +626,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_binary const auto tmp_ny = m_params.ny; #endif const auto tmp_nx = m_params.nx; - const auto p_E_binary_data = m_params.E_binary_data.dataPtr(); + const auto *const p_E_binary_data = m_params.E_binary_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp index ef00d95fd43..96d61d920f1 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -73,10 +73,11 @@ WarpXLaserProfiles::GaussianLaserProfile::init ( m_params.stc_direction[1]*m_common_params.p_X[1] + m_params.stc_direction[2]*m_common_params.p_X[2]; - if (arg < -1.0_rt || arg > 1.0_rt) + if (arg < -1.0_rt || arg > 1.0_rt) { m_params.theta_stc = 0._rt; - else + } else { m_params.theta_stc = std::acos(arg); + } #else m_params.theta_stc = 0.; #endif diff --git a/Source/Parallelization/CMakeLists.txt b/Source/Parallelization/CMakeLists.txt index 48f0d3a49bd..d38f8d6bbf8 100644 --- a/Source/Parallelization/CMakeLists.txt +++ b/Source/Parallelization/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) GuardCellManager.cpp WarpXComm.cpp WarpXRegrid.cpp + WarpXSumGuardCells.cpp ) endforeach() diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index c1ecc5db34c..ae5ab839d10 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -172,7 +172,7 @@ guardCellManager::Init ( // After pushing particle int ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); } ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); // Used if warpx.do_divb_cleaning = 1 @@ -314,8 +314,9 @@ guardCellManager::Init ( // If NCI filter, add guard cells in the z direction IntVect ng_NCIFilter = IntVect::TheZeroVector(); - if (do_fdtd_nci_corr) + if (do_fdtd_nci_corr) { ng_NCIFilter[WARPX_ZINDEX] = NCIGodfreyFilter::m_stencil_width; + } // Note: communications of guard cells for bilinear filter are handled // separately. diff --git a/Source/Parallelization/Make.package b/Source/Parallelization/Make.package index 629cfafea62..7930118a1d2 100644 --- a/Source/Parallelization/Make.package +++ b/Source/Parallelization/Make.package @@ -1,5 +1,6 @@ CEXE_sources += WarpXComm.cpp CEXE_sources += WarpXRegrid.cpp CEXE_sources += GuardCellManager.cpp +CEXE_sources += WarpXSumGuardCells.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parallelization diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 79c624e212a..7d5b25b1560 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -551,7 +551,7 @@ void WarpX::FillBoundaryE (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryE(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -608,7 +608,7 @@ void WarpX::FillBoundaryB (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryB(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -665,7 +665,7 @@ void WarpX::FillBoundaryE_avg(int lev, IntVect ng) { FillBoundaryE_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryE_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryE_avg(lev, PatchType::coarse, ng); } } void @@ -719,7 +719,7 @@ void WarpX::FillBoundaryB_avg (int lev, IntVect ng) { FillBoundaryB_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryB_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryB_avg(lev, PatchType::coarse, ng); } } void @@ -770,7 +770,7 @@ void WarpX::FillBoundaryF (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryF(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -780,7 +780,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); + if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -795,7 +795,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_cp[lev]) pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); + if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -824,7 +824,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); + if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -839,7 +839,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); + if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -1048,7 +1048,7 @@ WarpX::SyncRho ( { WARPX_PROFILE("WarpX::SyncRho()"); - if (!charge_fp[0]) return; + if (!charge_fp[0]) { return; } const int ncomp = charge_fp[0]->nComp(); // See comments in WarpX::SyncCurrent for an explanation of the algorithm. @@ -1330,7 +1330,7 @@ void WarpX::ApplyFilterandSumBoundaryRho ( const int glev = (patch_type == PatchType::fine) ? lev : lev-1; const std::unique_ptr& rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; - if (rho == nullptr) return; + if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); } @@ -1373,7 +1373,7 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( const int icomp, const int ncomp) { - if (!charge_fp[lev]) return; + if (!charge_fp[lev]) { return; } ApplyFilterandSumBoundaryRho(charge_fp, charge_cp, lev, PatchType::fine, icomp, ncomp); @@ -1452,7 +1452,7 @@ void WarpX::NodalSyncJ ( const int lev, PatchType patch_type) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine) { @@ -1478,7 +1478,7 @@ void WarpX::NodalSyncRho ( const int icomp, const int ncomp) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine && charge_fp[lev]) { diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 5cd5be168a9..dcea3cab58a 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -151,11 +151,11 @@ template void RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, const bool redistribute, const int lev) { - if (mf == nullptr) return; + if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); std::unique_ptr pmf; WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } mf = std::move(pmf); } @@ -164,7 +164,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { if (ba == boxArray(lev)) { - if (ParallelDescriptor::NProcs() == 1) return; + if (ParallelDescriptor::NProcs() == 1) { return; } // Fine patch for (int idim=0; idim < 3; ++idim) @@ -335,8 +335,9 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) + if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); + } } // Re-initialize the lattice element finder with the new ba and dm. diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 425ce320856..260ebb3871a 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -8,10 +8,6 @@ #ifndef WARPX_SUM_GUARD_CELLS_H_ #define WARPX_SUM_GUARD_CELLS_H_ -#include "Utils/WarpXAlgorithmSelection.H" - -#include - #include /** \brief Sum the values of `mf`, where the different boxes overlap @@ -26,20 +22,10 @@ * updates both the *valid* cells and *guard* cells. (This is because a * spectral solver requires the value of the sources over a large stencil.) */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = mf.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); -} + int icomp=0, int ncomp=1); /** \brief Sum the values of `src` where the different boxes overlap * (i.e. in the guard cells) and copy them into `dst` @@ -56,23 +42,10 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = dst.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - - dst.setVal(0., icomp, ncomp, n_updated_guards); -// ablastr::utils::communication::ParallelAdd(dst, src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); - dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); -} + int icomp=0, int ncomp=1); #endif // WARPX_SUM_GUARD_CELLS_H_ diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp new file mode 100644 index 00000000000..20ac73cab50 --- /dev/null +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -0,0 +1,51 @@ +/* Copyright 2019 Maxence Thevenet, Remi Lehe, Weiqun Zhang + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXSumGuardCells.H" + +#include "Utils/WarpXAlgorithmSelection.H" + +#include "WarpX.H" + +#include + +void +WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = mf.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); +} + + +void +WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, + const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = dst.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + + dst.setVal(0., icomp, ncomp, n_updated_guards); + dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); +} diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index be92e42d5fc..1de6b999e0b 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -7,9 +7,9 @@ #ifndef WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ #define WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ -#include "MCCProcess.H" #include "Particles/MultiParticleContainer.H" #include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" #include #include @@ -32,7 +32,7 @@ public: BackgroundMCCCollision ( BackgroundMCCCollision&& ) = delete; BackgroundMCCCollision& operator= ( BackgroundMCCCollision&& ) = delete; - amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); + [[nodiscard]] amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes) const; /** Perform the collisions * @@ -70,10 +70,10 @@ public: private: - amrex::Vector m_scattering_processes; - amrex::Vector m_ionization_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; - amrex::Gpu::DeviceVector m_ionization_processes_exe; + amrex::Vector m_scattering_processes; + amrex::Vector m_ionization_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + amrex::Gpu::DeviceVector m_ionization_processes_exe; bool init_flag = false; bool ionization_flag = false; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index ebf8cb47172..4cb16f6fa50 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -90,7 +90,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name amrex::Vector scattering_process_names; pp_collision_name.queryarr("scattering_processes", scattering_process_names); - // create a vector of MCCProcess objects from each scattering + // create a vector of ScatteringProcess objects from each scattering // process name for (const auto& scattering_process : scattering_process_names) { const std::string kw_cross_section = scattering_process + "_cross_section"; @@ -107,17 +107,17 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name pp_collision_name, kw_energy.c_str(), energy); } - MCCProcess process(scattering_process, cross_section_file, energy); + ScatteringProcess process(scattering_process, cross_section_file, energy); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != MCCProcessType::INVALID, - "Cannot add an unknown MCC process type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); // if the scattering process is ionization get the secondary species // only one ionization process is supported, the vector // m_ionization_processes is only used to make it simple to calculate // the maximum collision frequency with the same function used for // particle conserving processes - if (process.type() == MCCProcessType::IONIZATION) { + if (process.type() == ScatteringProcessType::IONIZATION) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!ionization_flag, "Background MCC only supports a single ionization process"); ionization_flag = true; @@ -133,8 +133,8 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name } #ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - amrex::Gpu::HostVector h_ionization_processes_exe; + amrex::Gpu::HostVector h_scattering_processes_exe; + amrex::Gpu::HostVector h_ionization_processes_exe; for (auto const& p : m_scattering_processes) { h_scattering_processes_exe.push_back(p.executor()); } @@ -162,7 +162,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name * ranges from 1e-4 to 5000 eV in 0.2 eV increments */ amrex::ParticleReal -BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) +BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) const { using namespace amrex::literals; amrex::ParticleReal nu, nu_max = 0.0; @@ -276,7 +276,7 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto const flvl = species1.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // firstly loop over particles box by box and do all particle conserving // scattering @@ -324,7 +324,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile auto T_a_func = m_background_temperature_func; // get collision parameters - auto scattering_processes = m_scattering_processes_exe.data(); + auto *scattering_processes = m_scattering_processes_exe.data(); auto const process_count = static_cast(m_scattering_processes_exe.size()); auto const total_collision_prob = m_total_collision_prob; @@ -352,7 +352,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile [=] AMREX_GPU_HOST_DEVICE (long ip, amrex::RandomEngine const& engine) { // determine if this particle should collide - if (amrex::Random(engine) > total_collision_prob) return; + if (amrex::Random(engine) > total_collision_prob) { return; } amrex::ParticleReal x, y, z; GetPosition.AsStored(ip, x, y, z); @@ -398,13 +398,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile nu_i += n_a * sigma_E * v_coll / nu_max; // check if this collision should be performed - if (col_select > nu_i) continue; + if (col_select > nu_i) { continue; } // charge exchange is implemented as a simple swap of the projectile // and target velocities which doesn't require any of the Lorentz // transformations below; note that if the projectile and target // have the same mass this is identical to back scattering - if (scattering_process.m_type == MCCProcessType::CHARGE_EXCHANGE) { + if (scattering_process.m_type == ScatteringProcessType::CHARGE_EXCHANGE) { ux[ip] = ua_x; uy[ip] = ua_y; uz[ip] = ua_z; @@ -433,13 +433,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // transform to COM frame ParticleUtils::doLorentzTransform(vx, vy, vz, uCOM_x, uCOM_y, uCOM_z); - if ((scattering_process.m_type == MCCProcessType::ELASTIC) - || (scattering_process.m_type == MCCProcessType::EXCITATION)) { + if ((scattering_process.m_type == ScatteringProcessType::ELASTIC) + || (scattering_process.m_type == ScatteringProcessType::EXCITATION)) { ParticleUtils::RandomizeVelocity( vx, vy, vz, sqrt(vx*vx + vy*vy + vz*vz), engine ); } - else if (scattering_process.m_type == MCCProcessType::BACK) { + else if (scattering_process.m_type == ScatteringProcessType::BACK) { // elastic scattering with cos(chi) = -1 (i.e. 180 degrees) vx *= -1.0_prt; vy *= -1.0_prt; @@ -501,7 +501,7 @@ void BackgroundMCCCollision::doBackgroundIonization m_mass1, sqrt_kb_m, m_background_temperature_func, t ); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(species1, species2, elec_tile, ion_tile, elec_tile, np_elec, np_ion, Filter, CopyElec, CopyIon, Transform ); diff --git a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt index 15742af465f..ca8db0c53ec 100644 --- a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt +++ b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt @@ -3,6 +3,5 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BackgroundMCCCollision.cpp - MCCProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H index cd9b74c72b8..73ed60d0ad7 100644 --- a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H +++ b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H @@ -4,10 +4,10 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ -#define WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#ifndef WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ +#define WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ -#include "MCCProcess.H" +#include "Particles/Collision/ScatteringProcess.H" #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -39,7 +39,7 @@ public: * ensure accurate energy calculations (otherwise errors occur with single * or mixed precision builds of WarpX). * - * @param[in] mcc_process an MCCProcess object associated with the ionization + * @param[in] mcc_process an ScatteringProcess object associated with the ionization * @param[in] mass colliding particle's mass (could also assume electron) * @param[in] total_collision_prob total probability for a collision to occur * @param[in] nu_max maximum collision frequency @@ -48,7 +48,7 @@ public: * @param[in] t the current simulation time */ ImpactIonizationFilterFunc( - MCCProcess const& mcc_process, + ScatteringProcess const& mcc_process, double const mass, amrex::ParticleReal const total_collision_prob, amrex::ParticleReal const nu_max, @@ -77,7 +77,7 @@ public: using std::sqrt; // determine if this particle should collide - if (Random(engine) > m_total_collision_prob) return false; + if (Random(engine) > m_total_collision_prob) { return false; } // get references to the particle to get its position const auto& p = ptd.getSuperParticle(i); @@ -108,7 +108,7 @@ public: } private: - MCCProcess::Executor m_mcc_process; + ScatteringProcess::Executor m_mcc_process; double m_mass; amrex::ParticleReal m_total_collision_prob = 0; amrex::ParticleReal m_nu_max; @@ -227,4 +227,4 @@ private: amrex::ParserExecutor<4> m_T_a_func; amrex::Real m_t; }; -#endif // WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#endif // WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/Make.package b/Source/Particles/Collision/BackgroundMCC/Make.package index 0b007e64e6d..242e85a8764 100644 --- a/Source/Particles/Collision/BackgroundMCC/Make.package +++ b/Source/Particles/Collision/BackgroundMCC/Make.package @@ -1,4 +1,3 @@ CEXE_sources += BackgroundMCCCollision.cpp -CEXE_sources += MCCProcess.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 83f74840478..8122d7225b4 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -104,7 +104,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar auto const flvl = species.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // loop over particles box by box #ifdef _OPENMP diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 58fe2c9911a..5c90dab25e6 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -87,8 +87,9 @@ public: BinaryCollision (std::string collision_name, MultiParticleContainer const * const mypc) : CollisionBase(collision_name) { - if(m_species_names.size() != 2) + if(m_species_names.size() != 2) { WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); + } m_isSameSpecies = (m_species_names[0] == m_species_names[1]); @@ -108,8 +109,8 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = default; - BinaryCollision& operator= ( BinaryCollision&& ) = default; + BinaryCollision ( BinaryCollision&& ) noexcept = default; + BinaryCollision& operator= ( BinaryCollision&& ) noexcept = default; /** Perform the collisions * @@ -156,8 +157,8 @@ public: auto copy_species1_data = device_copy_species1.data(); auto copy_species2_data = device_copy_species2.data(); #else - auto copy_species1_data = copy_species1.data(); - auto copy_species2_data = copy_species2.data(); + auto *copy_species1_data = copy_species1.data(); + auto *copy_species2_data = copy_species2.data(); #endif if (m_have_product_species){ species1.defineAllParticleTiles(); @@ -166,14 +167,14 @@ public: // Enable tiling amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) info.EnableTiling(species1.tile_size); + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(species1.tile_size); } // Loop over refinement levels for (int lev = 0; lev <= species1.finestLevel(); ++lev){ amrex::LayoutData* cost = WarpX::getCosts(lev); - // Loop over all grids/tiles at this level + // Loop over all grids/tiles at this level #ifdef AMREX_USE_OMP info.SetDynamic(true); #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -239,7 +240,7 @@ public: products_np.push_back(ptile_product.numParticles()); products_mass.push_back(product_species_vector[i]->getMass()); } - auto tile_products_data = tile_products.data(); + auto *tile_products_data = tile_products.data(); if ( m_isSameSpecies ) // species_1 == species_2 { @@ -341,7 +342,7 @@ public: p_pair_offsets[i_cell] : 0; // Do not collide if there is only one particle in the cell - if ( cell_stop_1 - cell_start_1 <= 1 ) return; + if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( @@ -367,7 +368,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, tile_products_data, + soa_1, soa_1, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_1, m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, @@ -447,11 +450,12 @@ public: const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) { p_n_pairs_in_each_cell[i_cell] = 0; - else + } else { p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } } ); @@ -502,7 +506,7 @@ public: // Do not collide if one species is missing in the cell if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) return; + cell_stop_2 - cell_start_2 < 1 ) { return; } // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); @@ -528,7 +532,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, tile_products_data, + soa_1, soa_2, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_2, m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 0a7aae9835e..b8a390ddb93 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -12,6 +12,8 @@ #include "Particles/MultiParticleContainer.H" +#include + enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToProtonTritiumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, @@ -36,6 +38,92 @@ namespace BinaryCollisionUtils{ MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); + + /** + * \brief Return (relativistic) collision energy, collision speed and + * Lorentz factor for transforming between the lab and center-of-momentum + * frames. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void get_collision_parameters ( + const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal& E_kin_COM, amrex::ParticleReal& v_rel_COM, + amrex::ParticleReal& lab_to_COM_lorentz_factor ) + { + // General notations in this function: + // x_sq denotes the square of x + // x_star denotes the value of x in the center of mass frame + + using namespace amrex::literals; + using namespace amrex::Math; + + constexpr auto one_pr = amrex::ParticleReal(1.); + constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); + constexpr double c_sq = PhysConst::c * PhysConst::c; + constexpr double inv_csq = 1.0 / c_sq; + + const amrex::ParticleReal m1_sq = m1*m1; + const amrex::ParticleReal m2_sq = m2*m2; + + // Compute Lorentz factor gamma in the lab frame + const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); + const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + + // Compute momenta + const amrex::ParticleReal p1x = u1x * m1; + const amrex::ParticleReal p1y = u1y * m1; + const amrex::ParticleReal p1z = u1z * m1; + const amrex::ParticleReal p2x = u2x * m2; + const amrex::ParticleReal p2y = u2y * m2; + const amrex::ParticleReal p2z = u2z * m2; + // Square norm of the total (sum between the two particles) momenta in the lab frame + const auto p_total_sq = static_cast( + powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) + ); + + // Total energy in the lab frame + // Note the use of `double` for energy since this calculation is + // prone to error with single precision. + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + // Total energy squared in the center of mass frame, calculated using the Lorentz invariance + // of the four-momentum norm + const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; + + // Kinetic energy in the center of mass frame + const double E_star = std::sqrt(E_star_sq); + E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); + + // Square of the norm of the momentum of one of the particles in the center of mass frame + // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle + // The expression below is specifically written in a form that avoids returning + // small negative numbers due to machine precision errors, for low-energy particles + const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); + const auto p_star_sq = static_cast( + m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) + + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) + ); + + // Lorentz factors in the center of mass frame + const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); + const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + + // relative velocity in the center of mass frame + v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + + // Cross sections and relative velocity are computed in the center of mass frame. + // On the other hand, the particle densities (weight over volume) in the lab frame are used. + // To take this disrepancy into account, it is needed to multiply the + // collision probability by the ratio between the Lorentz factors in the + // COM frame and the Lorentz factors in the lab frame (see + // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor + // is calculated here. + lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + } } #endif // BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 82f263e9daf..430bac5e548 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -118,16 +118,21 @@ namespace BinaryCollisionUtils{ CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { return CollisionType::ProtonBoronToAlphasFusion; + } WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); return CollisionType::Undefined; } diff --git a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt index 55bbf21e310..60933c83a21 100644 --- a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt @@ -6,3 +6,5 @@ foreach(D IN LISTS WarpX_DIMS) ParticleCreationFunc.cpp ) endforeach() + +add_subdirectory(DSMC) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index b54404d46ba..d7403eeacf9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -104,7 +104,7 @@ void ElasticCollisionPerez ( n12 = n12 / dV; } // Intra-species: Apply correction in collision rate - if (isSameSpecies) n12 *= T_PR(2.0); + if (isSameSpecies) { n12 *= T_PR(2.0); } // compute Debye length lmdD T_PR lmdD; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt new file mode 100644 index 00000000000..3575d53ad29 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -0,0 +1,7 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + DSMC.cpp + ) +endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H new file mode 100644 index 00000000000..c714cfdb133 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -0,0 +1,222 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies), Neil Zaim + * + * License: BSD-3-Clause-LBNL + */ +#ifndef COLLISION_FILTER_FUNC_H_ +#define COLLISION_FILTER_FUNC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/ScatteringProcess.H" + +#include + +/** + * \brief This function determines whether a collision occurs for a given + * pair of particles. + * + * @param[in] u1x,u1y,u1z momenta of the first colliding particle + * @param[in] u2x,u2y,u2z momenta of the second colliding particle + * @param[in] m1,m2 masses + * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] pair_index is the index of the colliding pair + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_reaction_weight stores the weight of the product particles + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal w1, amrex::ParticleReal w2, + const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, + index_type* AMREX_RESTRICT p_mask, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + const int& multiplier_ratio, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + const amrex::RandomEngine& engine) +{ + using namespace amrex::literals; + + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); + + // convert E_coll to eV + E_coll /= PhysConst::q_e; + + amrex::ParticleReal sigma_tot = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma_tot += scattering_process.getCrossSection(E_coll); + } + + // calculate total collision probability + amrex::ParticleReal exponent = ( + lab_to_COM_factor * multiplier_ratio * w_max + * sigma_tot * v_coll * dt / dV + ); + + // Compute actual collision probability that is always between zero and one + // In principle this is obtained by computing 1 - exp(-probability_estimate) + // However, the computation of this quantity can fail numerically when probability_estimate is + // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-exponent); + + // Now we determine if a collision should occur + if (amrex::Random(engine) < probability) + { + const amrex::ParticleReal random_number = amrex::Random(engine); + amrex::ParticleReal sigma = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma += scattering_process.getCrossSection(E_coll); + if (random_number <= sigma / sigma_tot) + { + p_mask[pair_index] = int(scattering_process.m_type); + p_pair_reaction_weight[pair_index] = w_min; + break; + } + } + } + else + { + p_mask[pair_index] = false; + } +} + +/** + * \brief Function that determines if a collision occurs and if so, what + * type. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] ptd_1,ptd_2 contain the particle data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionFilter ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + PData ptd_1, PData ptd_2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + amrex::RandomEngine const& engine) +{ + + amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const int NI1 = I1e - I1s; + const int NI2 = I2e - I2s; + const int max_N = amrex::max(NI1,NI2); + + int i1 = I1s; + int i2 = I2s; + int pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + const int c1 = amrex::max(NI2/NI1,1); + const int c2 = amrex::max(NI1/NI2,1); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; +#endif + + for (int k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, pair_index, p_mask, p_pair_reaction_weight, + max_N, process_count, scattering_processes, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } + ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++pair_index; + } +} + +#endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H new file mode 100644 index 00000000000..c1be307b811 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -0,0 +1,85 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef DSMC_H_ +#define DSMC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMC final + : public CollisionBase +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = amrex::DenseBins; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using index_type = ParticleBins::index_type; + +public: + + /** + * \brief Constructor of the DSMC class + * + * @param[in] collision_name the name of the collision + */ + DSMC (std::string collision_name); + + /** Perform the collisions + * + * @param cur_time Current time + * @param dt Time step size + * @param mypc Container of species involved + * + */ + void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; + + /** Perform all binary collisions within a tile + * + * \param[in] lev the mesh-refinement level + * \param[in] mfi iterator for multifab + * \param species_1 first species container + * \param species_2 second species container + * \param copy_species1 SmartCopy for species_1 + * \param copy_species2 SmartCopy for species_2 + * + */ + void doCollisionsWithinTile ( + amrex::Real dt, int lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2 ); + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; +}; + +#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp new file mode 100644 index 00000000000..70a14712a0c --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -0,0 +1,300 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "CollisionFilterFunc.H" +#include "DSMC.H" +#include "SplitAndScatterFunc.H" + + +DSMC::DSMC (const std::string collision_name) + : CollisionBase(collision_name) +{ + using namespace amrex::literals; + amrex::ParmParse pp_collision_name(collision_name); + +#if defined WARPX_DIM_RZ + amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); +#endif + + if(m_species_names.size() != 2) + { + amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); + } + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif +} + +void +DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) +{ + WARPX_PROFILE("DSMC::doCollisions()"); + + auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); + + // SmartCopy objects are created that will facilitate the particle splitting + // operation involved in DSMC collisions between particles with arbitrary + // weights. + SmartCopyFactory copy_factory_species1(species1, species1); + SmartCopyFactory copy_factory_species2(species2, species2); + auto copy_species1 = copy_factory_species1.getSmartCopy(); + auto copy_species2 = copy_factory_species2.getSmartCopy(); + + species1.defineAllParticleTiles(); + species2.defineAllParticleTiles(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + // Loop over refinement levels + for (int lev = 0; lev <= species1.finestLevel(); ++lev){ + + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // Loop over all grids/tiles at this level +#ifdef AMREX_USE_OMP + info.SetDynamic(true); +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + doCollisionsWithinTile(dt, lev, mfi, species1, species2, + copy_species1, copy_species2); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + + // Call redistribute to remove particles with negative ids + species1.Redistribute(lev, lev, 0, true, true); + species2.Redistribute(lev, lev, 0, true, true); + } +} + +void +DSMC::doCollisionsWithinTile( + amrex::Real dt, int const lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2) +{ + using namespace ParticleUtils; + using namespace amrex::literals; + + // get collision processes + auto *scattering_processes = m_scattering_processes_exe.data(); + int const process_count = static_cast(m_scattering_processes_exe.size()); + + // Extract particles in the tile that `mfi` points to + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + // Find the particles that are in each cell of this tile + ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); + + // Extract low-level data + int const n_cells = static_cast(bins_1.numBins()); + + // - Species 1 + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + amrex::ParticleReal m1 = species_1.getMass(); + const auto ptd_1 = ptile_1.getParticleTileData(); + + // - Species 2 + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + amrex::ParticleReal m2 = species_2.getMass(); + const auto ptd_2 = ptile_2.getParticleTileData(); + + amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); +#if defined WARPX_DIM_1D_Z + auto dV = geom.CellSize(0); +#elif defined WARPX_DIM_XZ + auto dV = geom.CellSize(0) * geom.CellSize(1); +#elif defined WARPX_DIM_RZ + amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box + const auto lo = lbound(cbx); + const auto hi = ubound(cbx); + int nz = hi.y-lo.y+1; + auto dr = geom.CellSize(0); + auto dz = geom.CellSize(1); +#elif defined(WARPX_DIM_3D) + auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); +#endif + + // In the following we set up the "mask" used for creating new particles + // (from splitting events). There is a mask index for every collision + // pair. Below we find the size of the mask based on the greater of the + // number of each species' particle in each cell. + amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); + index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); + + // Compute how many pairs in each cell and store in n_pairs_in_each_cell array + // For different species, the number of pairs in a cell is the number of particles of + // the species that has the most particles in that cell + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + // Particular case: no pair if a species has no particle in that cell + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + { + p_n_pairs_in_each_cell[i_cell] = 0; + } + else + { + p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } + } + ); + + // Start indices of the pairs in a cell. Will be used for particle creation + amrex::Gpu::DeviceVector pair_offsets(n_cells); + const index_type n_total_pairs = (n_cells == 0) ? 0: + amrex::Scan::ExclusiveSum(n_cells, + p_n_pairs_in_each_cell, pair_offsets.data()); + index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + + // Now we create the mask. In the DSMC scheme the mask will serve two + // purposes, 1) record whether a given pair should collide and 2) record + // the scattering process that should occur. + amrex::Gpu::DeviceVector mask(n_total_pairs); + index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); + + // Will be filled with the index of the first particle of a given pair + amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); + // Will be filled with the index of the second particle of a given pair + amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); + + // How much weight should be given to the produced particle - based on the + // weight of the collision partner which is not split + amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = + pair_reaction_weight.dataPtr(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + // Same but for the pairs + index_type const cell_start_pair = p_pair_offsets[i_cell]; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + + // Do not collide if one species is missing in the cell + if ( cell_stop_1 - cell_start_1 < 1 || + cell_stop_2 - cell_start_2 < 1 ) { return; } + + // shuffle + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); + ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); +#if defined WARPX_DIM_RZ + int ri = (i_cell - i_cell%nz) / nz; + auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; +#endif + // Call the function in order to perform collisions + // If there are product species, p_mask, p_pair_indices_1/2, and + // p_pair_reaction_weight are filled here + CollisionFilter( + cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, + indices_1, indices_2, + ptd_1, ptd_2, + m1, m2, dt, dV, + cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight, + process_count, scattering_processes, engine ); + } + ); + + const auto num_p_tile1 = ptile_1.numParticles(); + const auto num_p_tile2 = ptile_2.numParticles(); + + // Create the new product particles and define their initial values + // num_added: how many particles of each product species have been created + const int num_added = splitScatteringParticles( + n_total_pairs, + ptile_1, ptile_2, + p_mask, + copy_species1, copy_species2, + m1, m2, + p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight); + + if (num_added > 0) { + setNewParticleIDs(ptile_1, num_p_tile1, num_added); + setNewParticleIDs(ptile_2, num_p_tile2, num_added); + } +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package new file mode 100644 index 00000000000..b4cfab89c64 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -0,0 +1,3 @@ +CEXE_sources += DSMC.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H new file mode 100644 index 00000000000..c1fb7ee7e38 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -0,0 +1,165 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SPLIT_AND_SCATTER_FUNC_H_ +#define SPLIT_AND_SCATTER_FUNC_H_ + +#include "Particles/Collision/ScatteringProcess.H" + +/** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ +template ::value, int> foo = 0> +int splitScatteringParticles ( + const index_type& n_total_pairs, + Tile& ptile1, Tile& ptile2, + const Index* AMREX_RESTRICT mask, + CopyFunc&& copy1, CopyFunc&& copy2, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +{ + using namespace amrex; + + if (n_total_pairs == 0) { + return 0; + } + + Gpu::DeviceVector offsets(n_total_pairs); + Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + const auto ptile1_index = ptile1.numParticles(); + const auto ptile2_index = ptile2.numParticles(); + ptile1.resize(ptile1_index + num_added); + ptile2.resize(ptile2_index + num_added); + + const auto ptile1_data = ptile1.getParticleTileData(); + const auto ptile2_data = ptile2.getParticleTileData(); + + const Long minus_one_long = -1; + + ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept + { + if (mask[i]) + { + // First, make copies of the colliding particles + copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); + copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); + + // Now we adjust the properties of the original and child particles, + // starting with the parent particles + auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; + auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights. + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to -1. + Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); + if (w1 <= 0._prt) { + auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; + p.atomicSetID(minus_one_long); + } + + Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); + if (w2 <= 0._prt) { + auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; + p.atomicSetID(minus_one_long); + } + + // Set the child particle properties appropriately + ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; + ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; + + auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; + auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; + auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; + auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; + auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; + auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (m1 == m2) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + Abort("Uneven mass charge-exchange not implemented yet."); + } + } + else { + Abort("Unknown scattering process."); + } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; + } + }); + + Gpu::synchronize(); + return static_cast(num_added); +} +#endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Make.package b/Source/Particles/Collision/BinaryCollision/Make.package index 92e2f0b5dd1..393e6270345 100644 --- a/Source/Particles/Collision/BinaryCollision/Make.package +++ b/Source/Particles/Collision/BinaryCollision/Make.package @@ -1,4 +1,6 @@ CEXE_sources += BinaryCollisionUtils.cpp CEXE_sources += ParticleCreationFunc.cpp +include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC/Make.package + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 8b0e71763bc..07e0174438b 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -15,7 +15,6 @@ #include "Utils/WarpXConst.H" #include -#include #include #include @@ -67,89 +66,33 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part const NuclearFusionType& fusion_type, const amrex::RandomEngine& engine) { - // General notations in this function: - // x_sq denotes the square of x - // x_star denotes the value of x in the center of mass frame + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); using namespace amrex::literals; - using namespace amrex::Math; const amrex::ParticleReal w_min = amrex::min(w1, w2); const amrex::ParticleReal w_max = amrex::max(w1, w2); - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); - constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; - constexpr amrex::ParticleReal inv_csq = one_pr / ( c_sq ); - - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; - - // Compute Lorentz factor gamma in the lab frame - const amrex::ParticleReal g1 = std::sqrt( one_pr + (u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const amrex::ParticleReal g2 = std::sqrt( one_pr + (u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); - - // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; - // Square norm of the total (sum between the two particles) momenta in the lab frame - const amrex::ParticleReal p_total_sq = powi<2>(p1x + p2x) + - powi<2>(p1y+p2y) + - powi<2>(p1z+p2z); - - // Total energy in the lab frame - const amrex::ParticleReal E_lab = (m1 * g1 + m2 * g2) * c_sq; - // Total energy squared in the center of mass frame, calculated using the Lorentz invariance - // of the four-momentum norm - const amrex::ParticleReal E_star_sq = E_lab*E_lab - c_sq*p_total_sq; - - // Kinetic energy in the center of mass frame - const amrex::ParticleReal E_star = std::sqrt(E_star_sq); - const amrex::ParticleReal E_kin_star = E_star - (m1 + m2)*c_sq; - // Compute fusion cross section as a function of kinetic energy in the center of mass frame auto fusion_cross_section = amrex::ParticleReal(0.); if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - fusion_cross_section = ProtonBoronFusionCrossSection(E_kin_star); + fusion_cross_section = ProtonBoronFusionCrossSection(E_coll); } else if ((fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium)) { - fusion_cross_section = BoschHaleFusionCrossSection(E_kin_star, fusion_type, m1, m2); + fusion_cross_section = BoschHaleFusionCrossSection(E_coll, fusion_type, m1, m2); } - // Square of the norm of the momentum of one of the particles in the center of mass frame - // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle - // The expression below is specifically written in a form that avoids returning - // small negative numbers due to machine precision errors, for low-energy particles - const amrex::ParticleReal E_ratio = E_star/((m1 + m2)*c_sq); - const amrex::ParticleReal p_star_sq = m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio ); - - // Lorentz factors in the center of mass frame - const amrex::ParticleReal g1_star = std::sqrt(one_pr + p_star_sq / (m1_sq*c_sq)); - const amrex::ParticleReal g2_star = std::sqrt(one_pr + p_star_sq / (m2_sq*c_sq)); - - // relative velocity in the center of mass frame - const amrex::ParticleReal v_rel = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + - one_pr/(m2*g2_star)); - - // Fusion cross section and relative velocity are computed in the center of mass frame. - // On the other hand, the particle densities (weight over volume) in the lab frame are used. To - // take into account this discrepancy, we need to multiply the fusion probability by the ratio - // between the Lorentz factors in the COM frame and the Lorentz factors in the lab frame - // (see Perez et al., Phys.Plasmas.19.083104 (2012)) - const amrex::ParticleReal lab_to_COM_factor = g1_star*g2_star/(g1*g2); - // First estimate of probability to have fusion reaction amrex::ParticleReal probability_estimate = multiplier_ratio * fusion_multiplier * - lab_to_COM_factor * w_max * fusion_cross_section * v_rel * dt / dV; + lab_to_COM_factor * w_max * fusion_cross_section * v_coll * dt / dV; // Effective fusion multiplier amrex::ParticleReal fusion_multiplier_eff = fusion_multiplier; @@ -162,7 +105,7 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // We aim for a fusion probability of probability_target_value but take into account // the constraint that the fusion_multiplier cannot be smaller than one fusion_multiplier_eff = amrex::max(fusion_multiplier * - probability_target_value / probability_estimate , one_pr); + probability_target_value / probability_estimate , 1._prt); probability_estimate *= fusion_multiplier_eff/fusion_multiplier; } @@ -170,18 +113,8 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // In principle this is obtained by computing 1 - exp(-probability_estimate) // However, the computation of this quantity can fail numerically when probability_estimate is // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). - // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) - // The threshold exp_threshold at which we switch between the two formulas is determined by the - // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its - // Taylor expansion, i.e. the square of probability_estimate should be greater than the - // machine epsilon. -#ifdef AMREX_SINGLE_PRECISION_PARTICLES - constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); -#else - constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); -#endif - const amrex::ParticleReal probability = (probability_estimate < exp_threshold) ? - probability_estimate: one_pr - std::exp(-probability_estimate); + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-probability_estimate); // Get a random number const amrex::ParticleReal random_number = amrex::Random(engine); @@ -198,6 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } - - #endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index ca994ed1f56..dc830b477df 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -100,6 +100,7 @@ public: amrex::Vector operator() ( const index_type& n_total_pairs, const SoaData_type& soa_1, const SoaData_type& soa_2, + const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -115,7 +116,7 @@ public: { using namespace amrex::literals; - if (n_total_pairs == 0) return amrex::Vector(m_num_product_species, 0); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } // Compute offset array and allocate memory for the produced species amrex::Gpu::DeviceVector offsets(n_total_pairs); @@ -251,6 +252,27 @@ public: } }); + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + int start_index = int(products_np[i]); + int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); + } + amrex::Gpu::synchronize(); return num_added_vec; @@ -289,6 +311,7 @@ public: amrex::Vector operator() ( const index_type& /*n_total_pairs*/, const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 0d8ffc5b8ae..33629cd530f 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -19,45 +19,45 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); +{ + const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) - { - // Proton-Boron fusion only produces alpha particles - m_num_product_species = 1; - // Proton-Boron fusion produces 3 alpha particles per fusion reaction - m_num_products_host.push_back(3); + if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) + { + // Proton-Boron fusion only produces alpha particles + m_num_product_species = 1; + // Proton-Boron fusion produces 3 alpha particles per fusion reaction + m_num_products_host.push_back(3); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediately - m_num_products_device.push_back(3); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(3); #endif - } - else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) - { - m_num_product_species = 2; - m_num_products_host.push_back(1); - m_num_products_host.push_back(1); + } + else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) + { + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediately - m_num_products_device.push_back(1); - m_num_products_device.push_back(1); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); #endif - } - else - { - WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); - } + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); + } #ifdef AMREX_USE_GPU - m_num_products_device.resize(m_num_product_species); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), - m_num_products_host.end(), - m_num_products_device.begin()); - amrex::Gpu::streamSynchronize(); + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); #endif - } +} diff --git a/Source/Particles/Collision/CMakeLists.txt b/Source/Particles/Collision/CMakeLists.txt index f3183032179..62eb3b3ad9c 100644 --- a/Source/Particles/Collision/CMakeLists.txt +++ b/Source/Particles/Collision/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE CollisionHandler.cpp CollisionBase.cpp + ScatteringProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index 272cc056b1a..db79eaf1a01 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -29,7 +29,7 @@ public: virtual ~CollisionBase() = default; - int get_ndt() {return m_ndt;} + [[nodiscard]] int get_ndt() const {return m_ndt;} protected: diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 213e5edd577..5abee34c86a 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -10,6 +10,7 @@ #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -52,6 +53,9 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) else if (type == "background_stopping") { allcollisions[i] = std::make_unique(collision_names[i]); } + else if (type == "dsmc") { + allcollisions[i] = std::make_unique(collision_names[i]); + } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( diff --git a/Source/Particles/Collision/Make.package b/Source/Particles/Collision/Make.package index e9aad3691a3..9ec153879f4 100644 --- a/Source/Particles/Collision/Make.package +++ b/Source/Particles/Collision/Make.package @@ -1,5 +1,6 @@ CEXE_sources += CollisionHandler.cpp CEXE_sources += CollisionBase.cpp +CEXE_sources += ScatteringProcess.cpp include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/Make.package include $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC/Make.package diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H b/Source/Particles/Collision/ScatteringProcess.H similarity index 81% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.H rename to Source/Particles/Collision/ScatteringProcess.H index 9b844959e52..d05ea32e2fe 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -1,18 +1,20 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ -#define WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#ifndef WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ +#define WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ #include #include #include #include -enum class MCCProcessType { +enum class ScatteringProcessType { INVALID, ELASTIC, BACK, @@ -21,29 +23,29 @@ enum class MCCProcessType { IONIZATION, }; -class MCCProcess +class ScatteringProcess { public: - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, amrex::ParticleReal energy ); template - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, amrex::ParticleReal energy ); - ~MCCProcess() = default; + ~ScatteringProcess() = default; - MCCProcess (MCCProcess const&) = delete; - MCCProcess& operator= (MCCProcess const&) = delete; - MCCProcess (MCCProcess &&) = default; - MCCProcess& operator= (MCCProcess &&) = default; + ScatteringProcess (ScatteringProcess const&) = delete; + ScatteringProcess& operator= (ScatteringProcess const&) = delete; + ScatteringProcess (ScatteringProcess &&) = default; + ScatteringProcess& operator= (ScatteringProcess &&) = default; /** Read the given cross-section data file to memory. * @@ -99,7 +101,7 @@ public: amrex::ParticleReal* m_sigmas_data = nullptr; amrex::ParticleReal m_energy_lo, m_energy_hi, m_sigma_lo, m_sigma_hi, m_dE; amrex::ParticleReal m_energy_penalty; - MCCProcessType m_type; + ScatteringProcessType m_type; }; [[nodiscard]] @@ -121,12 +123,12 @@ public: [[nodiscard]] amrex::ParticleReal getMaxEnergyInput () const { return m_exe_h.m_energy_hi; } [[nodiscard]] amrex::ParticleReal getEnergyInputStep () const { return m_exe_h.m_dE; } - [[nodiscard]] MCCProcessType type () const { return m_exe_h.m_type; } + [[nodiscard]] ScatteringProcessType type () const { return m_exe_h.m_type; } private: static - MCCProcessType parseProcessType(const std::string& process); + ScatteringProcessType parseProcessType(const std::string& process); void init (const std::string& scattering_process, amrex::ParticleReal energy); @@ -142,4 +144,4 @@ private: int m_grid_size; }; -#endif // WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#endif // WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp similarity index 79% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp rename to Source/Particles/Collision/ScatteringProcess.cpp index bf2138bc78b..c15d11eea07 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -1,15 +1,17 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#include "MCCProcess.H" +#include "ScatteringProcess.H" #include "Utils/TextMsg.H" #include "WarpX.H" -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, const amrex::ParticleReal energy ) @@ -21,7 +23,7 @@ MCCProcess::MCCProcess ( } template -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, @@ -34,7 +36,7 @@ MCCProcess::MCCProcess ( } void -MCCProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) +ScatteringProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) { using namespace amrex::literals; m_exe_h.m_sigmas_data = m_sigmas_h.data(); @@ -72,44 +74,44 @@ MCCProcess::init (const std::string& scattering_process, const amrex::ParticleRe #endif } -MCCProcessType -MCCProcess::parseProcessType(const std::string& scattering_process) +ScatteringProcessType +ScatteringProcess::parseProcessType(const std::string& scattering_process) { if (scattering_process == "elastic") { - return MCCProcessType::ELASTIC; + return ScatteringProcessType::ELASTIC; } else if (scattering_process == "back") { - return MCCProcessType::BACK; + return ScatteringProcessType::BACK; } else if (scattering_process == "charge_exchange") { - return MCCProcessType::CHARGE_EXCHANGE; + return ScatteringProcessType::CHARGE_EXCHANGE; } else if (scattering_process == "ionization") { - return MCCProcessType::IONIZATION; + return ScatteringProcessType::IONIZATION; } else if (scattering_process.find("excitation") != std::string::npos) { - return MCCProcessType::EXCITATION; + return ScatteringProcessType::EXCITATION; } else { - return MCCProcessType::INVALID; + return ScatteringProcessType::INVALID; } } void -MCCProcess::readCrossSectionFile ( +ScatteringProcess::readCrossSectionFile ( const std::string cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { std::ifstream infile(cross_section_file); - if(!infile.is_open()) WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); + if(!infile.is_open()) { WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); } amrex::ParticleReal energy, sigma; while (infile >> energy >> sigma) { energies.push_back(energy); sigmas.push_back(sigma); } - if (infile.bad()) WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); + if (infile.bad()) { WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); } infile.close(); } void -MCCProcess::sanityCheckEnergyGrid ( +ScatteringProcess::sanityCheckEnergyGrid ( const amrex::Vector& energies, amrex::ParticleReal dE ) diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 50ed74439e5..d0db678dfda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -261,7 +261,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { using namespace amrex; - auto permutation = a_bins.permutationPtr(); + const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index fd74c9029ed..efe6efcc788 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,6 +26,246 @@ #include #include +/** + * \brief Kernel for the direct current deposition for thread thread_num + * \tparam depos_order deposition order + * \param xp, yp, zp The particle positions. + * \param wq The charge of the macroparticle + * \param vx,vy,vz The particle velocities + * \param jx_arr,jy_arr,jz_arr Array4 of current density, either full array or tile. + * \param jx_type,jy_type,jz_type The grid types along each direction, either NODE or CELL + * \param relative_time Time at which to deposit J, relative to the time of the + * current positions of the particles. When different than 0, + * the particle position will be temporarily modified to match + * the time of the deposition. + * \param dzi, dxi, dyi The inverse cell sizes + * \param zmin, xmin, ymin The lower bounds of the domain + * \param invvol The inverse volume of a grid cell + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void doDepositionShapeNKernel(const amrex::ParticleReal xp, + const amrex::ParticleReal yp, + const amrex::ParticleReal zp, + const amrex::ParticleReal wq, + const amrex::ParticleReal vx, + const amrex::ParticleReal vy, + const amrex::ParticleReal vz, + amrex::Array4 const& jx_arr, + amrex::Array4 const& jy_arr, + amrex::Array4 const& jz_arr, + amrex::IntVect const& jx_type, + amrex::IntVect const& jy_type, + amrex::IntVect const& jz_type, + const amrex::Real relative_time, + AMREX_D_DECL(const amrex::Real dzi, + const amrex::Real dxi, + const amrex::Real dyi), + AMREX_D_DECL(const amrex::Real zmin, + const amrex::Real xmin, + const amrex::Real ymin), + const amrex::Real invvol, + const amrex::Dim3 lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); +#endif +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(xp, yp); +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(yp); +#endif + + constexpr int zdir = WARPX_ZINDEX; + constexpr int NODE = amrex::IndexType::NODE; + constexpr int CELL = amrex::IndexType::CELL; + + // wqx, wqy wqz are particle current in each direction +#if defined(WARPX_DIM_RZ) + // In RZ, wqx is actually wqr, and wqy is wqtheta + // Convert to cylindrical at the mid point + const amrex::Real xpmid = xp + relative_time*vx; + const amrex::Real ypmid = yp + relative_time*vy; + const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); + amrex::Real costheta; + amrex::Real sintheta; + if (rpmid > 0._rt) { + costheta = xpmid/rpmid; + sintheta = ypmid/rpmid; + } else { + costheta = 1._rt; + sintheta = 0._rt; + } + const Complex xy0 = Complex{costheta, sintheta}; + const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); + const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); +#else + const amrex::Real wqx = wq*invvol*vx; + const amrex::Real wqy = wq*invvol*vy; +#endif + const amrex::Real wqz = wq*invvol*vz; + + // --- Compute shape factors + Compute_shape_factor< depos_order > const compute_shape_factor; +#if (AMREX_SPACEDIM >= 2) + // x direction + // Get particle position after 1/2 push back in position +#if defined(WARPX_DIM_RZ) + // Keep these double to avoid bug in single precision + const double xmid = (rpmid - xmin)*dxi; +#else + const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +#endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current + // sx_j[xyz] shape factor along x for the centering of each current + // There are only two possible centerings, node or cell centered, so at most only two shape factor + // arrays will be needed. + // Keep these double to avoid bug in single precision + double sx_node[depos_order + 1] = {0.}; + double sx_cell[depos_order + 1] = {0.}; + int j_node = 0; + int j_cell = 0; + if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { + j_node = compute_shape_factor(sx_node, xmid); + } + if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { + j_cell = compute_shape_factor(sx_cell, xmid - 0.5); + } + + amrex::Real sx_jx[depos_order + 1] = {0._rt}; + amrex::Real sx_jy[depos_order + 1] = {0._rt}; + amrex::Real sx_jz[depos_order + 1] = {0._rt}; + for (int ix=0; ix<=depos_order; ix++) + { + sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + } + + int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); + int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); + int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); +#endif //AMREX_SPACEDIM >= 2 + +#if defined(WARPX_DIM_3D) + // y direction + // Keep these double to avoid bug in single precision + const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + double sy_node[depos_order + 1] = {0.}; + double sy_cell[depos_order + 1] = {0.}; + int k_node = 0; + int k_cell = 0; + if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { + k_node = compute_shape_factor(sy_node, ymid); + } + if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { + k_cell = compute_shape_factor(sy_cell, ymid - 0.5); + } + amrex::Real sy_jx[depos_order + 1] = {0._rt}; + amrex::Real sy_jy[depos_order + 1] = {0._rt}; + amrex::Real sy_jz[depos_order + 1] = {0._rt}; + for (int iy=0; iy<=depos_order; iy++) + { + sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + } + int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); + int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); + int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); +#endif + + // z direction + // Keep these double to avoid bug in single precision + const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + double sz_node[depos_order + 1] = {0.}; + double sz_cell[depos_order + 1] = {0.}; + int l_node = 0; + int l_cell = 0; + if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { + l_node = compute_shape_factor(sz_node, zmid); + } + if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { + l_cell = compute_shape_factor(sz_cell, zmid - 0.5); + } + amrex::Real sz_jx[depos_order + 1] = {0._rt}; + amrex::Real sz_jy[depos_order + 1] = {0._rt}; + amrex::Real sz_jz[depos_order + 1] = {0._rt}; + for (int iz=0; iz<=depos_order; iz++) + { + sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + } + int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); + int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); + int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + + // Deposit current into jx_arr, jy_arr and jz_arr +#if defined(WARPX_DIM_1D_Z) + for (int iz=0; iz<=depos_order; iz++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+l_jx+iz, 0, 0, 0), + sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+l_jy+iz, 0, 0, 0), + sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+l_jz+iz, 0, 0, 0), + sz_jz[iz]*wqz); + } +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), + sx_jx[ix]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), + sx_jy[ix]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), + sx_jz[ix]*sz_jz[iz]*wqz); +#if defined(WARPX_DIM_RZ) + Complex xy = xy0; // Note that xy is equal to e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 on the weighting comes from the normalization of the modes + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); + xy = xy*xy0; + } +#endif + } + } +#elif defined(WARPX_DIM_3D) + for (int iz=0; iz<=depos_order; iz++){ + for (int iy=0; iy<=depos_order; iy++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), + sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), + sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), + sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); + } + } + } +#endif +} + /** * \brief Current Deposition for thread thread_num * \tparam depos_order deposition order @@ -113,10 +353,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jy_type = jy_fab.box().type(); amrex::IntVect const jz_type = jz_fab.box().type(); - constexpr int zdir = WARPX_ZINDEX; - constexpr int NODE = amrex::IndexType::NODE; - constexpr int CELL = amrex::IndexType::CELL; - // Loop over particles and deposit into jx_fab, jy_fab and jz_fab #if defined(WARPX_USE_GPUCLOCK) amrex::Real* cost_real = nullptr; @@ -126,208 +362,181 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_deposit, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), cost_real); #endif + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + // --- Get particle quantities const amrex::Real gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + amrex::Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; } - amrex::ParticleReal xp, yp, zp; - GetPosition(ip, xp, yp, zp); + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); - const amrex::Real vx = uxp[ip]*gaminv; - const amrex::Real vy = uyp[ip]*gaminv; - const amrex::Real vz = uzp[ip]*gaminv; - // wqx, wqy wqz are particle current in each direction -#if defined(WARPX_DIM_RZ) - // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylindrical at the mid point - const amrex::Real xpmid = xp + relative_time*vx; - const amrex::Real ypmid = yp + relative_time*vy; - const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } - const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); - const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); -#else - const amrex::Real wqx = wq*invvol*vx; - const amrex::Real wqy = wq*invvol*vy; + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } #endif - const amrex::Real wqz = wq*invvol*vz; +} - // --- Compute shape factors - Compute_shape_factor< depos_order > const compute_shape_factor; -#if (AMREX_SPACEDIM >= 2) - // x direction - // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) - // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; -#else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +/** + * \brief Direct current deposition for thread thread_num for the implicit scheme + * The only difference from doDepositionShapeN is in how the particle gamma + * is calculated. + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time n. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum at time n+1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + const amrex::ParticleReal * const uxp_n, + const amrex::ParticleReal * const uyp_n, + const amrex::ParticleReal * const uzp_n, + const amrex::ParticleReal * const uxp, + const amrex::ParticleReal * const uyp, + const amrex::ParticleReal * const uzp, + const int * const ion_lev, + amrex::FArrayBox& jx_fab, + amrex::FArrayBox& jy_fab, + amrex::FArrayBox& jz_fab, + const long np_to_deposit, + const std::array& dx, + const std::array& xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real* cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); #endif - // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current - // sx_j[xyz] shape factor along x for the centering of each current - // There are only two possible centerings, node or cell centered, so at most only two shape factor - // arrays will be needed. - // Keep these double to avoid bug in single precision - double sx_node[depos_order + 1] = {0.}; - double sx_cell[depos_order + 1] = {0.}; - int j_node = 0; - int j_cell = 0; - if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { - j_node = compute_shape_factor(sx_node, xmid); - } - if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { - j_cell = compute_shape_factor(sx_cell, xmid - 0.5); - } - amrex::Real sx_jx[depos_order + 1] = {0._rt}; - amrex::Real sx_jy[depos_order + 1] = {0._rt}; - amrex::Real sx_jz[depos_order + 1] = {0._rt}; - for (int ix=0; ix<=depos_order; ix++) - { - sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - } +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif - int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); - int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); - int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); -#endif //AMREX_SPACEDIM >= 2 + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + const bool do_ionization = ion_lev; + const amrex::Real dzi = 1.0_rt/dx[2]; +#if defined(WARPX_DIM_1D_Z) + const amrex::Real invvol = dzi; +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real invvol = dxi*dzi; +#elif defined(WARPX_DIM_3D) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real dyi = 1.0_rt/dx[1]; + const amrex::Real invvol = dxi*dyi*dzi; +#endif +#if (AMREX_SPACEDIM >= 2) + const amrex::Real xmin = xyzmin[0]; +#endif #if defined(WARPX_DIM_3D) - // y direction - // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; - double sy_node[depos_order + 1] = {0.}; - double sy_cell[depos_order + 1] = {0.}; - int k_node = 0; - int k_cell = 0; - if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { - k_node = compute_shape_factor(sy_node, ymid); - } - if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { - k_cell = compute_shape_factor(sy_cell, ymid - 0.5); - } - amrex::Real sy_jx[depos_order + 1] = {0._rt}; - amrex::Real sy_jy[depos_order + 1] = {0._rt}; - amrex::Real sy_jz[depos_order + 1] = {0._rt}; - for (int iy=0; iy<=depos_order; iy++) - { - sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - } - int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); - int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); - int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); + const amrex::Real ymin = xyzmin[1]; #endif + const amrex::Real zmin = xyzmin[2]; - // z direction - // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; - double sz_node[depos_order + 1] = {0.}; - double sz_cell[depos_order + 1] = {0.}; - int l_node = 0; - int l_cell = 0; - if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { - l_node = compute_shape_factor(sz_node, zmid); - } - if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { - l_cell = compute_shape_factor(sz_cell, zmid - 0.5); - } - amrex::Real sz_jx[depos_order + 1] = {0._rt}; - amrex::Real sz_jy[depos_order + 1] = {0._rt}; - amrex::Real sz_jz[depos_order + 1] = {0._rt}; - for (int iz=0; iz<=depos_order; iz++) - { - sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - } - int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); - int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); - int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + amrex::Array4 const& jx_arr = jx_fab.array(); + amrex::Array4 const& jy_arr = jy_fab.array(); + amrex::Array4 const& jz_arr = jz_fab.array(); + amrex::IntVect const jx_type = jx_fab.box().type(); + amrex::IntVect const jy_type = jy_fab.box().type(); + amrex::IntVect const jz_type = jz_fab.box().type(); - // Deposit current into jx_arr, jy_arr and jz_arr -#if defined(WARPX_DIM_1D_Z) - for (int iz=0; iz<=depos_order; iz++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+l_jx+iz, 0, 0, 0), - sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+l_jy+iz, 0, 0, 0), - sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+l_jz+iz, 0, 0, 0), - sz_jz[iz]*wqz); - } + // Loop over particles and deposit into jx_fab, jy_fab and jz_fab +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } #endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int iz=0; iz<=depos_order; iz++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), - sx_jx[ix]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), - sx_jy[ix]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), - sx_jz[ix]*sz_jz[iz]*wqz); -#if defined(WARPX_DIM_RZ) - Complex xy = xy0; // Note that xy is equal to e^{i m theta} - for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { - // The factor 2 on the weighting comes from the normalization of the modes - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); - xy = xy*xy0; - } + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); #endif - } - } -#elif defined(WARPX_DIM_3D) - for (int iz=0; iz<=depos_order; iz++){ - for (int iy=0; iy<=depos_order; iy++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), - sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), - sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), - sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); - } - } + + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); + + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; } -#endif + + const amrex::Real relative_time = 0._rt; + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); + } ); #if defined(WARPX_USE_GPUCLOCK) @@ -790,17 +999,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // computes min/max positions of current contributions #if !defined(WARPX_DIM_1D_Z) int dil = 1, diu = 1; - if (i_old < i_new) dil = 0; - if (i_old > i_new) diu = 0; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } #endif #if defined(WARPX_DIM_3D) int djl = 1, dju = 1; - if (j_old < j_new) djl = 0; - if (j_old > j_new) dju = 0; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } #endif int dkl = 1, dku = 1; - if (k_old < k_new) dkl = 0; - if (k_old > k_new) dku = 0; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } #if defined(WARPX_DIM_3D) @@ -927,6 +1136,403 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } +/** + * \brief Esirkepov Current Deposition for thread thread_num for implicit scheme + * The difference from doEsirkepovDepositionShapeN is in how the old and new + * particles positions are determined and in how the particle gamma is calculated. + * + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); + Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invdtdx = 1.0_rt / (dt*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + +#if !defined(WARPX_DIM_1D_Z) + amrex::Real const wqx = wq*invdtdx; +#endif +#if defined(WARPX_DIM_3D) + amrex::Real const wqy = wq*invdtdy; +#endif + amrex::Real const wqz = wq*invdtdz; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_new, sintheta_new; + if (rp_new > 0._rt) { + costheta_new = xp_new/rp_new; + sintheta_new = yp_new/rp_new; + } else { + costheta_new = 1._rt; + sintheta_new = 0._rt; + } + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + amrex::Real costheta_old, sintheta_old; + if (rp_old > 0._rt) { + costheta_old = xp_old/rp_old; + sintheta_old = yp_old/rp_old; + } else { + costheta_old = 1._rt; + sintheta_old = 0._rt; + } + const Complex xy_new0 = Complex{costheta_new, sintheta_new}; + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + const Complex xy_old0 = Complex{costheta_old, sintheta_old}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + +#if defined(WARPX_DIM_RZ) + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#endif + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_new[depos_order + 3] = {0.}; + double sx_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_new = compute_shape_factor(sx_new+1, x_new); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_new = compute_shape_factor(sy_new+1, y_new); + const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); +#endif + const int k_new = compute_shape_factor(sz_new+1, z_new); + const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil = 1, diu = 1; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl = 1, dju = 1; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } +#endif + int dkl = 1, dku = 1; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } + +#if defined(WARPX_DIM_3D) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int j=djl; j<=depos_order+2-dju; j++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*( + one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) + +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); + } + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdyj = 0._rt; + for (int j=djl; j<=depos_order+1-dju; j++) { + sdyj += wqy*(sy_old[j] - sy_new[j])*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); + } + } + } + for (int j=djl; j<=depos_order+2-dju; j++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*( + one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) + +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djr_cmplx = 2._rt *sdxi*xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djr_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djr_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + Real const sdyj = wq*vy*invvol*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); +#if defined(WARPX_DIM_RZ) + Complex xy_new = xy_new0; + Complex xy_mid = xy_mid0; + Complex xy_old = xy_old0; + // Throughout the following loop, xy_ takes the value e^{i m theta_} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + // The minus sign comes from the different convention with respect to Davidson et al. + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djt_cmplx.imag()); + xy_new = xy_new*xy_new0; + xy_mid = xy_mid*xy_mid0; + xy_old = xy_old*xy_old0; + } +#endif + } + } + for (int i=dil; i<=depos_order+2-diu; i++) { + Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djz_cmplx = 2._rt * sdzk * xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djz_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djz_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdxi = wq*vx*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+k_new-1+k, 0, 0, 0), sdxi); + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdyj = wq*vy*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+k_new-1+k, 0, 0, 0), sdyj); + } + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); + } +#endif + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } +#endif +} + /** * \brief Vay current deposition * ( Vay et al, 2013) @@ -1068,7 +1674,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, + uzp[ip] * uzp[ip] * invcsq); // Product of particle charges and weights amrex::Real wq = q * wp[ip]; - if (do_ionization) wq *= ion_lev[ip]; + if (do_ionization) { wq *= ion_lev[ip]; } // Current particle positions (in physical units) amrex::ParticleReal xp, yp, zp; diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 573491645ea..61aeb19aea1 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -33,9 +33,11 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT m_adk_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_exp_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_power; + const amrex::Real* AMREX_RESTRICT m_adk_correction_factors; int comp; int m_atomic_number; + int m_do_adk_correction = 0; GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; @@ -82,8 +84,10 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT a_adk_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_power, + const amrex::Real* AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset = 0) noexcept; template @@ -133,9 +137,16 @@ struct IonizationFilterFunc ); // Compute probability of ionization p - const amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * + amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * std::pow(E, m_adk_power[ion_lev]) * std::exp( m_adk_exp_prefactor[ion_lev]/E ); + // if requested, do Zhang's correction of ADK + if (m_do_adk_correction) { + const amrex::Real r = E / m_adk_correction_factors[3]; + w_dtau *= std::exp(m_adk_correction_factors[0]*r*r+m_adk_correction_factors[1]*r+ + m_adk_correction_factors[2]); + } + const amrex::Real p = 1._rt - std::exp( - w_dtau ); const amrex::Real random_draw = amrex::Random(engine); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index c3681a30cad..b7b91e4d4e3 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -30,15 +30,19 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_power, + const amrex::Real* const AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset) noexcept: m_ionization_energies{a_ionization_energies}, m_adk_prefactor{a_adk_prefactor}, m_adk_exp_prefactor{a_adk_exp_prefactor}, m_adk_power{a_adk_power}, + m_adk_correction_factors{a_adk_correction_factors}, comp{a_comp}, m_atomic_number{a_atomic_number}, + m_do_adk_correction{a_do_adk_correction}, m_Ex_external_particle{E_external_particle[0]}, m_Ey_external_particle{E_external_particle[1]}, m_Ez_external_particle{E_external_particle[2]}, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H index f80705fd043..3f9ef133477 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H @@ -170,8 +170,9 @@ public: amrex::ParticleReal, pxr_p::unit_system::SI>( px, py, pz); if (gamma_photon < pxr_m::two || - chi_phot < m_bw_minimum_chi_phot) + chi_phot < m_bw_minimum_chi_phot) { return 0; + } const auto is_out = pxr_bw::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp index 968ab8b85f1..da50d41dc8f 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp @@ -69,7 +69,7 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -80,8 +80,9 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( m_dndt_table = BW_dndt_table{raw_dndt_table}; m_pair_prod_table = BW_pair_prod_table{raw_pair_prod_table}; - if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) + if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) { return false; + } m_bw_minimum_chi_phot = bw_minimum_chi_phot; @@ -104,8 +105,9 @@ void BreitWheelerEngine::init_builtin_tables( vector BreitWheelerEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_pair_prod = m_pair_prod_table.serialize(); @@ -114,10 +116,12 @@ vector BreitWheelerEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_pair_prod) + } + for (const auto& tmp : data_pair_prod) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H index ceacb2016e8..3ae27e411b6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H @@ -153,8 +153,9 @@ public: const auto chi_part = QedUtils::chi_ele_pos( m_e*ux, m_e*uy, m_e*uz, ex, ey, ez, bx, by, bz); - if (chi_part < m_qs_minimum_chi_part) + if (chi_part < m_qs_minimum_chi_part) { return 0; + } const auto is_out = pxr_qs::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp index c6548d91612..59a881814c6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp @@ -68,7 +68,7 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -79,8 +79,9 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( m_dndt_table = QS_dndt_table{raw_dndt_table}; m_phot_em_table = QS_phot_em_table{raw_phot_em_table}; - if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) + if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) { return false; + } m_qs_minimum_chi_part = qs_minimum_chi_part; @@ -103,8 +104,9 @@ void QuantumSynchrotronEngine::init_builtin_tables( vector QuantumSynchrotronEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_phot_em = m_phot_em_table.serialize(); @@ -113,10 +115,12 @@ vector QuantumSynchrotronEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_phot_em) + } + for (const auto& tmp : data_phot_em) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index 54cf09fb3d7..354d57e2c98 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -164,7 +164,7 @@ struct GeometryFilter AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator () (const SuperParticleType& p, const amrex::RandomEngine&) const noexcept { - if ( !m_is_active ) return true; + if ( !m_is_active ) { return true; } return ! (AMREX_D_TERM( (p.pos(0) < m_domain.lo(0)) || (p.pos(0) > m_domain.hi(0) ), || (p.pos(1) < m_domain.lo(1)) || (p.pos(1) > m_domain.hi(1) ), || (p.pos(2) < m_domain.lo(2)) || (p.pos(2) > m_domain.hi(2) ))); diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 1b5cede3c6f..dd6b7276681 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -451,6 +451,435 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif } +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Esirkepov current deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNEsirkepovStencilImplicit ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_E_new[depos_order + 3] = {0.}; + double sx_E_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_E_new[depos_order + 3] = {0.}; + double sy_E_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_E_new[depos_order + 3] = {0.}; + double sz_E_old[depos_order + 3] = {0.}; + +#if defined(WARPX_DIM_3D) + double sx_B_new[depos_order + 3] = {0.}; + double sx_B_old[depos_order + 3] = {0.}; + double sy_B_new[depos_order + 3] = {0.}; + double sy_B_old[depos_order + 3] = {0.}; + double sz_B_new[depos_order + 3] = {0.}; + double sz_B_old[depos_order + 3] = {0.}; +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Special shape functions are needed for By which is cell + // centered in both x and z. One lower order shape function is used. + double sx_By_new[depos_order + 2] = {0.}; + double sx_By_old[depos_order + 2] = {0.}; + double sz_By_new[depos_order + 2] = {0.}; + double sz_By_old[depos_order + 2] = {0.}; +#endif + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_E_new = compute_shape_factor(sx_E_new+1, x_new); + const int i_E_old = compute_shifted_shape_factor(sx_E_old, x_old, i_E_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_E_new = compute_shape_factor(sy_E_new+1, y_new); + const int j_E_old = compute_shifted_shape_factor(sy_E_old, y_old, j_E_new); +#endif + const int k_E_new = compute_shape_factor(sz_E_new+1, z_new); + const int k_E_old = compute_shifted_shape_factor(sz_E_old, z_old, k_E_new); + +#if defined(WARPX_DIM_3D) + const int i_B_new = compute_shape_factor(sx_B_new+1, x_new - 0.5_rt); + const int i_B_old = compute_shifted_shape_factor(sx_B_old, x_old - 0.5_rt, i_B_new); + const int j_B_new = compute_shape_factor(sy_B_new+1, y_new - 0.5_rt); + const int j_B_old = compute_shifted_shape_factor(sy_B_old, y_old - 0.5_rt, j_B_new); + const int k_B_new = compute_shape_factor(sz_B_new+1, z_new - 0.5_rt); + const int k_B_old = compute_shifted_shape_factor(sz_B_old, z_old - 0.5_rt, k_B_new); +#endif + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil_E = 1, diu_E = 1; + if (i_E_old < i_E_new) { dil_E = 0; } + if (i_E_old > i_E_new) { diu_E = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl_E = 1, dju_E = 1; + if (j_E_old < j_E_new) { djl_E = 0; } + if (j_E_old > j_E_new) { dju_E = 0; } +#endif + int dkl_E = 1, dku_E = 1; + if (k_E_old < k_E_new) { dkl_E = 0; } + if (k_E_old > k_E_new) { dku_E = 0; } + +#if defined(WARPX_DIM_3D) + int dil_B = 1, diu_B = 1; + if (i_B_old < i_B_new) { dil_B = 0; } + if (i_B_old > i_B_new) { diu_B = 0; } + int djl_B = 1, dju_B = 1; + if (j_B_old < j_B_new) { djl_B = 0; } + if (j_B_old > j_B_new) { dju_B = 0; } + int dkl_B = 1, dku_B = 1; + if (k_B_old < k_B_new) { dkl_B = 0; } + if (k_B_old > k_B_new) { dku_B = 0; } +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Compute_shape_factor< depos_order-1 > compute_shape_factor_By; + Compute_shifted_shape_factor< depos_order-1 > compute_shifted_shape_factor_By; + const int i_By_new = compute_shape_factor_By(sx_By_new+1, x_new - 0.5_rt); + const int i_By_old = compute_shifted_shape_factor_By(sx_By_old, x_old - 0.5_rt, i_By_new); + const int k_By_new = compute_shape_factor_By(sz_By_new+1, z_new - 0.5_rt); + const int k_By_old = compute_shifted_shape_factor_By(sz_By_old, z_old - 0.5_rt, k_By_new); + int dil_By = 1, diu_By = 1; + if (i_By_old < i_By_new) { dil_By = 0; } + if (i_By_old > i_By_new) { diu_By = 0; } + int dkl_By = 1, dku_By = 1; + if (k_By_old < k_By_new) { dkl_By = 0; } + if (k_By_old > k_By_new) { dku_By = 0; } +#endif + +#if defined(WARPX_DIM_3D) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + amrex::Real sdzjk = one_third*(sy_E_new[j]*sz_E_new[k] + sy_E_old[j]*sz_E_old[k]) + +one_sixth*(sy_E_new[j]*sz_E_old[k] + sy_E_old[j]*sz_E_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdyik = one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_E; j<=depos_order+1-dju_E; j++) { + sdyj += (sy_E_old[j] - sy_E_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdzij = one_third*(sx_E_new[i]*sy_E_new[j] + sx_E_old[i]*sy_E_old[j]) + +one_sixth*(sx_E_new[i]*sy_E_old[j] + sx_E_old[i]*sy_E_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + amrex::Real sdzjk = one_third*(sy_B_new[j]*sz_B_new[k] + sy_B_old[j]*sz_B_old[k]) + +one_sixth*(sy_B_new[j]*sz_B_old[k] + sy_B_old[j]*sz_B_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_B; i<=depos_order+1-diu_B; i++) { + sdxi += (sx_B_old[i] - sx_B_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Bxp += Bx_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdyik = one_third*(sx_B_new[i]*sz_B_new[k] + sx_B_old[i]*sz_B_old[k]) + +one_sixth*(sx_B_new[i]*sz_B_old[k] + sx_B_old[i]*sz_B_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_B; j<=depos_order+1-dju_B; j++) { + sdyj += (sy_B_old[j] - sy_B_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Byp += By_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdzij = one_third*(sx_B_new[i]*sy_B_new[j] + sx_B_old[i]*sy_B_old[j]) + +one_sixth*(sx_B_new[i]*sy_B_old[j] + sx_B_old[i]*sy_B_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_B; k<=depos_order+1-dku_B; k++) { + sdzk += (sz_B_old[k] - sz_B_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bzp += Bz_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + Bzp += Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdyj; + } + } + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + Bxp += Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + } + } + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + Byp += By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 0)*sdyj; + } + } + +#ifdef WARPX_DIM_RZ + Complex xy_mid = xy_mid0; + + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + + // Gather field on particle Exp from field on grid ex_arr + // Gather field on particle Bzp from field on grid bz_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + const amrex::Real dEx = (+ Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBz = (+ Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Exp += dEx*sdxiov*sdzk; + Bzp += dBz*sdxiov*sdzk; + } + } + // Gather field on particle Eyp from field on grid ey_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + const amrex::Real dEy = (+ Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Eyp += dEy*sdyj; + } + } + // Gather field on particle Ezp from field on grid ez_arr + // Gather field on particle Bxp from field on grid bx_arr + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + const amrex::Real dEz = (+ Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBx = (+ Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Ezp += dEz*sdzkov*sdxi; + Bxp += dBx*sdzkov*sdxi; + } + } + // Gather field on particle Byp from field on grid by_arr + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + const amrex::Real dBy = (+ By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode)*xy_mid.imag()); + Byp += dBy*sdyj; + } + } + xy_mid = xy_mid*xy_mid0; + } + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real const sdzk = 0.5_rt*(sz_E_old[k] + sz_E_new[k]); + Exp += Ex_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Eyp += Ey_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Bzp += Bz_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + } + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bxp += Bx_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Byp += By_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Ezp += Ez_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + } +#endif +} + /** * \brief Field gather for particles * @@ -606,4 +1035,96 @@ void doGatherShapeN (const amrex::ParticleReal xp, } } + +/** + * \brief Field gather for a single particle + * + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half time level (n + half) + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param ex_arr,ey_arr,ez_arr Array4 of the electric field, either full array or tile. + * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. + * \param ex_type,ey_type,ez_type IndexType of the electric field + * \param bx_type,by_type,bz_type IndexType of the magnetic field + * \param dx_arr 3D cell spacing + * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param nox order of the particle shape function + * \param galerkin_interpolation whether to use lower order in v + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNImplicit ( + const amrex::ParticleReal xp_n, + const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + const amrex::ParticleReal xp_nph, + const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& ex_arr, + amrex::Array4 const& ey_arr, + amrex::Array4 const& ez_arr, + amrex::Array4 const& bx_arr, + amrex::Array4 const& by_arr, + amrex::Array4 const& bz_arr, + const amrex::IndexType ex_type, + const amrex::IndexType ey_type, + const amrex::IndexType ez_type, + const amrex::IndexType bx_type, + const amrex::IndexType by_type, + const amrex::IndexType bz_type, + const amrex::GpuArray& dx_arr, + const amrex::GpuArray& xyzmin_arr, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes, + const int nox, + const bool galerkin_interpolation) +{ + if (galerkin_interpolation) { + if (nox == 1) { + doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } else { + if (nox == 1) { + doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } +} + #endif // FIELDGATHER_H_ diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 5e74a409824..9fd534e33d9 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -58,7 +58,7 @@ struct GetExternalEBField std::optional d_lattice_element_finder; - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.has_value()); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -78,7 +78,7 @@ struct GetExternalEBField field_Bx, field_By, field_Bz); } - if (m_Etype == None && m_Btype == None) return; + if (m_Etype == None && m_Btype == None) { return; } amrex::ParticleReal Ex = 0._prt; amrex::ParticleReal Ey = 0._prt; diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 4aeb6ce05b9..8e3c679aa3a 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -31,12 +31,12 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_Etype = Unknown; m_Btype = Unknown; - if (mypc.m_E_ext_particle_s == "none") m_Etype = None; - if (mypc.m_B_ext_particle_s == "none") m_Btype = None; + if (mypc.m_E_ext_particle_s == "none") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "none") { m_Btype = None; } // These lines will be removed once the user interface is redefined and the CI tests updated - if (mypc.m_E_ext_particle_s == "constant") m_Etype = None; - if (mypc.m_B_ext_particle_s == "constant") m_Btype = None; + if (mypc.m_E_ext_particle_s == "constant") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "constant") { m_Btype = None; } if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function" || mypc.m_B_ext_particle_s == "parse_b_ext_particle_function" || @@ -68,10 +68,10 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset if (mypc.m_E_ext_particle_s == "repeated_plasma_lens" || mypc.m_B_ext_particle_s == "repeated_plasma_lens") { - if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") m_Etype = RepeatedPlasmaLens; - if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") m_Btype = RepeatedPlasmaLens; + if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") { m_Etype = RepeatedPlasmaLens; } + if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_Btype = RepeatedPlasmaLens; } m_dt = warpx.getdt(a_pti.GetLevel()); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 8605868a87e..4cddfa5a342 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -41,14 +41,14 @@ struct ScaleFields { using namespace amrex::literals; - if (!m_do_scale) return; + if (!m_do_scale) { return; } // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - const amrex::Real dtscale = m_dt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost); - if (0._rt < dtscale && dtscale < m_dt) + const amrex::Real dtscale = 1._rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; + if (0._rt < dtscale && dtscale < 1._rt) { Exp *= dtscale; Eyp *= dtscale; diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index b722daa40fc..e6fa308431c 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -10,6 +10,7 @@ #define WARPX_LaserParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "WarpXParticleContainer.H" @@ -58,8 +59,7 @@ public: amrex::ParticleTile, NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/, - const amrex::RandomEngine& /*engine*/) final {} + const int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; @@ -74,7 +74,7 @@ public: const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) final; + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 7d918d5dfb5..fd4e9397253 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -9,6 +9,7 @@ #include "LaserParticleContainer.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "Particles/LaserParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -279,7 +280,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, void LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) { - if (!m_enabled) return; + if (!m_enabled) { return; } // Input parameter injection_box contains small box where injection // should occur. @@ -321,7 +322,7 @@ LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) void LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) { - if (!m_enabled) return; + if (!m_enabled) { return; } const int dir = WarpX::moving_window_dir; if (do_continuous_injection and (WarpX::gamma_boost > 1)){ @@ -350,7 +351,7 @@ LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) void LaserParticleContainer::InitData () { - if (!m_enabled) return; + if (!m_enabled) { return; } // Call InitData on max level to inject one laser particle per // finest cell. @@ -367,7 +368,7 @@ LaserParticleContainer::InitData () void LaserParticleContainer::InitData (int lev) { - if (!m_enabled) return; + if (!m_enabled) { return; } // spacing of laser particles in the laser plane. // has to be done after geometry is set up. @@ -541,7 +542,7 @@ LaserParticleContainer::InitData (int lev) amrex::Vector particle_uy(np, 0.0); amrex::Vector particle_uz(np, 0.0); - if (Verbose()) amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); + if (Verbose()) { amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); } amrex::Vector> attr; attr.push_back(particle_w); amrex::Vector> attr_int; @@ -561,12 +562,12 @@ LaserParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, - Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition) + Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); - if (!m_enabled) return; + if (!m_enabled) { return; } Real t_lab = t; if (WarpX::gamma_boost > 1) { @@ -664,14 +665,14 @@ LaserParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } @@ -701,7 +702,7 @@ LaserParticleContainer::Evolve (int lev, void LaserParticleContainer::PostRestart () { - if (!m_enabled) return; + if (!m_enabled) { return; } Real Sx, Sy; const int lev = finestLevel(); @@ -869,7 +870,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, np, [=] AMREX_GPU_DEVICE (int i) { // Calculate the velocity according to the amplitude of E - const Real sign_charge = (pwp[i]>0) ? 1 : -1; + const Real sign_charge = (pwp[i]>0) ? -1 : 1; const Real v_over_c = sign_charge * tmp_mobility * amplitude[i]; AMREX_ALWAYS_ASSERT_WITH_MESSAGE(amrex::Math::abs(v_over_c) < amrex::Real(1.), "Error: calculated laser particle velocity greater than c." diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index c32ff07464d..285b0a9777c 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -14,6 +14,7 @@ #include "MultiParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Collision/CollisionHandler.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper_fwd.H" @@ -107,7 +108,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false); + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit); /// /// This pushes the particle positions by one half time step for all the species in the @@ -331,6 +333,9 @@ public: [[nodiscard]] int getSpeciesID (std::string product_str) const; + amrex::Vector>::iterator begin() {return allcontainers.begin();} + amrex::Vector>::iterator end() {return allcontainers.end();} + protected: #ifdef WARPX_QED diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 4eaf89c2aa4..a31f426a0e4 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -455,21 +455,22 @@ MultiParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { if (! skip_deposition) { jx.setVal(0.0); jy.setVal(0.0); jz.setVal(0.0); - if (cjx) cjx->setVal(0.0); - if (cjy) cjy->setVal(0.0); - if (cjz) cjz->setVal(0.0); - if (rho) rho->setVal(0.0); - if (crho) crho->setVal(0.0); + if (cjx) { cjx->setVal(0.0); } + if (cjy) { cjy->setVal(0.0); } + if (cjz) { cjz->setVal(0.0); } + if (rho) { rho->setVal(0.0); } + if (crho) { crho->setVal(0.0); } } for (auto& pc : allcontainers) { pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition); + rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); } } @@ -507,8 +508,9 @@ MultiParticleContainer::GetZeroChargeDensity (const int lev) const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } auto zero_rho = std::make_unique(nba, dmap, WarpX::ncomps, ng_rho); zero_rho->setVal(amrex::Real(0.0)); @@ -555,7 +557,7 @@ MultiParticleContainer::DepositCharge ( } // Push the particles in time, if needed - if (relative_time != 0.) PushX(relative_time); + if (relative_time != 0.) { PushX(relative_time); } bool const local = true; bool const reset = false; @@ -564,13 +566,13 @@ MultiParticleContainer::DepositCharge ( // Call the deposition kernel for each species for (auto& pc : allcontainers) { - if (pc->do_not_deposit) continue; + if (pc->do_not_deposit) { continue; } pc->DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } // Push the particles back in time - if (relative_time != 0.) PushX(-relative_time); + if (relative_time != 0.) { PushX(-relative_time); } #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) @@ -586,7 +588,7 @@ MultiParticleContainer::GetChargeDensity (int lev, bool local) std::unique_ptr rho = GetZeroChargeDensity(lev); for (auto& container : allcontainers) { - if (container->do_not_deposit) continue; + if (container->do_not_deposit) { continue; } const std::unique_ptr rhoi = container->GetChargeDensity(lev, true); MultiFab::Add(*rho, *rhoi, 0, 0, rho->nComp(), rho->nGrowVect()); } @@ -710,7 +712,7 @@ MultiParticleContainer::SetParticleDistributionMap (int lev, DistributionMapping void MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->ContinuousInjection(injection_box); } @@ -720,7 +722,7 @@ MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const void MultiParticleContainer::UpdateAntennaPosition (const amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->UpdateAntennaPosition(dt); } @@ -731,7 +733,7 @@ int MultiParticleContainer::doContinuousInjection () const { int warpx_do_continuous_injection = 0; - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ warpx_do_continuous_injection = 1; } @@ -746,7 +748,7 @@ MultiParticleContainer::doContinuousInjection () const void MultiParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ pc->ContinuousFluxInjection(t, dt); } } @@ -790,10 +792,10 @@ MultiParticleContainer::mapSpeciesProduct () #ifdef WARPX_QED if (m_do_qed_schwinger) { - m_qed_schwinger_ele_product = - getSpeciesID(m_qed_schwinger_ele_product_name); - m_qed_schwinger_pos_product = - getSpeciesID(m_qed_schwinger_pos_product_name); + m_qed_schwinger_ele_product = + getSpeciesID(m_qed_schwinger_ele_product_name); + m_qed_schwinger_pos_product = + getSpeciesID(m_qed_schwinger_pos_product_name); } #endif } @@ -870,7 +872,7 @@ MultiParticleContainer::doFieldIonization (int lev, auto& pc_product = allcontainers[pc_source->ionization_product]; const SmartCopyFactory copy_factory(*pc_source, *pc_product); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); auto Copy = copy_factory.getSmartCopy(); auto Transform = IonizationTransformFunc(); @@ -899,7 +901,7 @@ MultiParticleContainer::doFieldIonization (int lev, Bx[pti], By[pti], Bz[pti]); const auto np_dst = dst_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + const auto num_added = filterCopyTransformParticles<1>(*pc_product, dst_tile, src_tile, np_dst, Filter, Copy, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); @@ -976,11 +978,13 @@ void MultiParticleContainer::InitQED () } } - if(m_nspecies_quantum_sync != 0) + if(m_nspecies_quantum_sync != 0) { InitQuantumSync(); + } - if(m_nspecies_breit_wheeler !=0) + if(m_nspecies_breit_wheeler !=0) { InitBreitWheeler(); + } } @@ -1065,8 +1069,9 @@ void MultiParticleContainer::InitBreitWheeler () // considered for pair production. If a photon has chi < chi_min, // the optical depth is not evolved and photon generation is ignored amrex::Real bw_minimum_chi_part; - if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) + if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) { WARPX_ABORT_WITH_MESSAGE("qed_bw.chi_min should be provided!"); + } pp_qed_bw.query("lookup_table_mode", lookup_table_mode); if(lookup_table_mode.empty()){ @@ -1383,7 +1388,7 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; - const auto num_added = filterCreateTransformFromFAB<1>( dst_ele_tile, + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, Transform); @@ -1488,7 +1493,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, // in pc_product_ele and positrons in pc_product_pos for (auto& pc_source : allcontainers){ - if(!pc_source->has_breit_wheeler()) continue; + if(!pc_source->has_breit_wheeler()) { continue; } // Get product species auto& pc_product_ele = @@ -1498,7 +1503,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const SmartCopyFactory copy_factory_ele(*pc_source, *pc_product_ele); const SmartCopyFactory copy_factory_pos(*pc_source, *pc_product_pos); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPairGenerationFilterFunc(); const auto CopyEle = copy_factory_ele.getSmartCopy(); @@ -1536,7 +1541,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const auto np_dst_ele = dst_ele_tile.numParticles(); const auto np_dst_pos = dst_pos_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(*pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, src_tile, np_dst_ele, np_dst_pos, Filter, CopyEle, CopyPos, Transform); @@ -1578,7 +1583,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, allcontainers[pc_source->m_qed_quantum_sync_phot_product]; const SmartCopyFactory copy_factory_phot(*pc_source, *pc_product_phot); - auto phys_pc_ptr = + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPhotonEmissionFilterFunc(); @@ -1616,7 +1621,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, const auto np_dst = dst_tile.numParticles(); const auto num_added = - filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + filterCopyTransformParticles<1>(*pc_product_phot, dst_tile, src_tile, np_dst, Filter, CopyPhot, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index c9f66e6c2b0..3be0886425d 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -102,9 +102,9 @@ public: NamedComponentParticleContainer& operator= ( const NamedComponentParticleContainer & ) = delete; /** Move constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer ( NamedComponentParticleContainer && ) noexcept = default; /** Move operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) noexcept = default; /** Create an empty particle container * diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 1f4fb0372eb..78b090056b1 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -27,7 +27,7 @@ struct ParticleBoundaries void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); - bool CheckAll (ParticleBoundaryType bc); + [[nodiscard]] bool CheckAll (ParticleBoundaryType bc) const; void BuildReflectionModelParsers (); diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index a6e80717e81..5b51fa3fd25 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -54,7 +54,7 @@ ParticleBoundaries::SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType } bool -ParticleBoundaries::CheckAll (ParticleBoundaryType bc) +ParticleBoundaries::CheckAll (ParticleBoundaryType bc) const { return (data.xmin_bc == bc && data.xmax_bc == bc #ifdef WARPX_DIM_3D diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 662e1240f8b..bbec1f54e01 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -140,10 +140,10 @@ namespace ApplyParticleBoundaries { uy = ur*std::sin(y) + ut*std::cos(y); } #else - if (change_sign_ux) ux = -ux; - if (change_sign_uy) uy = -uy; + if (change_sign_ux) { ux = -ux; } + if (change_sign_uy) { uy = -uy; } #endif - if (change_sign_uz) uz = -uz; + if (change_sign_uz) { uz = -uz; } } } diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index b6eba51347c..54c4396379d 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -51,12 +51,15 @@ struct CopyAndTimestamp { int src_i, int dst_i) const noexcept { dst.m_aos[dst_i] = src.m_aos[src_i]; - for (int j = 0; j < SrcData::NAR; ++j) + for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_real; ++j) + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_int; ++j) + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; + } dst.m_runtime_idata[m_index][dst_i] = m_step; } }; @@ -115,7 +118,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () #endif // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { - if (m_do_boundary_buffer[i][ispecies]) m_do_any_boundary[i] = 1; + if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } } } @@ -146,7 +149,7 @@ void ParticleBoundaryBuffer::printNumParticles () const { { for (int iside = 0; iside < 2; ++iside) { - auto& buffer = m_particle_containers[2*idim+iside]; + const auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; @@ -210,7 +213,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { auto& species_buffer = buffer[ispecies]; - if (species_buffer.isDefined()) species_buffer.clearParticles(); + if (species_buffer.isDefined()) { species_buffer.clearParticles(); } } } @@ -227,13 +230,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (geom.isPeriodic(idim)) continue; + if (geom.isPeriodic(idim)) { continue; } for (int iside = 0; iside < 2; ++iside) { auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { - if (!m_do_boundary_buffer[2*idim+iside][i]) continue; + if (!m_do_boundary_buffer[2*idim+iside][i]) { continue; } const WarpXParticleContainer& pc = mypc.GetParticleContainer(i); if (!buffer[i].isDefined()) { @@ -250,13 +253,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; + if(plevel.find(index) == plevel.end()) { continue; } auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( lev, pti.index(), pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } auto predicate = IsOutsideDomainBoundary{plo, phi, idim, iside}; @@ -324,7 +327,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 09ff5cce6f0..870fc82bd0f 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -8,6 +8,12 @@ #ifndef DEFAULTINITIALIZATION_H_ #define DEFAULTINITIALIZATION_H_ +#include +#ifdef WARPX_QED +# include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" +# include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" +#endif + #include #include @@ -81,4 +87,197 @@ int initializeIntValue (const InitializationPolicy policy) noexcept } } +namespace ParticleCreation { + + /** + * \brief Default initialize runtime attributes in a tile. This routine does not initialize the + * first n_external_attr_real real attributes and the first n_external_attr_int integer + * attributes, which have been in principle externally set elsewhere. + * + * @tparam[in] The type of the particle tile to operate on (e.g. could use different allocators) + * @param[inout] ptile the tile in which attributes are initialized + * @param[in] n_external_attr_real The number of real attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] n_external_attr_int The number of integer attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] user_real_attribs The names of the real components for this particle tile + * @param[in] user_int_attribs The names of the int components for this particle tile + * @param[in] particle_comps map between particle component index and component name for real comps + * @param[in] particle_icomps map between particle component index and component name for int comps + * @param[in] user_real_attrib_parser the parser functions used to initialize the user real components + * @param[in] user_int_attrib_parser the parser functions used to initialize the user int components + * @param[in] do_qed_comps whether to initialize the qed components (these are usually handled by + * SmartCopy, but NOT when adding particles in AddNParticles) + * @param[in] p_bw_engine the engine to use for setting the breit-wheeler component for QED + * @param[in] p_qs_engine the engine to use for setting the quantum synchrotron component for QED + * @param[in] ionization_initial_level the ionization level particles created should start at + * @param[in] start the index to start initializing particles + * @param[in] stop the index to stop initializing particles + */ +template +void DefaultInitializeRuntimeAttributes (PTile& ptile, + const int n_external_attr_real, + const int n_external_attr_int, + const std::vector& user_real_attribs, + const std::vector& user_int_attribs, + const std::map& particle_comps, + const std::map& particle_icomps, + const std::vector& user_real_attrib_parser, + const std::vector& user_int_attrib_parser, +#ifdef WARPX_QED + const bool do_qed_comps, + BreitWheelerEngine* p_bw_engine, + QuantumSynchrotronEngine* p_qs_engine, +#endif + const int ionization_initial_level, + int start, int stop) +{ + using namespace amrex::literals; + + // Preparing data needed for user defined attributes + const auto n_user_real_attribs = static_cast(user_real_attribs.size()); + const auto n_user_int_attribs = static_cast(user_int_attribs.size()); + const auto get_position = GetParticlePosition(ptile); + const auto soa = ptile.getParticleTileData(); + const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; + const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; + const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; + constexpr int lev = 0; + const amrex::Real t = WarpX::GetInstance().gett_new(lev); + + // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes + for (int j = PIdx::nattribs + n_external_attr_real; j < ptile.NumRealComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetRealData(j).data(); +#ifdef WARPX_QED + // Current runtime comp is quantum synchrotron optical depth + if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && + particle_comps.at("opticalDepthQSR") == j) + { + if (!do_qed_comps) { continue; } + const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = + p_qs_engine->build_optical_depth_functor(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = quantum_sync_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = quantum_sync_get_opt(amrex::RandomEngine{}); + } + } + } + + // Current runtime comp is Breit-Wheeler optical depth + if (particle_comps.find("opticalDepthBW") != particle_comps.end() && + particle_comps.at("opticalDepthBW") == j) + { + if (!do_qed_comps) { continue; } + const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = + p_bw_engine->build_optical_depth_functor();; + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = breit_wheeler_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = breit_wheeler_get_opt(amrex::RandomEngine{}); + } + } + } +#endif + + for (int ia = 0; ia < n_user_real_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_comps.find(user_real_attribs[ia]) != particle_comps.end() && + particle_comps.at(user_real_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_real_attrib_parserexec = + user_real_attrib_parser[ia]->compile<7>(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + } + } + } + } + } + + // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes + for (int j = n_external_attr_int; j < ptile.NumIntComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetIntData(j).data(); + + // Current runtime comp is ionization level + if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && + particle_icomps.at("ionizationLevel") == j) + { + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + attr_ptr[ip] = ionization_initial_level; + }); + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = ionization_initial_level; + } + } + } + + for (int ia = 0; ia < n_user_int_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_icomps.find(user_int_attribs[ia]) != particle_icomps.end() && + particle_icomps.at(user_int_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_int_attrib_parserexec = + user_int_attrib_parser[ia]->compile<7>(); + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + }); + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + } + } + } + } + } +} + +} + #endif diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 7a85f59318f..095ebbf6a85 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -8,6 +8,8 @@ #ifndef FILTER_COPY_TRANSFORM_H_ #define FILTER_COPY_TRANSFORM_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" + #include #include @@ -47,23 +49,26 @@ * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, + Index* mask, Index dst_index, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst.resize(std::max(dst_index + num_added, dst.numParticles())); + auto old_np = dst.size(); + auto new_np = std::max(dst_index + num_added, dst.numParticles()); + dst.resize(new_np); - const auto p_offsets = offsets.dataPtr(); + auto *const p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst_data = dst.getParticleTileData(); @@ -80,6 +85,21 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst, + 0, 0, + pc.getUserRealAttribs(), pc.getUserIntAttribs(), + pc.getParticleComps(), pc.getParticleiComps(), + pc.getUserRealAttribParser(), + pc.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc.get_breit_wheeler_engine_ptr(), + pc.get_quantum_sync_engine_ptr(), +#endif + pc.getIonizationInitialLevel(), + old_np, new_np); + Gpu::synchronize(); return num_added; } @@ -121,19 +141,19 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index dst_index, PredFunc&& filter, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -142,9 +162,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst, src, mask.dataPtr(), dst_index, - std::forward(copy), - std::forward(transform)); + return filterCopyTransformParticles(pc, dst, src, mask.dataPtr(), dst_index, + std::forward(copy), + std::forward(transform)); } /** @@ -188,10 +208,10 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, Index dst1_index, Index dst2_index, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -199,15 +219,20 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst1_data = dst1.getParticleTileData(); @@ -230,6 +255,35 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -276,9 +330,9 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index dst1_index, Index dst2_index, PredFunc&& filter, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -286,11 +340,11 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -299,7 +353,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst1, dst2, src, mask.dataPtr(), + return filterCopyTransformParticles(pc1, pc2, dst1, dst2, src, mask.dataPtr(), dst1_index, dst2_index, std::forward(copy1), std::forward(copy2), diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 38dc1d6daf2..2a4c9fccad0 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -8,6 +8,7 @@ #ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" #include @@ -42,19 +43,20 @@ * * \return num_added the number of particles that were written to dst1 and dst2. */ -template ::value, int> foo = 0> -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, - const FAB *src_FAB, const Index* mask, - const Index dst1_index, const Index dst2_index, - CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, + DstTile& dst1, DstTile& dst2, const amrex::Box box, + const FAB *src_FAB, const Index* mask, + const Index dst1_index, const Index dst2_index, + CreateFunc1&& create1, CreateFunc2&& create2, + TransFunc&& transform) noexcept { using namespace amrex; const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } auto & warpx = WarpX::GetInstance(); const int level_0 = 0; @@ -83,10 +85,15 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B Gpu::DeviceVector offsets(ncells); auto total = amrex::Scan::ExclusiveSum(ncells, mask, offsets.data()); const Index num_added = N*total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto dst1_data = dst1.getParticleTileData(); const auto dst2_data = dst2.getParticleTileData(); @@ -130,6 +137,35 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -166,10 +202,10 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B * * \return num_added the number of particles that were written to dst1 and dst2. */ -template -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, const amrex::Box box, const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, @@ -184,11 +220,11 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B auto arrNumPartCreation = NumPartCreation.array(); const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } Gpu::DeviceVector mask(ncells); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); // for loop over all cells in the box. We apply the filter function to each cell // and store the result in arrNumPartCreation. If the result is strictly greater than @@ -201,7 +237,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B p_mask[mask_position] = (arrNumPartCreation(i,j,k) > 0); }); - return filterCreateTransformFromFAB(dst1, dst2, box, &NumPartCreation, + return filterCreateTransformFromFAB(pc1, pc2, dst1, dst2, box, &NumPartCreation, mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index da5637a7919..2c04baa18bb 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -52,16 +52,20 @@ struct SmartCopy dst.m_aos[i_dst] = src.m_aos[i_src]; // initialize the real components - for (int j = 0; j < DstData::NAR; ++j) + for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < dst.m_num_runtime_real; ++j) + } + for (int j = 0; j < dst.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][i_dst] = initializeRealValue(m_policy_real[j+DstData::NAR], engine); + } // initialize the int components - for (int j = 0; j < DstData::NAI; ++j) + for (int j = 0; j < DstData::NAI; ++j) { dst.m_idata[j][i_dst] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < dst.m_num_runtime_int; ++j) + } + for (int j = 0; j < dst.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][i_dst] = initializeIntValue(m_policy_int[j+DstData::NAI]); + } // copy the shared real components for (int j = 0; j < m_num_copy_real; ++j) diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 1be3bd79cf3..67d7767a5d3 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -63,16 +63,20 @@ struct SmartCreate prt.m_aos[i_prt].id() = id; // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) + for (int j = 0; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < prt.m_num_runtime_real; ++j) + } + for (int j = 0; j < prt.m_num_runtime_real; ++j) { prt.m_runtime_rdata[j][i_prt] = initializeRealValue(m_policy_real[j+PartData::NAR], engine); + } // initialize the int components - for (int j = 0; j < PartData::NAI; ++j) + for (int j = 0; j < PartData::NAI; ++j) { prt.m_idata[j][i_prt] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < prt.m_num_runtime_int; ++j) + } + for (int j = 0; j < prt.m_num_runtime_int; ++j) { prt.m_runtime_idata[j][i_prt] = initializeIntValue(m_policy_int[j+PartData::NAI]); + } } }; diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index c9d0eae3c4b..34afac53482 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_PhotonParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -69,7 +70,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX(WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -124,7 +126,8 @@ public: int const /*lev*/, int const /*depos_lev*/, amrex::Real const /*dt*/, - amrex::Real const /*relative_time*/) override {} + amrex::Real const /*relative_time*/, + PushType /*push_type*/) override {} }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 28dfcd8e280..aa9f04be224 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -185,7 +185,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, np_to_push, [=] AMREX_GPU_DEVICE (long i, auto exteb_control, auto qed_control) { - if (do_copy) copyAttribs(i); + if (do_copy) { copyAttribs(i); } ParticleReal x, y, z; GetPosition(i, x, y, z); @@ -205,17 +205,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc if constexpr (exteb_control == has_exteb) { getExternalEB(i, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } #ifdef WARPX_QED - [[maybe_unused]] auto& evolve_opt_tmp = evolve_opt; - [[maybe_unused]] auto p_optical_depth_BW_tmp = p_optical_depth_BW; - [[maybe_unused]] auto ux_tmp = ux; // for nvhpc - [[maybe_unused]] auto uy_tmp = uy; - [[maybe_unused]] auto uz_tmp = uz; + [[maybe_unused]] const auto& evolve_opt_tmp = evolve_opt; + [[maybe_unused]] auto *p_optical_depth_BW_tmp = p_optical_depth_BW; + [[maybe_unused]] auto *ux_tmp = ux; // for nvhpc + [[maybe_unused]] auto *uy_tmp = uy; + [[maybe_unused]] auto *uz_tmp = uz; [[maybe_unused]] auto dt_tmp = dt; if constexpr (qed_control == has_qed) { evolve_opt(ux[i], uy[i], uz[i], Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -240,7 +240,8 @@ PhotonParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // This does gather, push and deposit. // Push and deposit have been re-written for photons @@ -252,6 +253,6 @@ PhotonParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ef642fc718a..a12ae75f629 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -11,6 +11,7 @@ #define WARPX_PhysicalParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED @@ -105,6 +106,7 @@ public: * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) * \param skip_deposition Skip the charge and current deposition. + * \param push_type Type of particle push, explicit or implicit. Defaults to explicit * * Evolve iterates over particle iterator (each box) and performs filtering, * field gather, particle push and current deposition for all particles @@ -134,7 +136,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -150,6 +153,20 @@ public: amrex::Real dt, ScaleFields scaleFields, DtType a_dt_type=DtType::Full); + void ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full); + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, @@ -207,7 +224,7 @@ public: void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, - amrex::Real t_lab = 0.); + amrex::Real t_lab = 0.) const; void AddGaussianBeam ( PlasmaInjector const& plasma_injector, @@ -236,7 +253,7 @@ public: amrex::Gpu::HostVector& particle_uy, amrex::Gpu::HostVector& particle_uz, amrex::Gpu::HostVector& particle_w, - amrex::Real t_lab= 0.); + amrex::Real t_lab= 0.) const; /** * \brief Default initialize runtime attributes in a tile. This routine does not initialize the @@ -255,8 +272,7 @@ public: NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) final; + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather @@ -286,12 +302,12 @@ public: * \param Bx Field array before filtering (not modified) * \param By Field array before filtering (not modified) * \param Bz Field array before filtering (not modified) - * \param exfab pointer to the Ex field (modified) - * \param eyfab pointer to the Ey field (modified) - * \param ezfab pointer to the Ez field (modified) - * \param bxfab pointer to the Bx field (modified) - * \param byfab pointer to the By field (modified) - * \param bzfab pointer to the Bz field (modified) + * \param ex_ptr pointer to the Ex field (modified) + * \param ey_ptr pointer to the Ey field (modified) + * \param ez_ptr pointer to the Ez field (modified) + * \param bx_ptr pointer to the Bx field (modified) + * \param by_ptr pointer to the By field (modified) + * \param bz_ptr pointer to the Bz field (modified) * * The NCI Godfrey filter is applied on Ex, the result is stored in filtered_Ex * and the pointer exfab is modified (before this function is called, it points to Ex @@ -307,9 +323,9 @@ public: const amrex::FArrayBox& Ex, const amrex::FArrayBox& Ey, const amrex::FArrayBox& Ez, const amrex::FArrayBox& Bx, const amrex::FArrayBox& By, const amrex::FArrayBox& Bz, - amrex::FArrayBox const * & exfab, amrex::FArrayBox const * & eyfab, - amrex::FArrayBox const * & ezfab, amrex::FArrayBox const * & bxfab, - amrex::FArrayBox const * & byfab, amrex::FArrayBox const * & bzfab); + amrex::FArrayBox const * & ex_ptr, amrex::FArrayBox const * & ey_ptr, + amrex::FArrayBox const * & ez_ptr, amrex::FArrayBox const * & bx_ptr, + amrex::FArrayBox const * & by_ptr, amrex::FArrayBox const * & bz_ptr); /** * \brief This function determines if resampling should be done for the current species, and @@ -353,11 +369,35 @@ public: (std::shared_ptr ptr) override; //__________ + BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { + return m_shr_p_bw_engine.get(); + } + + QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const override { + return m_shr_p_qs_engine.get(); + } + PhotonEmissionFilterFunc getPhotonEmissionFilterFunc (); PairGenerationFilterFunc getPairGenerationFilterFunc (); #endif + std::vector getUserIntAttribs () const override { + return m_user_int_attribs; + } + + std::vector getUserRealAttribs () const override { + return m_user_real_attribs; + } + + amrex::Vector< amrex::Parser* > getUserIntAttribParser () const override { + return GetVecOfPtrs(m_user_int_attrib_parser); + } + + amrex::Vector< amrex::Parser* > getUserRealAttribParser () const override { + return GetVecOfPtrs(m_user_real_attrib_parser); + } + protected: std::string species_name; std::vector> plasma_injectors; @@ -408,9 +448,9 @@ protected: /* Vector of user-defined real attributes for species, species_name */ std::vector m_user_real_attribs; /* Vector of user-defined parser for initializing user-defined integer attributes */ - std::vector< std::unique_ptr > m_user_int_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_int_attrib_parser; /* Vector of user-defined parser for initializing user-defined real attributes */ - std::vector< std::unique_ptr > m_user_real_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_real_attrib_parser; }; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 182577a81cf..0d9180c090e 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -21,6 +21,7 @@ #endif #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "Particles/Pusher/CopyParticleAttribs.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/PushSelector.H" @@ -343,7 +344,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_species_name.query("do_field_ionization", do_field_ionization); pp_species_name.query("do_resampling", do_resampling); - if (do_resampling) m_resampler = Resampling(species_name); + if (do_resampling) { m_resampler = Resampling(species_name); } //check if Radiation Reaction is enabled and do consistency checks pp_species_name.query("do_classical_radiation_reaction", do_classical_radiation_reaction); @@ -365,12 +366,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp #ifdef WARPX_QED pp_species_name.query("do_qed_quantum_sync", m_do_qed_quantum_sync); - if (m_do_qed_quantum_sync) + if (m_do_qed_quantum_sync) { AddRealComp("opticalDepthQSR"); + } pp_species_name.query("do_qed_breit_wheeler", m_do_qed_breit_wheeler); - if (m_do_qed_breit_wheeler) + if (m_do_qed_breit_wheeler) { AddRealComp("opticalDepthBW"); + } if(m_do_qed_quantum_sync){ pp_species_name.get("qed_quantum_sync_phot_product_species", @@ -468,7 +471,7 @@ void PhysicalParticleContainer::InitData () } void PhysicalParticleContainer::MapParticletoBoostedFrame ( - ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) + ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) const { // Map the particles from the lab frame to the boosted frame. // This boosts the particle to the lab frame and calculates @@ -777,110 +780,21 @@ PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, const int n_external_attr_real, - const int n_external_attr_int, - const amrex::RandomEngine& engine) + const int n_external_attr_int) { - using namespace amrex::literals; - - const int np = pinned_tile.numParticles(); - - // Preparing data needed for user defined attributes - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto get_position = GetParticlePosition(pinned_tile); - const auto soa = pinned_tile.getParticleTileData(); - const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; - const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; - const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; - constexpr int lev = 0; - const amrex::Real t = WarpX::GetInstance().gett_new(lev); - -#ifndef WARPX_QED - amrex::ignore_unused(engine); -#endif - - // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes - for (int j = PIdx::nattribs + n_external_attr_real; j < NumRealComps() ; ++j) - { - amrex::Vector attr_temp(np, 0.0_prt); + ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, + n_external_attr_real, n_external_attr_int, + m_user_real_attribs, m_user_int_attribs, + particle_comps, particle_icomps, + amrex::GetVecOfPtrs(m_user_real_attrib_parser), + amrex::GetVecOfPtrs(m_user_int_attrib_parser), #ifdef WARPX_QED - // Current runtime comp is quantum synchrotron optical depth - if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && - particle_comps["opticalDepthQSR"] == j) - { - const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = quantum_sync_get_opt(engine); - } - } - - // Current runtime comp is Breit-Wheeler optical depth - if (particle_comps.find("opticalDepthBW") != particle_comps.end() && - particle_comps["opticalDepthBW"] == j) - { - const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = breit_wheeler_get_opt(engine); - } - } + true, + m_shr_p_bw_engine.get(), + m_shr_p_qs_engine.get(), #endif - - for (int ia = 0; ia < n_user_real_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_comps.find(m_user_real_attribs[ia]) != particle_comps.end() && - particle_comps[m_user_real_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_real_attrib_parserexec = - m_user_real_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = user_real_attrib_parserexec(xp, yp, zp, - ux[i], uy[i], uz[i], t); - } - } - } - - pinned_tile.push_back_real(j, attr_temp.data(), attr_temp.data() + np); - } - - // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes - for (int j = n_external_attr_int; j < NumIntComps() ; ++j) - { - amrex::Vector attr_temp(np, 0); - - // Current runtime comp is ionization level - if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && - particle_icomps["ionizationLevel"] == j) - { - for (int i = 0; i < np; ++i) { - attr_temp[i] = ionization_initial_level; - } - } - - for (int ia = 0; ia < n_user_int_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_icomps.find(m_user_int_attribs[ia]) != particle_icomps.end() && - particle_icomps[m_user_int_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_int_attrib_parserexec = - m_user_int_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = static_cast( - user_int_attrib_parserexec(xp, yp, zp, ux[i], uy[i], uz[i], t)); - } - } - } - - pinned_tile.push_back_int(j, attr_temp.data(), attr_temp.data() + np); - } - + ionization_initial_level, + 0,pinned_tile.numParticles()); } @@ -896,7 +810,7 @@ PhysicalParticleContainer::CheckAndAddParticle ( Gpu::HostVector& particle_uy, Gpu::HostVector& particle_uz, Gpu::HostVector& particle_w, - Real t_lab) + Real t_lab) const { if (WarpX::gamma_boost > 1.) { MapParticletoBoostedFrame(x, y, z, ux, uy, uz, t_lab); @@ -999,7 +913,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // If no part_realbox is provided, initialize particles in the whole domain const Geometry& geom = Geom(lev); - if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); + if (!part_realbox.ok()) { part_realbox = geom.ProbDomain(); } const int num_ppc = plasma_injector.num_particles_per_cell; #ifdef WARPX_DIM_RZ @@ -1107,7 +1021,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1135,12 +1049,15 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const auto zlim = GpuArray{lo.z,(lo.z+hi.z)/2._rt,hi.z}; const auto checker = [&](){ - for (const auto& x : xlim) - for (const auto& y : ylim) - for (const auto& z : zlim) + for (const auto& x : xlim) { + for (const auto& y : ylim) { + for (const auto& z : zlim) { if (inj_pos->insideBounds(x,y,z) and (inj_rho->getDensity(x,y,z) > 0) ) { return 1; } + } + } + } return 0; }; const int flag_pcount = checker(); @@ -1248,12 +1165,14 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1275,7 +1194,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); #ifdef WARPX_DIM_RZ const bool rz_random_theta = m_rz_random_theta; #endif @@ -1286,7 +1205,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const auto index = overlap_box.index(iv); #ifdef WARPX_DIM_RZ Real theta_offset = 0._rt; - if (rz_random_theta) theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; + if (rz_random_theta) { theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; } #endif Real scale_fac = 0.0_rt; @@ -1538,15 +1457,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, scale_fac = dx[0]*dx[1]/num_ppc_real; // When emission is in the r direction, the emitting surface is a cylinder. // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 0) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 0) { scale_fac /= dx[0]; } // When emission is in the z direction, the emitting surface is an annulus // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[1]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[1]; } // When emission is in the theta direction (flux_normal_axis == 1), // the emitting surface is a rectangle, within the plane of the simulation #elif defined(WARPX_DIM_1D_Z) scale_fac = dx[0]/num_ppc_real; - if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[0]; } #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1679,7 +1598,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1798,12 +1717,14 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1828,7 +1749,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { @@ -2038,7 +1959,8 @@ PhysicalParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition) + Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); @@ -2061,8 +1983,9 @@ PhysicalParticleContainer::Evolve (int lev, const auto t_lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) + for (int i = 0; i < TmpIdx::nattribs; ++i) { tmp_particle_data[t_lev][index][i].resize(np); + } } } @@ -2166,10 +2089,17 @@ PhysicalParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_START(blp_fg); const auto np_to_push = np_gather; const auto gather_lev = lev; - PushPX(pti, exfab, eyfab, ezfab, - bxfab, byfab, bzfab, - Ex.nGrowVect(), e_is_nodal, - 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } if (np_gather < np) { @@ -2200,11 +2130,19 @@ PhysicalParticleContainer::Evolve (int lev, // Field gather and push for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - PushPX(pti, cexfab, ceyfab, cezfab, - cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, - nfine_gather, np-nfine_gather, - lev, lev-1, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } } WARPX_PROFILE_VAR_STOP(blp_fg); @@ -2212,8 +2150,8 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition if (!skip_deposition) { - // Deposit at t_{n+1/2} - const amrex::Real relative_time = -0.5_rt * dt; + // Deposit at t_{n+1/2} with explicit push + const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; @@ -2221,14 +2159,14 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } // end of "if electrostatic_solver_id == ElectrostaticSolverAlgo::None" } // end of "if do_not_push" @@ -2547,7 +2485,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("PhysicalParticleContainer::PushP()"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -2653,7 +2591,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } // Externally applied E and B-field in Cartesian co-ordinates - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2743,7 +2681,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, (gather_lev==(lev )), "Gather buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_push == 0) return; + if (np_to_push == 0) { return; } // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); @@ -2842,7 +2780,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #ifdef WARPX_QED const auto do_sync = m_do_qed_quantum_sync; amrex::Real t_chi_max = 0.0; - if (do_sync) t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } QuantumSynchrotronEvolveOpticalDepth evolve_opt; amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; @@ -2903,7 +2841,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2914,34 +2852,48 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (!do_sync) #endif { - doParticlePush<0>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, #ifdef WARPX_QED - t_chi_max, + t_chi_max, #endif - dt); + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } #ifdef WARPX_QED else { if constexpr (qed_control == has_qed) { - doParticlePush<1>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, - t_chi_max, - dt); + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } } #endif #ifdef WARPX_QED [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto foo_podq = p_optical_depth_QSR; - [[maybe_unused]] auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc if constexpr (qed_control == has_qed) { if (local_has_quantum_sync) { evolve_opt(ux[ip], uy[ip], uz[ip], @@ -2955,10 +2907,265 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, }); } +/* \brief Perform the implicit particle push operation in one fused kernel + * The main difference from PushPX is the order of operations: + * - push position by 1/2 dt + * - gather fields + * - push velocity by dt + * - average old and new velocity to get time centered value + * The routines ends with both position and velocity at the half time level. + */ +void +PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || + (gather_lev==(lev )), + "Gather buffers only work for lev-1"); + // If no particles, do not do anything + if (np_to_push == 0) { return; } + + // Get cell size on gather_lev + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. + Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Add guard cells to the box. + box.grow(ngEB); + + auto setPosition = SetParticlePosition(pti, offset); + + const auto getExternalEB = GetExternalEBField(pti, offset); + + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + + // Lower corner of tile box physical domain (take into account Galilean shift) + const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + + const Dim3 lo = lbound(box); + + bool galerkin_interpolation = WarpX::galerkin_interpolation; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr() + offset; + +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); + CopyParticleAttribs copyAttribs; + if (do_copy) { + copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + } + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + } + + // Loop over the particles and update their momentum + const amrex::ParticleReal q = this->charge; + const amrex::ParticleReal m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; +#ifdef WARPX_QED + const auto do_sync = m_do_qed_quantum_sync; + amrex::Real t_chi_max = 0.0; + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } + + QuantumSynchrotronEvolveOpticalDepth evolve_opt; + amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; + const bool local_has_quantum_sync = has_quantum_sync(); + if (local_has_quantum_sync) { + evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); + p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + } +#endif + + const auto t_do_not_gather = do_not_gather; + + enum exteb_flags : int { no_exteb, has_exteb }; + enum qed_flags : int { no_qed, has_qed }; + + int exteb_runtime_flag = getExternalEB.isNoOp() ? no_exteb : has_exteb; +#ifdef WARPX_QED + int qed_runtime_flag = (local_has_quantum_sync || do_sync) ? has_qed : no_qed; +#else + int qed_runtime_flag = no_qed; +#endif + + // Using this version of ParallelFor with compile time options + // improves performance when qed or external EB are not used by reducing + // register pressure. + amrex::ParallelFor(TypeList, + CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, + auto qed_control) + { + // Position advance starts from the position at the start of the step + // but uses the most recent velocity. +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal xp = x_n[ip]; + amrex::ParticleReal xp_n = x_n[ip]; +#else + amrex::ParticleReal xp = 0._rt; + amrex::ParticleReal xp_n = 0._rt; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal yp = y_n[ip]; + amrex::ParticleReal yp_n = y_n[ip]; +#else + amrex::ParticleReal yp = 0._rt; + amrex::ParticleReal yp_n = 0._rt; +#endif + amrex::ParticleReal zp = z_n[ip]; + amrex::ParticleReal zp_n = z_n[ip]; + + UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); + setPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; + + if(!t_do_not_gather){ + // first gather E and B to the particle positions + doGatherShapeNImplicit(xp_n, yp_n, zp_n, xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, galerkin_interpolation); + } + + // Externally applied E and B-field in Cartesian co-ordinates + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; + if constexpr (exteb_control == has_exteb) { + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + } + + scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + // The momentum push starts with the velocity at the start of the step + ux[ip] = ux_n[ip]; + uy[ip] = uy_n[ip]; + uz[ip] = uz_n[ip]; + +#ifdef WARPX_QED + if (!do_sync) +#endif + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, +#ifdef WARPX_QED + t_chi_max, +#endif + dt); + } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } +#endif + +#ifdef WARPX_QED + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } + } +#else + amrex::ignore_unused(qed_control); +#endif + + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + }); +} + void PhysicalParticleContainer::InitIonizationModule () { - if (!do_field_ionization) return; + if (!do_field_ionization) { return; } const ParmParse pp_species_name(species_name); if (charge != PhysConst::q_e){ ablastr::warn_manager::WMRecordWarning("Species", @@ -2967,10 +3174,15 @@ PhysicalParticleContainer::InitIonizationModule () "overriding user value and setting charge = q_e."); charge = PhysConst::q_e; } + utils::parser::queryWithParser(pp_species_name, "do_adk_correction", do_adk_correction); + utils::parser::queryWithParser( pp_species_name, "ionization_initial_level", ionization_initial_level); pp_species_name.get("ionization_product_species", ionization_product_name); pp_species_name.get("physical_element", physical_element); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + physical_element == "H" || !do_adk_correction, + "Correction to ADK by Zhang et al., PRA 90, 043410 (2014) only works with Hydrogen"); // Add runtime integer component for ionization level AddIntComp("ionizationLevel"); // Get atomic number and ionization energies from file @@ -3005,6 +3217,18 @@ PhysicalParticleContainer::InitIonizationModule () h_ionization_energies.begin(), h_ionization_energies.end(), ionization_energies.begin()); + adk_correction_factors.resize(4); + if (do_adk_correction) { + Vector h_correction_factors(4); + constexpr int offset_corr = 0; // hard-coded: only Hydrogen + for(int i=0; i<4; i++){ + h_correction_factors[i] = table_correction_factors[i+offset_corr]; + } + Gpu::copyAsync(Gpu::hostToDevice, + h_correction_factors.begin(), h_correction_factors.end(), + adk_correction_factors.begin()); + } + Real const* AMREX_RESTRICT p_ionization_energies = ionization_energies.data(); Real * AMREX_RESTRICT p_adk_power = adk_power.data(); Real * AMREX_RESTRICT p_adk_prefactor = adk_prefactor.data(); @@ -3042,8 +3266,10 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, adk_prefactor.dataPtr(), adk_exp_prefactor.dataPtr(), adk_power.dataPtr(), + adk_correction_factors.dataPtr(), particle_icomps["ionizationLevel"], - ion_atomic_number}; + ion_atomic_number, + do_adk_correction}; } PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector (int i) diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 1f0cee67421..4f961fe94aa 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -49,9 +49,9 @@ struct CopyParticleAttribs CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, long a_offset = 0) noexcept { - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); uxp = attribs[PIdx::ux].dataPtr() + a_offset; uyp = attribs[PIdx::uy].dataPtr() + a_offset; diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index 637d2ac74db..2a6ac8a7d6c 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -12,7 +12,6 @@ #include "Particles/Pusher/UpdateMomentumBorisWithRadiationReaction.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Particles/Pusher/UpdateMomentumVay.H" -#include "Particles/Pusher/UpdatePosition.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -21,13 +20,9 @@ #include /** - * \brief Push position and momentum for a single particle + * \brief Push momentum for a single particle * * \tparam do_sync Whether to include quantum synchrotron radiation (QSR) - * \param GetPosition A functor for returning the particle position. - * \param SetPosition A functor for setting the particle position. - * \param copyAttribs A functor for storing the old u and x - * \param i The index of the particle to work on * \param ux, uy, uz Particle momentum * \param Ex, Ey, Ez Electric field on particles. * \param Bx, By, Bz Magnetic field on particles. @@ -36,41 +31,34 @@ * \param a_q Charge of this species. * \param pusher_algo 0: Boris, 1: Vay, 2: HigueraCary * \param do_crr Whether to do the classical radiation reaction - * \param do_copy Whether to copy the old x and u for the BTD * \param t_chi_max Cutoff chi for QSR * \param dt Time step size */ template AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void doParticlePush(const GetParticlePosition& GetPosition, - const SetParticlePosition& SetPosition, - const CopyParticleAttribs& copyAttribs, - const long i, - amrex::ParticleReal& ux, - amrex::ParticleReal& uy, - amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, - const amrex::ParticleReal Ey, - const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, - const amrex::ParticleReal By, - const amrex::ParticleReal Bz, - const int ion_lev, - const amrex::ParticleReal m, - const amrex::ParticleReal a_q, - const int pusher_algo, - const int do_crr, - const int do_copy, +void doParticleMomentumPush(amrex::ParticleReal& ux, + amrex::ParticleReal& uy, + amrex::ParticleReal& uz, + const amrex::ParticleReal Ex, + const amrex::ParticleReal Ey, + const amrex::ParticleReal Ez, + const amrex::ParticleReal Bx, + const amrex::ParticleReal By, + const amrex::ParticleReal Bz, + const int ion_lev, + const amrex::ParticleReal m, + const amrex::ParticleReal a_q, + const int pusher_algo, + const int do_crr, #ifdef WARPX_QED - const amrex::Real t_chi_max, + const amrex::Real t_chi_max, #endif - const amrex::Real dt) + const amrex::Real dt) { amrex::ParticleReal qp = a_q; if (ion_lev) { qp *= ion_lev; } - if (do_copy) copyAttribs(i); if (do_crr) { #ifdef WARPX_QED amrex::ignore_unused(t_chi_max); @@ -88,10 +76,6 @@ void doParticlePush(const GetParticlePosition& GetPosition, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); } - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else #endif { @@ -99,35 +83,19 @@ void doParticlePush(const GetParticlePosition& GetPosition, UpdateMomentumBorisWithRadiationReaction(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } } else if (pusher_algo == ParticlePusherAlgo::Boris) { UpdateMomentumBoris( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::Vay) { UpdateMomentumVay( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { UpdateMomentumHigueraCary( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } //else { // amrex::Abort("Unknown particle pusher"); // } diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 4bce5c10211..4743c79c00f 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -16,7 +16,10 @@ /** \brief Push the particle's positions over one timestep, - * given the value of its momenta `ux`, `uy`, `uz` */ + * given the value of its momenta `ux`, `uy`, `uz`. + * This uses the standard leapfrog algorithm + * x^{n+1} - x^{n} = dt*u^{n+1/2}/gamma^{n+1/2} + */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, @@ -42,4 +45,43 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti z += uz * inv_gamma * dt; } +/** \brief Push the particle's positions over one timestep, + * given the value of its momenta `ux`, `uy`, `uz`. + * The implicit version is the Crank-Nicolson scheme, + * x^{n+1} - x^{n} = dt*(u^{n+1} + u^{n})/(gamma^{n+1} + gamma^{n}) + * See Eqs. 15 and 17 in Chen, JCP 407 (2020) 109228. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, + const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, + const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, + const amrex::Real dt ) +{ + using namespace amrex::literals; + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The ux,uy,uz are the velocities at time level n+1/2 + const amrex::ParticleReal ux_np1 = 2._prt*ux - ux_n; + const amrex::ParticleReal uy_np1 = 2._prt*uy - uy_n; + const amrex::ParticleReal uz_np1 = 2._prt*uz - uz_n; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (ux_n*ux_n + uy_n*uy_n + uz_n*uz_n)*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (ux_np1*ux_np1 + uy_np1*uy_np1 + uz_np1*uz_np1)*inv_c2); + const amrex::ParticleReal inv_gamma = 2.0_prt/(gamma_n + gamma_np1); + + // Update positions over one time step +#if (AMREX_SPACEDIM >= 2) + x += ux * inv_gamma * dt; +#else + amrex::ignore_unused(x); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D + y += uy * inv_gamma * dt; +#else + amrex::ignore_unused(y); +#endif + z += uz * inv_gamma * dt; +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index e3910286c91..680e33ebe6a 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -71,8 +71,8 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); const auto n_cells = static_cast(bins.numBins()); - const auto indices = bins.permutationPtr(); - const auto cell_offsets = bins.offsetsPtr(); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); const amrex::Real target_ratio = m_target_ratio; const int min_ppc = m_min_ppc; @@ -89,8 +89,9 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // do nothing for cells with less particles than min_ppc // (this intentionally includes skipping empty cells, too) - if (cell_numparts < min_ppc) + if (cell_numparts < min_ppc) { return; + } amrex::Real average_weight = 0._rt; // First loop over cell particles to compute average particle weight in the cell diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index 0d5bf0d0f0e..bc20420ea6e 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_RigidInjectedParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -84,7 +85,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index d0e19af155a..d50851ffb1b 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -117,9 +117,9 @@ RigidInjectedParticleContainer::RemapParticles() for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const auto& attribs = pti.GetAttribs(); - const auto uxp = attribs[PIdx::ux].dataPtr(); - const auto uyp = attribs[PIdx::uy].dataPtr(); - const auto uzp = attribs[PIdx::uz].dataPtr(); + const auto *const uxp = attribs[PIdx::ux].dataPtr(); + const auto *const uyp = attribs[PIdx::uy].dataPtr(); + const auto *const uzp = attribs[PIdx::uz].dataPtr(); const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -299,7 +299,8 @@ RigidInjectedParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // Update location of injection plane in the boosted frame @@ -324,7 +325,7 @@ RigidInjectedParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } void @@ -334,7 +335,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("RigidInjectedParticleContainer::PushP"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -447,7 +448,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 60e54b5d59b..a0b4ed63a30 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -89,8 +89,14 @@ struct Compute_shifted_shape_factor const T x_old, const int i_new) const { - if constexpr (depos_order == 1){ - const auto i = static_cast(x_old); + if constexpr (depos_order == 0){ + const auto i = static_cast(std::floor(x_old + T(0.5))); + const int i_shift = i - i_new; + sx[1+i_shift] = T(1.0); + return i; + } + else if constexpr (depos_order == 1){ + const auto i = static_cast(std::floor(x_old)); const int i_shift = i - i_new; const T xint = x_old - T(i); sx[1+i_shift] = T(1.0) - xint; diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 18bebed185f..58511cfd5e7 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -76,7 +76,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - Find the indices that reorder particles so that the last particles // are in the larger buffer fillWithConsecutiveIntegers( pid ); - auto const sep = stablePartition( pid.begin(), pid.end(), inexflag ); + auto *const sep = stablePartition( pid.begin(), pid.end(), inexflag ); // At the end of this step, `pid` contains the indices that should be used to // reorder the particles, and `sep` is the position in the array that // separates the particles that deposit/gather on the fine patch (first part) @@ -110,7 +110,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); - auto const sep2 = stablePartition( sep, pid.end(), inexflag ); + auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { nfine_gather = iteratorDistance(pid.begin(), sep2); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 024cd8d9157..33aa71d1c7d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -13,6 +13,7 @@ #include "WarpXParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ParticleBoundaries.H" #include "SpeciesPhysicalProperties.H" @@ -151,7 +152,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false) = 0; + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit) = 0; virtual void PostRestart () = 0; @@ -166,8 +168,7 @@ public: NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) = 0; + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. @@ -250,7 +251,8 @@ public: int lev, int depos_lev, amrex::Real dt, - amrex::Real relative_time); + amrex::Real relative_time, + PushType push_type); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and @@ -404,6 +406,19 @@ public: */ void defineAllParticleTiles () noexcept; + virtual std::vector getUserIntAttribs () const { return std::vector{}; } + + virtual std::vector getUserRealAttribs () const { return std::vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserIntAttribParser () const { return amrex::Vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserRealAttribParser () const { return amrex::Vector{}; } + +#ifdef WARPX_QED + virtual BreitWheelerEngine* get_breit_wheeler_engine_ptr () const {return nullptr;} + virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} +#endif + protected: int species_id; @@ -430,6 +445,7 @@ protected: int do_continuous_injection = 0; int do_field_ionization = 0; + int do_adk_correction = 0; int ionization_product; std::string ionization_product_name; int ion_atomic_number; @@ -438,6 +454,8 @@ protected: amrex::Gpu::DeviceVector adk_power; amrex::Gpu::DeviceVector adk_prefactor; amrex::Gpu::DeviceVector adk_exp_prefactor; + /** for correction in Zhang et al., PRA 90, 043410 (2014). a1, a2, a3, Ecrit. */ + amrex::Gpu::DeviceVector adk_correction_factors; std::string physical_element; int do_resampling = 0; @@ -473,6 +491,9 @@ public: using TmpParticles = amrex::Vector >; TmpParticles getTmpParticleData () const noexcept {return tmp_particle_data;} + + int getIonizationInitialLevel () const noexcept {return ionization_initial_level;} + protected: TmpParticles tmp_particle_data; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index d44bc8ad9c3..f1b19dd8407 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -279,9 +279,9 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, pinned_tile.push_back_int(j, attr_int[j].data() + ibegin, attr_int[j].data() + iend); } + pinned_tile.resize(np); // Default initialize the other real and integer runtime attributes - DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int, - amrex::RandomEngine{}); + DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int); auto old_np = particle_tile.numParticles(); auto new_np = old_np + pinned_tile.numParticles(); @@ -323,17 +323,17 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, amrex::MultiFab * const jx, amrex::MultiFab * const jy, amrex::MultiFab * const jz, long const offset, long const np_to_deposit, int const thread_num, const int lev, int const depos_lev, - amrex::Real const dt, amrex::Real const relative_time) + amrex::Real const dt, amrex::Real const relative_time, PushType push_type) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // If user decides not to deposit - if (do_not_deposit) return; + if (do_not_deposit) { return; } // Number of guard cells for local deposition of J const WarpX& warpx = WarpX::GetInstance(); @@ -470,7 +470,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, { auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); const int ntiles = numTilesInBox(box, true, bin_size); @@ -502,6 +502,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_STOP(blp_get_max_tilesize); // Now pick current deposition algorithm + if (push_type == PushType::Implicit) { + amrex::Abort("Cannot do shared memory deposition with implicit algorithm"); + } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Esirkepov algorithm"); } @@ -538,29 +541,80 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // If not doing shared memory deposition, call normal kernels else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doChargeConservingDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doChargeConservingDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doChargeConservingDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { + if (push_type == PushType::Implicit) { + WARPX_ABORT_WITH_MESSAGE("The Vay algorithm cannot be used with implicit algorithm."); + } if (WarpX::nox == 1){ doVayDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, @@ -584,27 +638,61 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WarpX::load_balance_costs_update_algo); } } else { // Direct deposition - if (WarpX::nox == 1){ - doDepositionShapeN<1>( + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doDepositionShapeN<2>( + } else if (WarpX::nox == 2){ + doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doDepositionShapeN<3>( + } else if (WarpX::nox == 3){ + doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doDepositionShapeNImplicit<1>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeNImplicit<2>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeNImplicit<3>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } } @@ -653,7 +741,7 @@ WarpXParticleContainer::DepositCurrent ( DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), - 0, np, thread_num, lev, lev, dt, relative_time); + 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP } @@ -693,7 +781,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -795,7 +883,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); @@ -824,15 +912,15 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; - const auto offsets_ptr = bins.offsetsPtr(); - auto tbox_ptr = tboxes.dataPtr(); - auto permutation = bins.permutationPtr(); + auto *const offsets_ptr = bins.offsetsPtr(); + auto *tbox_ptr = tboxes.dataPtr(); + auto *permutation = bins.permutationPtr(); amrex::ParallelFor(bins.numBins(), [=] AMREX_GPU_DEVICE (int ibin) { const auto bin_start = offsets_ptr[ibin]; @@ -857,7 +945,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - const auto boxes_ptr = tboxes.dataPtr(); + auto *const boxes_ptr = tboxes.dataPtr(); reduce_op.eval(tboxes.size(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { @@ -1014,7 +1102,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, { // Reset the rho array if reset is True int const nc = WarpX::ncomps; - if (reset) rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); + if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } // Loop over particle tiles and deposit charge on each level #ifdef AMREX_USE_OMP @@ -1081,8 +1169,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #else const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -1107,7 +1196,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { for (int lev = 0; lev <= nLevels; ++lev) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const auto wp = pti.GetAttribs(PIdx::w).data(); + auto *const wp = pti.GetAttribs(PIdx::w).data(); reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int ip) @@ -1117,7 +1206,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { total_charge = get<0>(reduce_data.value()); - if (!local) ParallelDescriptor::ReduceRealSum(total_charge); + if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } total_charge *= this->charge; return total_charge; } @@ -1230,7 +1319,7 @@ amrex::ParticleReal WarpXParticleContainer::maxParticleVelocity(bool local) { } } - if (!local) ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); + if (!local) { ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); } return max_v; } @@ -1248,7 +1337,7 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) { WARPX_PROFILE("WarpXParticleContainer::PushX()"); - if (do_not_push) return; + if (do_not_push) { return; } amrex::LayoutData* costs = WarpX::getCosts(lev); @@ -1323,7 +1412,7 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, const ParticleLocData& pld, const int lev) { - if (not do_splitting) return; + if (not do_splitting) { return; } // Tag particle if goes to higher level. // It will be split later in the loop @@ -1345,7 +1434,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ WARPX_PROFILE("WarpXParticleContainer::ApplyBoundaryConditions()"); // Periodic boundaries are handled in AMReX code - if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) return; + if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) { return; } auto boundary_conditions = m_boundary_conditions.data; @@ -1384,7 +1473,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ ParticleType& p = pp[i]; // skip particles that are already flagged for removal - if (p.id() < 0) return; + if (p.id() < 0) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 2ceb17be116..e42b8268fe1 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -86,7 +86,15 @@ void init_WarpX (py::module& m) "Evolve the simulation the specified number of steps" ) - // from AmrCore->AmrMesh + // from amrex::AmrCore / amrex::AmrMesh + .def_property_readonly("max_level", + [](WarpX const & wx){ return wx.maxLevel(); }, + "The maximum mesh-refinement level for the simulation." + ) + .def_property_readonly("finest_level", + [](WarpX const & wx){ return wx.finestLevel(); }, + "The currently finest level of mesh-refinement used. This is always less or equal to max_level." + ) .def("Geom", //[](WarpX const & wx, int const lev) { return wx.Geom(lev); }, py::overload_cast< int >(&WarpX::Geom, py::const_), @@ -112,7 +120,15 @@ void init_WarpX (py::module& m) }, py::arg("multifab_name"), py::return_value_policy::reference_internal, - "Return MultiFabs by name, e.g., 'Efield_aux[x][l=0]', 'Efield_cp[x][l=0]', ..." + R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" ) .def("multi_particle_container", [](WarpX& wx){ return &wx.GetPartContainer(); }, @@ -140,22 +156,26 @@ void init_WarpX (py::module& m) // Expose functions to get the current simulation step and time .def("getistep", [](WarpX const & wx, int lev){ return wx.getistep(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current step on mesh-refinement level ``lev``." ) .def("gett_new", [](WarpX const & wx, int lev){ return wx.gett_new(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time on mesh-refinement level ``lev``." ) .def("getdt", [](WarpX const & wx, int lev){ return wx.getdt(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time step size on mesh-refinement level ``lev``." ) .def("set_potential_on_eb", [](WarpX& wx, std::string potential) { wx.m_poisson_boundary_handler.setPotentialEB(potential); }, - py::arg("potential") + py::arg("potential"), + "Sets the EB potential string and updates the function parser." ) ; diff --git a/Source/Utils/Parser/IntervalsParser.cpp b/Source/Utils/Parser/IntervalsParser.cpp index 6af4ce6a8c3..6564003abdf 100644 --- a/Source/Utils/Parser/IntervalsParser.cpp +++ b/Source/Utils/Parser/IntervalsParser.cpp @@ -92,7 +92,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto insplit = ablastr::utils::text::split_string>( inconcatenated, m_separator); @@ -102,7 +102,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const SliceParser temp_slice(inslc); m_slices.push_back(temp_slice); if ((temp_slice.getPeriod() > 0) && - (temp_slice.getStop() >= temp_slice.getStart())) m_activated = true; + (temp_slice.getStop() >= temp_slice.getStart())) { m_activated = true; } } } @@ -155,7 +155,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto const insplit = ablastr::utils::text::split_string>( inconcatenated, std::string(1,m_separator)); diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index 6d430d4a619..c2da9577947 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -135,7 +135,7 @@ amrex::Parser utils::parser::makeParser ( parser.registerVariables(varnames); std::set symbols = parser.symbols(); - for (auto const& v : varnames) symbols.erase(v); + for (auto const& v : varnames) { symbols.erase(v); } // User can provide inputs under this name, through which expressions // can be provided for arbitrary variables. PICMI inputs are aware of diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index ecd6b7c2739..b04176d4d83 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -181,8 +181,8 @@ namespace ParticleUtils { */ AMREX_GPU_HOST_DEVICE AMREX_INLINE bool containsInclusive (amrex::RealBox const& tilebox, amrex::XDim3 const point) { - const auto xlo = tilebox.lo(); - const auto xhi = tilebox.hi(); + const auto *const xlo = tilebox.lo(); + const auto *const xhi = tilebox.hi(); return AMREX_D_TERM((xlo[0] <= point.x) && (point.x <= xhi[0]), && (xlo[1] <= point.y) && (point.y <= xhi[1]), && (xlo[2] <= point.z) && (point.z <= xhi[2])); diff --git a/Source/Utils/Physics/IonizationEnergiesTable.H b/Source/Utils/Physics/IonizationEnergiesTable.H index f956e308dbe..b7c82e88247 100644 --- a/Source/Utils/Physics/IonizationEnergiesTable.H +++ b/Source/Utils/Physics/IonizationEnergiesTable.H @@ -1963,4 +1963,11 @@ namespace utils::physics } +constexpr int correction_factors_length = 4; +// a1, a2, a3, Ecrit from Zhang et al., PRA 90, 043410 (2014). +constexpr amrex::Real table_correction_factors[correction_factors_length]{ + // H + amrex::Real(0.11714), amrex::Real(-0.90933), amrex::Real(-0.06034), amrex::Real(32125000000.) +}; + #endif // #ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_ diff --git a/Source/Utils/RelativeCellPosition.cpp b/Source/Utils/RelativeCellPosition.cpp index 1a108b54b56..d30da1841cb 100644 --- a/Source/Utils/RelativeCellPosition.cpp +++ b/Source/Utils/RelativeCellPosition.cpp @@ -22,8 +22,9 @@ utils::getRelativeCellPosition(amrex::MultiFab const& mf) // WarpX::grid_type==GridType::Collocated means: all indices/directions on CellIndex::NODE for (int d = 0; d < AMREX_SPACEDIM; d++) { - if (idx_type.cellCentered(d)) + if (idx_type.cellCentered(d)) { relative_position.at(d) = 0.5; + } } return relative_position; diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 7ad33409900..87db0ae9b9b 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -23,6 +23,17 @@ struct MediumForEM { }; }; +/** + * \brief struct to select the overall evolve scheme + */ +struct EvolveScheme { + enum { + Explicit = 0, + ImplicitPicard = 1, + SemiImplicitPicard = 2 + }; +}; + /** * \brief struct to select algorithm for macroscopic Maxwell solver LaxWendroff (semi-implicit) represents sigma*E = sigma*0.5*(E^(n) + E^(n+1)) diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b361aab14e9..75c488134e0 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -23,6 +23,13 @@ // Define dictionary with correspondence between user-input strings, // and corresponding integer for use inside the code +const std::map evolve_scheme_to_int = { + {"explicit", EvolveScheme::Explicit }, + {"implicit_picard", EvolveScheme::ImplicitPicard }, + {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, + {"default", EvolveScheme::Explicit } +}; + const std::map grid_to_int = { {"collocated", GridType::Collocated}, {"staggered", GridType::Staggered}, @@ -147,7 +154,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ // Pick the right dictionary std::map algo_to_int; - if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { + if (0 == std::strcmp(pp_search_key, "evolve_scheme")) { + algo_to_int = evolve_scheme_to_int; + } else if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { algo_to_int = electromagnetic_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "grid_type")) { algo_to_int = grid_to_int; @@ -159,8 +168,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = current_deposition_algo_to_int; if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || - WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) + WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) { algo_to_int["default"] = CurrentDepositionAlgo::Direct; + } } else if (0 == std::strcmp(pp_search_key, "charge_deposition")) { algo_to_int = charge_deposition_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "field_gathering")) { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 6f2a97f5550..fdcb7cb8da4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -144,7 +144,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (step == end_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Stopping moving window"); } - if (!moving_window_active(step)) return 0; + if (!moving_window_active(step)) { return 0; } // Update the continuous position of the moving window, // and of the plasma injection @@ -165,7 +165,7 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real* cdx = geom[0].CellSize(); const int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); - if (num_shift_base == 0) return 0; + if (num_shift_base == 0) { return 0; } // update the problem domain. Note the we only do this on the base level because // amrex::Geometry objects share the same, static RealBox. @@ -222,16 +222,16 @@ WarpX::MoveWindow (const int step, bool move_j) if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { use_Bparser = true; - if (dim == 0) Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); - if (dim == 1) Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); - if (dim == 2) Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); + if (dim == 0) { Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); } + if (dim == 1) { Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); } + if (dim == 2) { Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); } } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { use_Eparser = true; - if (dim == 0) Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); - if (dim == 1) Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); - if (dim == 2) Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); + if (dim == 0) { Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); } + if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } + if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index 8d87dfccc9b..192fc9e9152 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -32,7 +32,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) const auto dx = Geom(lev).CellSizeArray(); amrex::ParserExecutor<3> ref_parser; - if (ref_patch_parser) ref_parser = ref_patch_parser->compile<3>(); + if (ref_patch_parser) { ref_parser = ref_patch_parser->compile<3>(); } const auto ftlo = fine_tag_lo; const auto fthi = fine_tag_hi; #ifdef AMREX_USE_OMP diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 327998a61f3..5b917a81fdf 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -146,7 +146,7 @@ void ConvertLabParamsToBoost() ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - if (gamma_boost <= 1.) return; + if (gamma_boost <= 1.) { return; } Vector prob_lo(AMREX_SPACEDIM); Vector prob_hi(AMREX_SPACEDIM); @@ -313,8 +313,9 @@ void CheckGriddingForRZSpectral () const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); // only check for PSATD in RZ - if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { return; + } int max_level; Vector n_cell(AMREX_SPACEDIM, -1); @@ -419,10 +420,12 @@ void ReadBCParams () const ParmParse pp_boundary("boundary"); pp_boundary.queryarr("field_lo", field_BC_lo, 0, AMREX_SPACEDIM); pp_boundary.queryarr("field_hi", field_BC_hi, 0, AMREX_SPACEDIM); - if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) + if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; - if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) + } + if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; + } AMREX_ALWAYS_ASSERT(field_BC_lo.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(field_BC_hi.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(particle_BC_lo.size() == AMREX_SPACEDIM); diff --git a/Source/Utils/WarpXVersion.cpp b/Source/Utils/WarpXVersion.cpp index 43978a4795a..41abfebb38c 100644 --- a/Source/Utils/WarpXVersion.cpp +++ b/Source/Utils/WarpXVersion.cpp @@ -17,10 +17,11 @@ WarpX::Version () #ifdef WARPX_GIT_VERSION version = std::string(WARPX_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } std::string @@ -30,8 +31,9 @@ WarpX::PicsarVersion () #ifdef PICSAR_GIT_VERSION version = std::string(PICSAR_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } diff --git a/Source/WarpX.H b/Source/WarpX.H index bf39d9275bf..c9aa24d3973 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -37,6 +37,7 @@ #endif #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" @@ -120,6 +121,12 @@ public: void Evolve (int numsteps = -1); + void EvolveImplicitPicardInit (int lev); + void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); + void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); + void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, + amrex::Vector, 3 > >& Efield_n); + MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } @@ -157,6 +164,14 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; + //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + static short evolve_scheme; + //! The maximum number of Picard iterations to do each time step + static int max_picard_iterations; + //! The tolerance for the Picard iteration convergence + static amrex::Real picard_iteration_tolerance; + //! Flags whether the Picard iterations are required to converge + static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). */ @@ -420,6 +435,25 @@ public: const std::string& name, std::optional initial_value); + /** + * \brief + * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) + * and optionally initialize it. This also adds the MultiFab + * to the map of MultiFabs (used to ease the access to MultiFabs from the Python + * interface + * + * \param mf[out] The MultiFab unique pointer to be allocated + * \param mf_model[in] The MultiFab to model + * \param name[in] The name of the MultiFab to use in the map + * \param initial_value[in] The optional initial value + */ + static void AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + int level, + const std::string& name, + std::optional initial_value = {}); + // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) // This is a convenience for the Python interface, allowing all MultiFabs // to be easily referenced from Python. @@ -693,7 +727,7 @@ public: #ifdef WARPX_DIM_RZ // Applies the boundary conditions that are specific to the axis when in RZ. - void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev); + void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const; #endif /** @@ -739,8 +773,10 @@ public: void doQEDEvents (int lev); #endif - void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); - void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false); + void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false, + PushType push_type=PushType::Explicit); + void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false, + PushType push_type=PushType::Explicit); // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)). // Caller must make sure fp and cp have ghost cells filled. @@ -984,6 +1020,16 @@ public: char field, int lev, PatchType patch_type); + /** + * \brief Load field values from a user-specified openPMD file, + * for the fields Ex, Ey, Ez, Bx, By, Bz + */ + void LoadExternalFieldsFromFile (int lev); + + /** + * \brief Load field values from a user-specified openPMD file + * for a specific field (specified by `F_name`) + */ void ReadExternalFieldFromFile ( std::string read_fields_from_path, amrex::MultiFab* mf, std::string F_name, std::string F_component); @@ -1173,8 +1219,8 @@ protected: //! Make a new level from scratch using provided BoxArray and //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. - void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& new_grids, + const amrex::DistributionMapping& new_dmap) final; //! Make a new level using provided BoxArray and //! DistributionMapping and fill with interpolated coarse level @@ -1223,13 +1269,15 @@ private: void AddExternalFields (); - void OneStep_nosub (amrex::Real t); - void OneStep_sub1 (amrex::Real t); + void OneStep_nosub (amrex::Real cur_time); + void OneStep_sub1 (amrex::Real cur_time); + + void OneStep_ImplicitPicard(amrex::Real cur_time); /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ - void OneStep_multiJ (amrex::Real t); + void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( const amrex::Vector,3>>& J_fp, @@ -1298,8 +1346,8 @@ private: void InitFromScratch (); - void AllocLevelData (int lev, const amrex::BoxArray& new_grids, - const amrex::DistributionMapping& new_dmap); + void AllocLevelData (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm); [[nodiscard]] amrex::DistributionMapping GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, int lev) const; @@ -1433,6 +1481,12 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; + // Implicit, fields at start of step and from the previous iteration + amrex::Vector, 3 > > Efield_n; + amrex::Vector, 3 > > Bfield_n; + amrex::Vector, 3 > > Efield_save; + amrex::Vector, 3 > > Bfield_save; + // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 597a561ae2c..f3008e6157d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -111,6 +111,10 @@ short WarpX::charge_deposition_algo; short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; +short WarpX::evolve_scheme; +int WarpX::max_picard_iterations = 10; +Real WarpX::picard_iteration_tolerance = 1.e-7; +bool WarpX::require_picard_convergence = true; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -540,13 +544,13 @@ WarpX::ReadParameters () if(std::string str_abort_on_warning_threshold; pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ std::optional abort_on_warning_threshold = std::nullopt; - if (str_abort_on_warning_threshold == "high") + if (str_abort_on_warning_threshold == "high") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; - else if (str_abort_on_warning_threshold == "medium" ) + } else if (str_abort_on_warning_threshold == "medium" ) { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; - else if (str_abort_on_warning_threshold == "low") + } else if (str_abort_on_warning_threshold == "low") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; - else { + } else { WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); } @@ -771,7 +775,7 @@ WarpX::ReadParameters () // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) WarpX::use_filter = false; + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { WarpX::use_filter = false; } #endif // Read filter and fill IntVect filter_npass_each_dir with @@ -843,8 +847,9 @@ WarpX::ReadParameters () const bool shared_tilesize_is_specified = utils::parser::queryArrWithParser(pp_warpx, "shared_tilesize", vect_shared_tilesize, 0, AMREX_SPACEDIM); if (shared_tilesize_is_specified){ - for (int i=0; i(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering - if (grid_type == GridType::Collocated) galerkin_interpolation = false; + if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } #ifdef WARPX_DIM_RZ // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1 @@ -1141,6 +1146,7 @@ WarpX::ReadParameters () current_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "current_deposition")); charge_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "charge_deposition")); particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); + evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || @@ -1202,7 +1208,7 @@ WarpX::ReadParameters () } // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) galerkin_interpolation = false; + if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } // With the PSATD solver, momentum-conserving field gathering // combined with mesh refinement does not seem to work correctly @@ -1221,6 +1227,44 @@ WarpX::ReadParameters () macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); } + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); + utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); + utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Direct, + "Only Esirkepov or Direct current deposition supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, + "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + particle_pusher_algo == ParticlePusherAlgo::Boris || + particle_pusher_algo == ParticlePusherAlgo::HigueraCary, + "Only the Boris and Higuera particle pushers are supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_gathering_algo != GatheringAlgo::MomentumConserving, + "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); + + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !galerkin_interpolation, + "With implicit and semi-implicit schemes and direct deposition, the Galerkin field gathering must be turned off in order to conserve energy"); + } + + if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + galerkin_interpolation, + "With implicit and semi-implicit schemes and Esirkepov deposition, the Galerkin field gathering must be turned on in order to conserve energy"); + } + } + // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; pp_algo.queryarr("load_balance_intervals", load_balance_intervals_string_vec); @@ -1228,8 +1272,9 @@ WarpX::ReadParameters () load_balance_intervals_string_vec); pp_algo.query("load_balance_with_sfc", load_balance_with_sfc); // Knapsack factor only used with non-SFC strategy - if (!load_balance_with_sfc) + if (!load_balance_with_sfc) { pp_algo.query("load_balance_knapsack_factor", load_balance_knapsack_factor); + } utils::parser::queryWithParser(pp_algo, "load_balance_efficiency_ratio_threshold", load_balance_efficiency_ratio_threshold); load_balance_costs_update_algo = static_cast(GetAlgorithmInteger(pp_algo, "load_balance_costs_update")); @@ -1313,8 +1358,9 @@ WarpX::ReadParameters () pp_warpx, "sort_bin_size", vect_sort_bin_size, 0, AMREX_SPACEDIM); if (sort_bin_size_is_specified){ - for (int i=0; i(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -2651,7 +2703,7 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -2704,7 +2756,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -3191,8 +3243,8 @@ bool WarpX::isAnyBoundaryPML() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) { return true; } } return false; } @@ -3262,3 +3314,21 @@ WarpX::AliasInitMultiFab ( } multifab_map[name_with_suffix] = mf.get(); } + +void +WarpX::AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + const int level, + const std::string& name, + std::optional initial_value) +{ + const auto name_with_suffix = TagWithLevelSuffix(name, level); + const auto tag = amrex::MFInfo().SetTag(name_with_suffix); + mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), mf_model.nGrowVect(), tag); + if (initial_value) { + mf->setVal(*initial_value); + } + multifab_map[name_with_suffix] = mf.get(); +} diff --git a/Source/ablastr/coarsen/average.H b/Source/ablastr/coarsen/average.H index 315938acb48..621b4cb54e2 100644 --- a/Source/ablastr/coarsen/average.H +++ b/Source/ablastr/coarsen/average.H @@ -72,20 +72,22 @@ namespace ablastr::coarsen::average // Compute number of points for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { np[l] = 1; // no coarsening - else + } else { np[l] = cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (2 * (cr[l] - 1) + 1) * sf[l] * sc[l]; // nodal + } } // Compute starting indices of source array (fine) for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { idx_min[l] = ic[l]; // no coarsening - else + } else { idx_min[l] = ic[l] * cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (ic[l] * cr[l] - cr[l] + 1) * sf[l] * sc[l]; // nodal + } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.H b/Source/ablastr/coarsen/sample.H index 6ef0962168a..80eac14e833 100644 --- a/Source/ablastr/coarsen/sample.H +++ b/Source/ablastr/coarsen/sample.H @@ -67,14 +67,14 @@ namespace ablastr::coarsen::sample // Compute number of points for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening - else np[l] = 2-sf[l]; + if ( cr[l] == 1 ) { np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening + } else { np[l] = 2-sf[l]; } } // Compute starting indices of source array (fine) for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening - else idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); + if ( cr[l] == 1 ) { idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening + } else { idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index ab5b135309b..b5661037c7e 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,9 +41,10 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) + if ( crse_ratio > amrex::IntVect(1) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, "source fine MultiFab does not have enough guard cells for this interpolation" ); @@ -118,9 +119,9 @@ namespace ablastr::coarsen::sample "source MultiFab converted to staggering of destination MultiFab is not coarsenable" ); ba_tmp.coarsen( crse_ratio ); - if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) + if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) { Loop( mf_dst, mf_src, dcomp, scomp, ncomp, ngrowvect, crse_ratio ); - else + } else { // Cannot coarsen into MultiFab with different BoxArray or DistributionMapping: // 1) create temporary MultiFab on coarsened version of source BoxArray with same DistributionMapping diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 06416168e88..eaeadfbb2fb 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -162,7 +162,7 @@ computePhi (amrex::Vector const & rho, const bool always_use_bnorm = (max_norm_b > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "ElectrostaticSolver", "Max norm of rho is 0", @@ -306,9 +306,11 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) - if (post_phi_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 6f824c3be34..a8833b070b8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -122,7 +122,7 @@ computeVectorPotential ( amrex::Vector > co const bool always_use_bnorm = (max_comp_J > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "MagnetostaticSolver", "Max norm of J is 0", @@ -242,9 +242,11 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) - if (post_A_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } } // namepace Magnetostatic diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 821a3e0d9a9..da24af94dfb 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -73,8 +73,9 @@ deposit_charge (typename T_PC::ParIterType& pti, { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); - if (num_rho_deposition_guards.has_value()) + if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), "num_rho_deposition_guards are larger than allocated!"); // particle shape @@ -82,14 +83,16 @@ deposit_charge (typename T_PC::ParIterType& pti, // used for MR when we want to deposit for a subset of the particles on the level in the // current box; with offset, we start at a later particle index - if (!np_to_deposit.has_value()) + if (!np_to_deposit.has_value()) { np_to_deposit = pti.numParticles(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_deposit.value() + offset <= pti.numParticles(), "np_to_deposit + offset are out-of-bounds for particle iterator"); int const lev = pti.GetLevel(); - if (!depos_lev.has_value()) + if (!depos_lev.has_value()) { depos_lev = lev; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev.value() == (lev-1)) || (depos_lev.value() == (lev )), "Deposition buffers only work for lev or lev-1"); @@ -100,7 +103,7 @@ deposit_charge (typename T_PC::ParIterType& pti, } // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // Extract deposition order and check that particles shape fits within the guard cells. // NOTE: In specific situations where the staggering of rho and the charge deposition algorithm diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index f23f13773c1..6b476d78114 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -21,8 +21,9 @@ namespace ablastr::profiler AMREX_FORCE_INLINE void device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) + if (do_device_synchronize) { amrex::Gpu::synchronize(); + } } /** An object that conditionally calls device_synchronize() on destruction diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index c33e6a1d154..bfe34d7d875 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -124,7 +124,7 @@ void FillBoundary (amrex::Vector const &mf, bool do_single_precision_comms, const amrex::Periodicity &period, std::optional nodal_sync) { - for (auto x : mf) { + for (auto *x : mf) { ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period, nodal_sync); } } @@ -180,7 +180,7 @@ void OverrideSync (amrex::MultiFab &mf, { BL_PROFILE("ablastr::utils::communication::OverrideSync"); - if (mf.ixType().cellCentered()) return; + if (mf.ixType().cellCentered()) { return; } if (do_single_precision_comms) { diff --git a/Source/ablastr/utils/Serialization.H b/Source/ablastr/utils/Serialization.H index 29173a8eeec..b4dd2efa8ac 100644 --- a/Source/ablastr/utils/Serialization.H +++ b/Source/ablastr/utils/Serialization.H @@ -73,8 +73,9 @@ namespace ablastr::utils::serialization ", except vectors of std::string."); put_in(static_cast(val.size()), vec); - for (const auto &el : val) + for (const auto &el : val) { put_in(el, vec); + } } } @@ -145,8 +146,9 @@ namespace ablastr::utils::serialization const auto length = get_out(it); std::vector res(length); - for (int i = 0; i < length; ++i) + for (int i = 0; i < length; ++i) { res[i] = get_out(it); + } return res; } diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 18d43409ec3..9521e5f9c3d 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -146,8 +146,9 @@ SignalHandling::CheckSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } // We assume that signals will definitely be delivered to rank 0, // and may be delivered to other ranks as well. For coordination, @@ -171,12 +172,12 @@ SignalHandling::CheckSignals () } #if defined(AMREX_USE_MPI) - auto comm = amrex::ParallelDescriptor::Communicator(); // Due to a bug in Cray's MPICH 8.1.13 implementation (CUDA builds on Perlmutter@NERSC in 2022), // we cannot use the MPI_CXX_BOOL C++ datatype here. See WarpX PR #3029 and NERSC INC0183281 static_assert(sizeof(bool) == 1, "We communicate bools as 1 byte-sized type in MPI"); BL_MPI_REQUIRE(MPI_Ibcast(signal_actions_requested, SIGNAL_REQUESTS_SIZE, - MPI_BYTE, 0, comm,&signal_mpi_ibcast_request)); + MPI_BYTE, 0, amrex::ParallelDescriptor::Communicator(), + &signal_mpi_ibcast_request)); #endif } @@ -185,8 +186,9 @@ SignalHandling::WaitSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } #if defined(AMREX_USE_MPI) BL_MPI_REQUIRE(MPI_Wait(&signal_mpi_ibcast_request, MPI_STATUS_IGNORE)); diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index 4ba36ee82e9..629eab8c25d 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -98,25 +98,27 @@ namespace std::string abl_msg_logger::PriorityToString(const Priority& priority) { - if(priority == Priority::high) + if(priority == Priority::high) { return "high"; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return "medium"; - else + } else { return "low"; + } } Priority abl_msg_logger::StringToPriority(const std::string& priority_string) { - if(priority_string == "high") + if(priority_string == "high") { return Priority::high; - else if (priority_string == "medium") + } else if (priority_string == "medium") { return Priority::medium; - else if (priority_string == "low") + } else if (priority_string == "low") { return Priority::low; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Priority string '" + priority_string + "' not recognized"); + } //this silences a "non-void function does not return a value in all control paths" warning return Priority::low; @@ -223,8 +225,9 @@ std::vector Logger::get_msgs() const { auto res = std::vector{}; - for (const auto& msg_w_counter : m_messages) + for (const auto& msg_w_counter : m_messages) { res.emplace_back(msg_w_counter.first); + } return res; } @@ -233,8 +236,9 @@ std::vector Logger::get_msgs_with_counter() const { auto res = std::vector{}; - for (const auto& msg : m_messages) + for (const auto& msg : m_messages) { res.emplace_back(MsgWithCounter{msg.first, msg.second}); + } return res; } @@ -246,8 +250,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const #ifdef AMREX_USE_MPI // Trivial case of only one rank - if (m_num_procs == 1) + if (m_num_procs == 1) { return one_rank_gather_msgs_with_counter_and_ranks(); + } // Find out who is the "gather rank" and how many messages it has const auto my_msgs = get_msgs(); @@ -256,8 +261,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const find_gather_rank_and_its_msgs(how_many_msgs); // If the "gather rank" has zero messages there are no messages at all - if(gather_rank_how_many_msgs == 0) + if(gather_rank_how_many_msgs == 0) { return std::vector{}; + } // All the ranks receive the msgs of the "gather rank" as a byte array const auto serialized_gather_rank_msgs = @@ -347,7 +353,7 @@ Logger::compute_msgs_with_counter_and_ranks( const std::vector& displacements, const int gather_rank) const { - if(m_rank != gather_rank) return std::vector{}; + if(m_rank != gather_rank) { return std::vector{}; } std::vector msgs_with_counter_and_ranks; @@ -369,8 +375,9 @@ Logger::compute_msgs_with_counter_and_ranks( #pragma omp parallel for #endif for(int rr = 0; rr < m_num_procs; ++rr){ //for each rank - if(rr == gather_rank) // (skip gather_rank) + if(rr == gather_rank) { // (skip gather_rank) continue; + } // get counters generated by rank rr auto it = all_data.begin() + displacements[rr]; @@ -461,8 +468,9 @@ void Logger::swap_with_io_rank( if (gather_rank != m_io_rank){ if(m_rank == gather_rank){ auto package = std::vector{}; - for (const auto& el: msgs_with_counter_and_ranks) + for (const auto& el: msgs_with_counter_and_ranks) { abl_ser::put_in_vec(el.serialize(), package); + } auto package_size = static_cast(package.size()); amrex::ParallelDescriptor::Send(&package_size, 1, m_io_rank, 0); @@ -510,9 +518,10 @@ get_serialized_gather_rank_msgs( amrex::ParallelDescriptor::Bcast( &size_serialized_gather_rank_msgs, 1, gather_rank); - if (!is_gather_rank) + if (!is_gather_rank) { serialized_gather_rank_msgs.resize( size_serialized_gather_rank_msgs); + } amrex::ParallelDescriptor::Bcast( serialized_gather_rank_msgs.data(), @@ -554,9 +563,10 @@ compute_package_for_gather_rank( // Add the additional messages seen by the current rank to the package abl_ser::put_in(static_cast(msgs_to_send.size()), package); - for (const auto& el : msgs_to_send) + for (const auto& el : msgs_to_send) { abl_ser::put_in_vec( MsgWithCounter{el.first, el.second}.serialize(), package); + } return package; } diff --git a/Source/ablastr/utils/timer/Timer.H b/Source/ablastr/utils/timer/Timer.H index efbb7d6a2bb..5df37493d61 100644 --- a/Source/ablastr/utils/timer/Timer.H +++ b/Source/ablastr/utils/timer/Timer.H @@ -44,7 +44,7 @@ namespace ablastr::utils::timer * * @return the duration */ - double get_duration () noexcept; + [[nodiscard]] double get_duration () const noexcept; /** @@ -53,7 +53,7 @@ namespace ablastr::utils::timer * * @return the maximum duration across all the MPI ranks */ - double get_global_duration (); + [[nodiscard]] double get_global_duration () const; private: diff --git a/Source/ablastr/utils/timer/Timer.cpp b/Source/ablastr/utils/timer/Timer.cpp index 5682a07c290..096c079fa2a 100644 --- a/Source/ablastr/utils/timer/Timer.cpp +++ b/Source/ablastr/utils/timer/Timer.cpp @@ -24,13 +24,13 @@ Timer::record_stop_time() noexcept } double -Timer::get_duration () noexcept +Timer::get_duration () const noexcept { return m_stop_time - m_start_time; } double -Timer::get_global_duration () +Timer::get_global_duration () const { auto duration = this->get_duration(); amrex::ParallelDescriptor::ReduceRealMax( diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index 889f2df848d..ee052ded299 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -29,15 +29,16 @@ namespace const abl_msg_logger::Priority& priority) { using namespace abl_msg_logger; - if (priority == Priority::low) + if (priority == Priority::low) { return WarnPriority::low; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return WarnPriority::medium; - else if (priority == Priority::high) + } else if (priority == Priority::high) { return WarnPriority::high; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Parsing Priority to WarnPriority has failed"); + } return WarnPriority::high; } @@ -59,10 +60,11 @@ void WarnManager::RecordWarning( WarnPriority priority) { auto msg_priority = abl_msg_logger::Priority::high; - if(priority == WarnPriority::low) + if(priority == WarnPriority::low) { msg_priority = abl_msg_logger::Priority::low; - else if(priority == WarnPriority::medium) + } else if(priority == WarnPriority::medium) { msg_priority = abl_msg_logger::Priority::medium; + } if(m_always_warn_immediately){ @@ -86,10 +88,11 @@ void WarnManager::RecordWarning( if(m_abort_on_warning_threshold){ auto abort_priority = abl_msg_logger::Priority::high; - if(m_abort_on_warning_threshold == WarnPriority::low) + if(m_abort_on_warning_threshold == WarnPriority::low) { abort_priority = abl_msg_logger::Priority::low; - else if(m_abort_on_warning_threshold == WarnPriority::medium) + } else if(m_abort_on_warning_threshold == WarnPriority::medium) { abort_priority = abl_msg_logger::Priority::medium; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( msg_priority < abort_priority, @@ -130,8 +133,9 @@ std::string WarnManager::PrintGlobalWarnings(const std::string& when) const auto all_warnings = m_p_logger->collective_gather_msgs_with_counter_and_ranks(); - if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) + if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) { return "[see I/O rank message]"; + } std::sort(all_warnings.begin(), all_warnings.end(), [](const auto& a, const auto& b){ @@ -217,23 +221,25 @@ std::string WarnManager::PrintWarnMsg( { std::stringstream ss; ss << "* --> "; - if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) + if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) { ss << "[!!!]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) { ss << "[!! ]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) { ss << "[! ]"; - else + } else { ss << "[???]"; + } ss << " [" + msg_with_counter.msg.topic << "] "; - if(msg_with_counter.counter == 2) + if(msg_with_counter.counter == 2) { ss << "[raised twice]\n"; - else if(msg_with_counter.counter == 1) + } else if(msg_with_counter.counter == 1) { ss << "[raised once]\n"; - else + } else { ss << "[raised " << msg_with_counter.counter << " times]\n"; + } ss << MsgFormatter(msg_with_counter.msg.text, warn_line_size, warn_tab_size); @@ -248,8 +254,9 @@ std::string WarnManager::PrintWarnMsg( std::string raised_by = "@ Raised by: "; if (!msg_with_counter_and_ranks.all_ranks){ - for (const auto rr : msg_with_counter_and_ranks.ranks) + for (const auto rr : msg_with_counter_and_ranks.ranks) { raised_by += " " + std::to_string(rr); + } } else{ raised_by += "ALL\n"; @@ -296,8 +303,9 @@ WarnManager::MsgFormatter( msg, line_size-prefix_length); std::stringstream ss_out; - for (const auto& line : wrapped_text) + for (const auto& line : wrapped_text) { ss_out << prefix << line << "\n"; + } return ss_out.str(); } diff --git a/Tools/machines/karolina-it4i/cleanup.sh b/Tools/machines/karolina-it4i/cleanup.sh new file mode 100755 index 00000000000..b4d46edd1e4 --- /dev/null +++ b/Tools/machines/karolina-it4i/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ${HOME}/.spack ${WORK}/src/warpx ${WORK}/spack ${HOME}/.local/lib/python3.11 diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh deleted file mode 100755 index 2695435738f..00000000000 --- a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_cpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_cpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/cpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-cpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-cpu-build --target install --parallel 16 -rm -rf $HOME/src/adios2-cpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-cpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-cpu-build -Duse_openmp=ON -Dcpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-cpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-cpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-cpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-cpu -python3 -m venv ${SW_DIR}/venvs/warpx-cpu -source ${SW_DIR}/venvs/warpx-cpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh new file mode 100755 index 00000000000..0435b5e2926 --- /dev/null +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ################################# +# +set -eu -o pipefail + +# Check: ########################################################## +# +# Was karolina_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your karolina_warpx.profile file!" + echo "Please edit its line 2 to continue!" + return +fi + +# download and activate spack +# this might take about ~ 1 hour +if [ ! -d "$WORK/spack" ] +then + git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack + source $WORK/spack/share/spack/setup-env.sh +else + # If the directory exists, checkout v0.21.0 branch + cd $WORK/spack + git checkout v0.21.0 + git pull origin v0.21.0 + source $WORK/spack/share/spack/setup-env.sh + + # Delete spack env if present + spack env deactivate || true + spack env rm -y warpx-karolina-cuda || true + + cd - +fi + +# create and activate the spack environment +spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml +spack env activate warpx-karolina-cuda +spack install + +# Python ########################################################## +# +python -m pip install --user --upgrade pandas +python -m pip install --user --upgrade matplotlib +# optional +#python -m pip install --user --upgrade yt + +# install or update WarpX dependencies +python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade lasy + +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# python -m pip install --user --upgrade -r $WORK/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/karolina-it4i/install_gpu_dependencies.sh deleted file mode 100755 index a03bb008816..00000000000 --- a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_gpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-gpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-gpu-build --target install --parallel 12 -rm -rf $HOME/src/adios2-gpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/blaspp-gpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/lapackpp-gpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-gpu -python3 -m venv ${SW_DIR}/venvs/warpx-gpu -source ${SW_DIR}/venvs/warpx-gpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example deleted file mode 100644 index c0a3ed53ee3..00000000000 --- a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example +++ /dev/null @@ -1,61 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module load GCCcore/11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load OpenMPI/4.1.4-GCC-11.3.0 - -# optional: for QED support with detailed tables -module load Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -module load OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#module load ccache - -# optional: for Python bindings or libEnsemble -module load Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/cpu/venvs/warpx-cpu" ] -then - source ${HOME}/sw/karolina/cpu/venvs/warpx-cpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -#alias getNode="salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -#alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.qsub b/Tools/machines/karolina-it4i/karolina_gpu.qsub deleted file mode 100644 index 274184ed1ca..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu.qsub +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Authors: Axel Huebl, Andrei Berceanu -# License: BSD-3-Clause-LBNL - -#PBS -q qgpu -#PBS -N WarpX -# Use two full nodes, 8 GPUs per node, 16 GPUs total -#PBS -l select=2:ncpus=128:ngpus=8:mpiprocs=8:ompthreads=16,walltime=00:10:00 -#PBS -A - -cd ${PBS_O_WORKDIR} - -# executable & inputs file or python interpreter & PICMI script here -EXE=./warpx.rz -INPUTS=inputs_rz - -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 - -# run -mpirun -np ${PBS_NP} bash -c " - export CUDA_VISIBLE_DEVICES=\${OMPI_COMM_WORLD_LOCAL_RANK}; - ${EXE} ${INPUTS}" \ - > output.txt diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch new file mode 100644 index 00000000000..6171ff03abc --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -0,0 +1,40 @@ +#!/bin/bash -l + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +#SBATCH --account= +#SBATCH --partition=qgpu +#SBATCH --time=00:10:00 +#SBATCH --job-name=WarpX +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus-per-node=8 +#SBATCH --gpu-bind=single:1 + +#SBATCH --mail-type=ALL +# change me! +#SBATCH --mail-user=someone@example.com +#SBATCH --chdir=/scratch/project//it4i-/runs/warpx + +#SBATCH -o stdout_%j +#SBATCH -e stderr_%j + +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=16 +export SRUN_CPUS_PER_TASK=16 + +# set user rights to u=rwx;g=r-x;o=--- +umask 0027 + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx.rz +INPUTS=./inputs_rz + +# run +srun -K1 ${EXE} ${INPUTS} diff --git a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example deleted file mode 100644 index 174598acaac..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example +++ /dev/null @@ -1,65 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module purge -ml GCCcore/11.3.0 -ml CUDA/11.7.0 -ml OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -ml CMake/3.23.1-GCCcore-11.3.0 - -# optional: for QED support with detailed tables -ml Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -ml OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#ml ccache - -# optional: for Python bindings or libEnsemble -ml Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/gpu/venvs/warpx-gpu" ] -then - source ${HOME}/sw/karolina/gpu/venvs/warpx-gpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -alias getNode="qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00 -I" -# an alias to run a command on a batch node for up to 1hr -# usage: runNode -alias runNode='echo -e "#!/bin/bash\nmpirun -n 4 $1" | qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00' - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example new file mode 100644 index 00000000000..1a4eda19a23 --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -0,0 +1,65 @@ +# please set your project account, ie DD-N-N +export proj="" # change me! + +# Name and Path of this Script ################### (DO NOT change!) +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) + +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file!" + echo "Please edit its line 2 to continue!" + return +fi + +# set env variable storing the path to the work directory +# please check if your project ID belongs to proj1, proj2, proj3 etc +export WORK="/mnt/proj/${proj,,}/${USER}" # change me! +mkdir -p WORK + +# clone warpx +# you can also clone your own fork here, eg git@github.com:/WarpX.git +if [ ! -d "$WORK/src/warpx" ] +then + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx +fi + +# load required modules +module purge +module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + +source $WORK/spack/share/spack/setup-env.sh && spack env activate warpx-karolina-cuda && { + echo "Spack environment 'warpx-karolina-cuda' activated successfully." +} || { + echo "Failed to activate Spack environment 'warpx-karolina-cuda'. Please run install_dependencies.sh." +} + +# Text Editor for Tools ########################## (edit this line) +# examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" +#export EDITOR="nano" # change me! + +# allocate an interactive shell for one hour +# usage: getNode 2 # allocates two interactive nodes (default: 1) +function getNode() { + if [ -z "$1" ] ; then + numNodes=1 + else + numNodes=$1 + fi + export OMP_NUM_THREADS=16 + srun --time=1:00:00 --nodes=$numNodes --ntasks=$((8 * $numNodes)) --ntasks-per-node=8 --cpus-per-task=16 --exclusive --gpus-per-node=8 -p qgpu -A $proj --pty bash +} + +# Environment ##################################################### +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH="8.0" +export SCRATCH="/scratch/project/${proj,,}/${USER}" + +# optimize CPU microarchitecture for AMD EPYC 7763 (zen3) +export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml new file mode 100644 index 00000000000..1cb6a4ac209 --- /dev/null +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -0,0 +1,80 @@ +spack: + specs: +# Karolina's openssl version is deprecated + - openssl certs=system + - pkgconfig + - ccache + - cmake@3.26.5 + - cuda@11.7.0 + - openmpi@4.1.4 +atomics + - fftw + - hdf5@1.14.0 + - adios2@2.9.2 ~mgard + - blaspp + - lapackpp + - boost@1.81.0 +program_options +atomic ~python + - python@3.11.6 + - py-pip + - py-setuptools + - py-wheel + - py-cython + - py-mpi4py + - py-numpy@1.24.2 + - openpmd-api@0.15.2 +python + - py-periodictable@1.5.0 + - py-h5py +# optional +# - py-libensemble +nlopt + + packages: + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + buildable: False + cuda: + externals: + - spec: cuda@11.7.0 + modules: + - CUDA/11.7.0 + buildable: False + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.4 +atomics +cuda %gcc@11.3.0 + modules: + - OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + libfabric: + externals: + - spec: libfabric@1.15.1 %gcc@11.3.0 + modules: + - libfabric/1.15.1-GCCcore-11.3.0 + buildable: False + all: + target: [zen3] + compiler: [gcc@11.3.0] + variants: +mpi ~fortran +cuda cuda_arch=80 + providers: + mpi: [openmpi@4.1.4] + cuda: [cuda@11.7.0] + + compilers: + - compiler: + modules: [GCCcore/11.3.0] + operating_system: centos7 + paths: + cc: /apps/all/GCCcore/11.3.0/bin/gcc + cxx: /apps/all/GCCcore/11.3.0/bin/g++ + f77: /apps/all/GCCcore/11.3.0/bin/gfortran + fc: /apps/all/GCCcore/11.3.0/bin/gfortran + spec: gcc@=11.3.0 + target: x86_64 + flags: {} + environment: {} + extra_rpaths: [] + + view: true + concretizer: + reuse: false + unify: true diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 13ab2ead605..54cd33fe476 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -6,6 +6,12 @@ export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi # required dependencies +module load gpu +module load PrgEnv-gnu +module load craype +module load craype-x86-milan +module load craype-accel-nvidia80 +module load cudatoolkit module load cmake/3.22.0 # optional: for QED support with detailed tables diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index 5d88bb9aeed..e41521b5815 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -11,7 +11,7 @@ module load nano # required dependencies module load cmake/3.20.2 module load gcc/9.3.0 -module load cuda/11.3.1 +module load cuda/11.7.1 # optional: faster re-builds module load ccache diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 442ac425f68..b04682cf91c 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -251,7 +251,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 23.12 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.01 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -270,7 +270,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "ecaa46d0be4b5c79b8806e48e3469000d8bb7252" +set(WarpX_amrex_branch "f1ec8df75c562d2a4822cea84d284cf8e72c2e14" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 57e378c26a0..175d3cab79e 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 23.12 CONFIG REQUIRED) + find_package(pyAMReX 24.01 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "0b2d3f6b160991834534164b7391080fabc48ddb" +set(WarpX_pyamrex_branch "0283db4c8825b5e094b9184fd42a267ca843c61c" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 8da087155ba..5dd813829c7 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - +cd amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 5228b7b3521..d3efeaaacd5 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '23.12', + version = '24.01', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',