From 1f0afdca5e1dee4a4fd7ddd0fd82f17c85acd92b Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 7 Nov 2023 16:08:52 -0500 Subject: [PATCH 01/50] Build libROM on Quartz (#208) * add libROM to CMakelists.txt * Add shell_script * Minor changes * Add module file --- CMakeLists.txt | 24 ++++++++++++++++ scripts/build_quartz_libROM.sh | 52 ++++++++++++++++++++++++++++++++++ scripts/modules.quartz | 14 +++++++++ 3 files changed, 90 insertions(+) create mode 100644 scripts/build_quartz_libROM.sh create mode 100644 scripts/modules.quartz diff --git a/CMakeLists.txt b/CMakeLists.txt index 177d29bf..f1deb89c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,30 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) endif(${SCALAPACK_FOUND}) endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) +# libROM (optional) +set(USE_LIBROM False CACHE BOOL "Build with libROM") +set(LIBROM_PATH "" CACHE STRING "Path of libROM") +if(USE_LIBROM) + if(NOT LIBROM_PATH) + message(FATAL_ERROR "libROM PATH not specified.") + else(NOT LIBROM_PATH) + set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") + message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") + include(ExternalProject) + ExternalProject_Add( + libROM + SOURCE_DIR ${LIBROM_SCRIPTS_PATH} + CONFIGURE_COMMAND "" + BINARY_DIR ${LIBROM_PATH} + BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake + INSTALL_COMMAND "" + ) + include_directories(${LIBROM_PATH}/lib) + link_directories(${LIBROM_PATH}/build/lib) + endif(NOT LIBROM_PATH) +endif(USE_LIBROM) + # ARPACK (optional) set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package") if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT) diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh new file mode 100644 index 00000000..dc2b689b --- /dev/null +++ b/scripts/build_quartz_libROM.sh @@ -0,0 +1,52 @@ +##! /bin/csh -f +## An example script to build on LLNL Peloton systems. +## For now, this script assumes intel/ mkl libraries are being used. + +# load some modules +source scripts/modules.quartz + +# set some environment variables. Set them explicitly or use loaded module path (preferred) +# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +# benign cmake warnings +#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +setenv SCALAPACK_ROOT ${MKLROOT} +setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 + +# We need to define the cmake blas vendor option here to find the right one. +set BLAS_VENDOR = Intel10_64lp + +# manually set the location of BLACS libraries for scalapack +set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +set MGMOL_ROOT = `pwd` + +set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +mkdir -p ${INSTALL_DIR} + +set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# clone the libROM GitHub repo in BUILD_DIR +set USE_LIBROM="On" +set LIBROM_PATH = ${BUILD_DIR}/libROM +git clone https://github.com/LLNL/libROM + +# call cmake +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_CXX_COMPILER=mpic++ \ + -DCMAKE_Fortran_COMPILER=mpif77 \ + -DMPIEXEC_NUMPROC_FLAG="-n" \ + -DBLA_VENDOR=${BLAS_VENDOR} \ + -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ + -DUSE_LIBROM=${USE_LIBROM} \ + -DLIBROM_PATH=${LIBROM_PATH} \ + -DCMAKE_BUILD_TYPE=DEBUG \ + .. + +# call make install +make -j +make install + +# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ +# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/modules.quartz b/scripts/modules.quartz new file mode 100644 index 00000000..220e0a31 --- /dev/null +++ b/scripts/modules.quartz @@ -0,0 +1,14 @@ +module load intel/2022.1.0 +module load hdf5-parallel/1.14.0 +#module load hdf5-parallel +#module load boost +module load mkl +module load cmake/3.14.5 +module load python +## manually add boost path +setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +#setenv MKLROOT $LIBRARY_PATH +#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 +#setenv HDF5ROOT $LD_LIBRARY_PATH +#setenv HDF5ROOT From f7e1f22410e972823848e986a7b448d17717c1ae Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 7 Nov 2023 13:16:41 -0800 Subject: [PATCH 02/50] github ci workflow and docker image building (#215) * Update script to analyze forces on constraints * initial ci workflow files. * default branch name release. * create build directory for compilation. * minor fix --------- Co-authored-by: Jean-Luc Fattebert --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++ .github/workflows/docker_image.yml | 59 ++++++++++++++++++++++++++++++ docker/Dockerfile | 38 +++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker_image.yml create mode 100644 docker/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..a88d7b48 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI +on: + workflow_dispatch: {} + pull_request: + types: [opened, labeled, synchronize] + branches: + - release + # push: + # branches: + # - release + +jobs: + docker-image: + uses: ./.github/workflows/docker_image.yml + build: + runs-on: ubuntu-latest + needs: [docker-image] + container: + image: ghcr.io/llnl/mgmol/mgmol_env:latest + options: --user 1001 --privileged + volumes: + - /mnt:/mnt + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + - name: Check out pylibROM + uses: actions/checkout@v1 + with: + submodules: 'true' + - name: Git Submodules status + run: | + git submodule status + - name: cmake + run: | + mkdir build + cd build + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + - name: make + run: | + cd build && make -j 4 + diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 00000000..90a2447b --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,59 @@ +name: docker-image +on: + workflow_call: + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: llnl/mgmol/mgmol_env + DOCKERPATH: docker + +jobs: + docker-ci: + runs-on: ubuntu-latest + name: "docker env" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: Ana06/get-changed-files@v2.2.0 + id: files + - name: DockerPATH configuration + run: echo "DOCKERPATH=$DOCKERPATH" + - name: DockerPATH - check if files in docker path changed + if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml') + run: | + echo "CI container needs rebuilding..." + echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV + - name: Log into registry ${{ env.REGISTRY }} + if: env.CI_NEEDS_REBUILD + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + if: env.CI_NEEDS_REBUILD + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + flavor: latest=true + - name: Build Container motd + if: env.CI_NEEDS_REBUILD + run: | + echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo mgmol_env/CI Development Container" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + chmod 755 ${{env.DOCKERPATH}}/motd.sh + cat ${{env.DOCKERPATH}}/motd.sh + - name: Docker Image - Build and push + if: env.CI_NEEDS_REBUILD + uses: docker/build-push-action@v3 + with: + push: true + context: ${{ env.DOCKERPATH }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..4ccd50ac --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:22.04 + +ENV ENVDIR=env + +# install sudo +RUN apt-get -yq update && apt-get -yq install sudo + +WORKDIR /$ENVDIR + +# install packages +RUN sudo apt-get install -yq git +RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libboost-all-dev +RUN sudo apt-get install -yq vim +RUN sudo apt-get install -yq git-lfs +RUN sudo apt-get install -yq valgrind +RUN sudo apt-get install -yq wget +RUN sudo apt-get install -yq astyle + +# install lldb and gdb for debugging +RUN sudo apt-get install -yq lldb gdb + +RUN sudo apt-get clean -q + +# NOTE: currently docker does not have a way to set environment variable with a command output. +# The following environment variable should use $(uname -m) for different architecture. +# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ +ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ + +# create and switch to a user +ENV USERNAME=test +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME +RUN adduser $USERNAME sudo +USER $USERNAME +WORKDIR /home/$USERNAME + From 5ec2dbedd62a3f674c3ce1fcaaf90950dcf01410 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 9 Apr 2024 05:40:11 -0700 Subject: [PATCH 03/50] Base for librom-mgmol interface development (#219) * changed the Signal.h header file name. * additional routine to find scalapack library, if nothing is found. * ci workflow with test and format * changed branch name * ctest verbose * added --oversubscribe to cmake MPIEXEC_PREFLAGS variable. * parallel hdf5 package fix. also librom is included. * ci workflow only checks the first 20 tests. * minor fix * rom-fpmd driver and librom dependency in cmake. librom must be compiled before mgmol cmake. * cmake fix and toolchain file for LC quartz. * bug fix on FindSCALAPACK.cmake * librom-mgmol installation script for LC quartz. --- .github/workflows/ci.yml | 45 ++++++++--- CMakeLists.txt | 20 +---- cmake_modules/FindSCALAPACK.cmake | 21 ++++- cmake_modules/FindlibROM.cmake | 11 +++ cmake_toolchains/quartz.default.cmake | 6 ++ docker/Dockerfile | 24 ++++-- scripts/build_quartz_libROM.sh | 69 +++++++++-------- scripts/build_ubuntu22_openmpi.sh | 2 +- scripts/modules.quartz | 21 +++-- src/CMakeLists.txt | 21 +++++ src/MGmol.cc | 2 +- src/md.cc | 2 +- src/rom_main.cc | 102 +++++++++++++++++++++++++ src/tools/{Signal.h => Signal_mgmol.h} | 4 +- src/tools/Timeout.h | 2 +- 15 files changed, 274 insertions(+), 78 deletions(-) create mode 100644 cmake_modules/FindlibROM.cmake create mode 100644 cmake_toolchains/quartz.default.cmake create mode 100644 src/rom_main.cc rename src/tools/{Signal.h => Signal_mgmol.h} (94%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88d7b48..05942636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, labeled, synchronize] branches: - - release + - 'ROMFPMD' # push: # branches: # - release @@ -25,23 +25,48 @@ jobs: uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - name: Set Swap Space - uses: pierotofy/set-swap-space@master - with: - swap-size-gb: 10 - - name: Check out pylibROM + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol uses: actions/checkout@v1 with: submodules: 'true' - - name: Git Submodules status - run: | - git submodule status - name: cmake run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" - name: make run: | cd build && make -j 4 + - name: test + run: | + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format diff --git a/CMakeLists.txt b/CMakeLists.txt index f1deb89c..6898f97b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,24 +127,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) set(USE_LIBROM False CACHE BOOL "Build with libROM") set(LIBROM_PATH "" CACHE STRING "Path of libROM") if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") if(NOT LIBROM_PATH) - message(FATAL_ERROR "libROM PATH not specified.") - else(NOT LIBROM_PATH) - set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) - message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") - message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") - include(ExternalProject) - ExternalProject_Add( - libROM - SOURCE_DIR ${LIBROM_SCRIPTS_PATH} - CONFIGURE_COMMAND "" - BINARY_DIR ${LIBROM_PATH} - BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake - INSTALL_COMMAND "" - ) - include_directories(${LIBROM_PATH}/lib) - link_directories(${LIBROM_PATH}/build/lib) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) endif(USE_LIBROM) # ARPACK (optional) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index dde12c26..74e58e7d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT}) endif(DEFINED ENV{SCALAPACK_ROOT}) if(SCALAPACK_ROOT) - set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT}) + set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64) list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR}) endif(SCALAPACK_ROOT) @@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() -unset(SCALAPACK_NAMES) - mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +# Search for some default library paths +if (NOT SCALAPACK_FOUND) + find_library(SCALAPACK_LIBRARY + NAMES ${SCALAPACK_NAMES} + PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib + /opt/local/lib /opt/sw/lib /sw/lib + ENV LD_LIBRARY_PATH + ENV DYLD_FALLBACK_LIBRARY_PATH + ENV DYLD_LIBRARY_PATH + ENV SCALAPACKDIR + ENV BLACSDIR) + + FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +endif() + +unset(SCALAPACK_NAMES) + if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ccd50ac..a097df9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,23 +10,35 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind RUN sudo apt-get install -yq wget -RUN sudo apt-get install -yq astyle +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format # install lldb and gdb for debugging RUN sudo apt-get install -yq lldb gdb RUN sudo apt-get clean -q -# NOTE: currently docker does not have a way to set environment variable with a command output. -# The following environment variable should use $(uname -m) for different architecture. -# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ -ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 # create and switch to a user ENV USERNAME=test diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index dc2b689b..458dd662 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -1,52 +1,61 @@ -##! /bin/csh -f +#!/bin/bash ## An example script to build on LLNL Peloton systems. ## For now, this script assumes intel/ mkl libraries are being used. -# load some modules +## load some modules source scripts/modules.quartz -# set some environment variables. Set them explicitly or use loaded module path (preferred) -# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid -# benign cmake warnings -#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib -setenv SCALAPACK_ROOT ${MKLROOT} -setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 - -# We need to define the cmake blas vendor option here to find the right one. -set BLAS_VENDOR = Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 - -set MGMOL_ROOT = `pwd` - -set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz mkdir -p ${INSTALL_DIR} -set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +BUILD_DIR=${MGMOL_ROOT}/build_quartz mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} # clone the libROM GitHub repo in BUILD_DIR -set USE_LIBROM="On" -set LIBROM_PATH = ${BUILD_DIR}/libROM +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM +cd libROM +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} # call cmake -cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_CXX_COMPILER=mpic++ \ - -DCMAKE_Fortran_COMPILER=mpif77 \ - -DMPIEXEC_NUMPROC_FLAG="-n" \ - -DBLA_VENDOR=${BLAS_VENDOR} \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DUSE_LIBROM=${USE_LIBROM} \ -DLIBROM_PATH=${LIBROM_PATH} \ - -DCMAKE_BUILD_TYPE=DEBUG \ .. +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + # call make install -make -j -make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..a0ca470a 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install diff --git a/scripts/modules.quartz b/scripts/modules.quartz index 220e0a31..82433acb 100644 --- a/scripts/modules.quartz +++ b/scripts/modules.quartz @@ -1,12 +1,19 @@ -module load intel/2022.1.0 +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake module load hdf5-parallel/1.14.0 -#module load hdf5-parallel -#module load boost -module load mkl -module load cmake/3.14.5 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + module load python -## manually add boost path -setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH #setenv MKLROOT $LIBRARY_PATH #setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7aec664..1575135f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -193,3 +193,24 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) diff --git a/src/MGmol.cc b/src/MGmol.cc index b43c70e3..0d1703d9 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal.h" +#include "Signal_mgmol.h" std::set Signal::recv_; template diff --git a/src/md.cc b/src/md.cc index 1adab46b..6b43048e 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal.h" +#include "Signal_mgmol.h" #include "SpreadsAndCenters.h" #include "tools.h" diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..c7ec278e --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,102 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include +#include +#include +#include +using namespace std; + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_CNR +#include +#endif + +#include + +#include "Control.h" +#include "DistMatrix.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "MatricesBlacsContext.h" +#include "Mesh.h" +#include "PackedCommunicationBuffer.h" +#include "ReplicatedWorkSpace.h" +#include "SparseDistMatrix.h" +#include "magma_singleton.h" +#include "tools.h" + +#include +#include +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +//#include "MemTrack.h" + +int main(int argc, char** argv) +{ + // change handling of memory allocation errors + set_new_handler(noMoreMemory); + + cout.sync_with_stdio(); + + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Initialization failed!!!" << endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mype); + assert(mype > -1); + onpe0 = (mype == 0); + + CAROM::Vector librom_vector(10, false); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Finalize failed!!!" << endl; + } + + time_t tt; + time(&tt); + if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/tools/Signal.h b/src/tools/Signal_mgmol.h similarity index 94% rename from src/tools/Signal.h rename to src/tools/Signal_mgmol.h index 8e8d0a21..245a1ca2 100644 --- a/src/tools/Signal.h +++ b/src/tools/Signal_mgmol.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal.h" +// #include "Signal_mgmol.h" // set Signal::recv_; // // A signal can be registered using, e.g. diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index 18940a78..db60d719 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal.h" +#include "Signal_mgmol.h" #if PCS #include From b436fe5fa586110581cffecc8d8ab481ee5ebab9 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 2 May 2024 13:02:57 -0700 Subject: [PATCH 04/50] Update ROMFPMD branch (#233) * Fix build with HDF5P on (#231) * LoadOrbitalsFromRestartFile and miscellaneous changes (#226) * Signal.h -> mgmol_Signal.h * Carbyne example files. * fixed FindSCALAPACK.cmake to search for default paths * MGmol::loadOrbitalFromRestartFile * nullptr initialization --------- Co-authored-by: Jean-Luc Fattebert --- CMakeLists.txt | 5 +- examples/Carbyne/carbyne.cfg | 26 ++++++ examples/Carbyne/carbyne.in | 14 ++++ scripts/build_condo-mod.sh | 1 + src/Ions.cc | 26 +++--- src/MGmol.cc | 2 +- src/MGmol.h | 4 +- src/OrbitalsExtrapolation.h | 2 +- src/md.cc | 86 +++++++++++++++++++- src/tools/Timeout.h | 2 +- src/tools/{Signal_mgmol.h => mgmol_Signal.h} | 4 +- 11 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 examples/Carbyne/carbyne.cfg create mode 100644 examples/Carbyne/carbyne.in rename src/tools/{Signal_mgmol.h => mgmol_Signal.h} (94%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6898f97b..859d3377 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,6 @@ else() #search for HDF5 if(MGMOL_USE_HDF5P) message(STATUS "Use HDF5 parallel capability") set(HDF5_PREFER_PARALLEL True) - add_definitions(-DMGMOL_USE_HDF5P) endif() message(STATUS "HDF5_ROOT: ${HDF5_ROOT}") find_package(HDF5 REQUIRED COMPONENTS C HL) @@ -73,6 +72,10 @@ else() #search for HDF5 message(FATAL_ERROR "Required HDF5 package not found.") endif (${HDF5_FOUND}) endif() +if(MGMOL_USE_HDF5P) + add_definitions(-DMGMOL_USE_HDF5P) +endif() + set(MGMOL_WITH_LIBXC FALSE CACHE BOOL "Compile with LIBXC") if(${MGMOL_WITH_LIBXC}) diff --git a/examples/Carbyne/carbyne.cfg b/examples/Carbyne/carbyne.cfg new file mode 100644 index 00000000..d153e6f6 --- /dev/null +++ b/examples/Carbyne/carbyne.cfg @@ -0,0 +1,26 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=200 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=3 diff --git a/examples/Carbyne/carbyne.in b/examples/Carbyne/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/examples/Carbyne/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 diff --git a/scripts/build_condo-mod.sh b/scripts/build_condo-mod.sh index f7be0c3b..d9abe034 100755 --- a/scripts/build_condo-mod.sh +++ b/scripts/build_condo-mod.sh @@ -21,6 +21,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ -DBLA_VENDOR=${BLAS_VENDOR} \ + -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.3" \ diff --git a/src/Ions.cc b/src/Ions.cc index ca5120ab..e2cc9916 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -47,9 +47,10 @@ double Ions::max_Vl_radius_ = -1.; double Ions::max_Vnl_radius_ = -1.; template -void writeData2d(hid_t file_id, std::string datasetname, std::vector& data, - const int n, T element) +void writeData2d(HDFrestart& h5f_file, std::string datasetname, + std::vector& data, const size_t n, T element) { + hid_t file_id = h5f_file.file_id(); #ifdef MGMOL_USE_HDF5P if (h5f_file.useHdf5p()) { @@ -67,7 +68,7 @@ void writeData2d(hid_t file_id, std::string datasetname, std::vector& data, else #endif { - size_t dims[2] = { data.size()/n, n }; + size_t dims[2] = { data.size() / n, n }; mgmol_tools::write2d(file_id, datasetname, data, dims); } } @@ -753,7 +754,7 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Atomic_numbers"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -788,12 +789,11 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) // write data hid_t file_id = h5f_file.file_id(); - if (file_id >= 0) { std::string datasetname("/Atomic_names"); std::string empty; - writeData2d(file_id, datasetname, data, 1, empty); + writeData2d(h5f_file, datasetname, data, 1, empty); } } @@ -864,7 +864,7 @@ void Ions::writeLockedAtomNames(HDFrestart& h5f_file) { std::string datasetname("/LockedAtomsNames"); std::string empty; - writeData2d(file_id, datasetname, data, 1, empty); + writeData2d(h5f_file, datasetname, data, 1, empty); } } @@ -900,7 +900,7 @@ void Ions::writeAtomicIDs(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Atomic_IDs"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -937,7 +937,7 @@ void Ions::writeAtomicNLprojIDs(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/AtomicNLproj_IDs"); - writeData2d(file_id, datasetname, data, 1, -1); + writeData2d(h5f_file, datasetname, data, 1, -1); } } @@ -975,7 +975,7 @@ void Ions::writePositions(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_positions"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } @@ -1138,7 +1138,7 @@ void Ions::writeVelocities(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_velocities"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } @@ -1184,7 +1184,7 @@ void Ions::writeRandomStates(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_RandomStates"); - writeData2d(file_id, datasetname, data, 3, (unsigned short)0); + writeData2d(h5f_file, datasetname, data, 3, (unsigned short)0); } } @@ -1343,7 +1343,7 @@ void Ions::writeForces(HDFrestart& h5f_file) if (file_id >= 0) { std::string datasetname("/Ionic_forces"); - writeData2d(file_id, datasetname, data, 3, 1.e32); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); } } diff --git a/src/MGmol.cc b/src/MGmol.cc index 0d1703d9..0af62ebc 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" std::set Signal::recv_; template diff --git a/src/MGmol.h b/src/MGmol.h index 23b507c4..e63e7069 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -102,7 +102,7 @@ class MGmol : public MGmolInterface double total_energy_; ConstraintSet* constraints_; - OrbitalsExtrapolation* orbitals_extrapol_; + OrbitalsExtrapolation* orbitals_extrapol_ = nullptr; float md_time_; int md_iteration_; @@ -301,6 +301,8 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + + OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/OrbitalsExtrapolation.h b/src/OrbitalsExtrapolation.h index 9f2348f7..47c1a21b 100644 --- a/src/OrbitalsExtrapolation.h +++ b/src/OrbitalsExtrapolation.h @@ -44,7 +44,7 @@ class OrbitalsExtrapolation virtual short getNumOrbitalExtrapolations() { return 0; } protected: - OrbitalsType* orbitals_minus1_; + OrbitalsType* orbitals_minus1_ = nullptr; }; #endif diff --git a/src/md.cc b/src/md.cc index 6b43048e..ec288839 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" #include "SpreadsAndCenters.h" #include "tools.h" @@ -674,5 +674,89 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) delete orbitals_extrapol_; } +template +OrbitalsType* MGmol::loadOrbitalFromRestartFile(const std::string filename) +{ + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + /* For now, we only consider double-precision hdf5 I/O. */ + assert(ct.restart_info > 3); + assert((ct.AtomsDynamic() == AtomsDynamicType::MD) || (ct.AtomsDynamic() == AtomsDynamicType::Quench)); + + HDFrestart h5file(filename, myPEenv, ct.out_restart_file_type); + int ierr; + + OrbitalsType *restart_orbitals = nullptr; + + /* This corresponds to MGmol::initial */ + { + // Copy from current orbital, instead of constructing brand-new one + restart_orbitals = new OrbitalsType("ForLoading", *current_orbitals_, false); + + /* This corresponds to MGmol::read_restart_data */ + { + ierr = restart_orbitals->read_func_hdf5(h5file); + } // read_restart_data + } // initial() + + /* This corresponds to MGmol::md */ + { + int flag_extrapolated_data = 0; + if (onpe0) + { + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0"); + } + MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + + /* + If extrapolated function exists, + then function is set as previous orbitals, + while the extrapolated function is set as the current orbitals. + This is how the restart file is saved via dumprestartFile. + */ + if (flag_extrapolated_data) + { + orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation()); + + if (onpe0) os_ << "Create new orbitals_minus1..." << std::endl; + + orbitals_extrapol_->setupPreviousOrbitals(&restart_orbitals, + proj_matrices_, lrs_, local_cluster_, currentMasks_, + corrMasks_, h5file); + } + + /* main workflow delete h5f_file_ here, meaning the loading is over. */ + } // md() + + ierr = h5file.close(); + mmpi.allreduce(&ierr, 1, MPI_MIN); + if (ierr < 0) + { + if (onpe0) + (*MPIdata::serr) + << "loadRestartFile: cannot close file..." << std::endl; + return nullptr; + } + + /* + In returning restart_orbitals, + we hope that the wavefunctions in restart_orbitals are all set. + At least the following functions should return proper data loaded from the file: + + restart_orbitals->getLocNumpt() + restart_orbitals->chromatic_number() + restart_orbitals->getPsi(idx) (for int idx) + */ + return restart_orbitals; +} + template class MGmol; template class MGmol; diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index db60d719..25751b2a 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal_mgmol.h" +#include "mgmol_Signal.h" #if PCS #include diff --git a/src/tools/Signal_mgmol.h b/src/tools/mgmol_Signal.h similarity index 94% rename from src/tools/Signal_mgmol.h rename to src/tools/mgmol_Signal.h index 245a1ca2..33c47b3e 100644 --- a/src/tools/Signal_mgmol.h +++ b/src/tools/mgmol_Signal.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal_mgmol.h" +// #include "mgmol_Signal.h" // set Signal::recv_; // // A signal can be registered using, e.g. From 4ae35994862453bb0e7badf6230b11bcc88b17bd Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 2 May 2024 13:33:25 -0700 Subject: [PATCH 05/50] Parsing ROM related options (#228) * librom compilation setup. * rom config options. * parsing rom config option to Control. --- .gitignore | 1 + CMakeLists.txt | 7 +++++ src/CMakeLists.txt | 14 +++++++-- src/Control.cc | 67 ++++++++++++++++++++++++++++++++++++++++++ src/Control.h | 12 ++++++++ src/MGmol_prototypes.h | 6 ++++ src/mgmol_config.h.in | 19 ++++++++++++ src/read_config.cc | 24 +++++++++++++++ src/rom_Control.h | 42 ++++++++++++++++++++++++++ 9 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 src/mgmol_config.h.in create mode 100644 src/rom_Control.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..81c6b154 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/mgmol_config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 859d3377..75752a3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,10 @@ if(USE_LIBROM) endif(NOT LIBROM_PATH) find_package(libROM REQUIRED) + + if(libROM_FOUND) + set(MGMOL_HAS_LIBROM 1) + endif(libROM_FOUND) endif(USE_LIBROM) # ARPACK (optional) @@ -261,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra") include_directories("${PROJECT_SOURCE_DIR}/src/tools") include_directories("${PROJECT_SOURCE_DIR}/src") +include_directories("${LIBROM_PATH}/lib") +link_libraries(${LIBROM_LIB}) + # add subdirectories for source files, tests add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1575135f..962c163c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,7 @@ +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY) + add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) @@ -8,7 +11,8 @@ add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) -set(link_libs mgmol_distmatrix +set(link_libs + mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices mgmol_numerical_kernels @@ -161,7 +165,11 @@ add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +if(USE_LIBROM) + target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES}) + target_link_libraries(mgmol_src ${LIBROM_LIB}) +endif(USE_LIBROM) target_link_libraries(mgmol_src ${link_libs}) if(${MGMOL_WITH_MAGMA}) @@ -196,7 +204,7 @@ install(TARGETS mgmol-opt DESTINATION bin) # build ROM executable if(USE_LIBROM) add_executable(mgmol-rom rom_main.cc) - target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(mgmol-rom mgmol_src ${link_libs}) target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) diff --git a/src/Control.cc b/src/Control.cc index 7d8dc33f..ce3023e3 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1917,6 +1917,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // synchronize all processors sync(); + +#ifdef MGMOL_HAS_LIBROM + setROMOptions(vm); +#endif } int Control::checkOptions() @@ -2051,3 +2055,66 @@ void Control::printPoissonOptions(std::ostream& os) } os << std::endl; } + +void Control::setROMOptions(const boost::program_options::variables_map& vm) +{ + printWithTimeStamp("Control::setROMOptions()...", std::cout); + + if (onpe0) + { + std::string str = vm["ROM.stage"].as(); + if (str.compare("offline") == 0) + rom_pri_option.rom_stage = ROMStage::OFFLINE; + else if (str.compare("online") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE; + else if (str.compare("build") == 0) + rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("none") == 0) + rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; + + rom_pri_option.restart_file_fmt = vm["ROM.offline.restart_filefmt"].as(); + rom_pri_option.restart_file_minidx = vm["ROM.offline.restart_min_idx"].as(); + rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); + rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + } // onpe0 + + // synchronize all processors + syncROMOptions(); +} + +void Control::syncROMOptions() +{ + if (onpe0 && verbose > 0) + (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl; + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); + mmpi.bcast(rom_pri_option.basis_file, comm_global_); + + auto bcast_check = [](int mpirc) { + if (mpirc != MPI_SUCCESS) + { + (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; + MPI_Abort(comm_global_, 2); + } + }; + + short rom_stage = (short)static_cast(rom_pri_option.rom_stage); + int mpirc; + mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + rom_pri_option.rom_stage = static_cast(rom_stage); +} \ No newline at end of file diff --git a/src/Control.h b/src/Control.h index f1fd8779..1ef7a047 100644 --- a/src/Control.h +++ b/src/Control.h @@ -13,6 +13,10 @@ #include "Species.h" #include "Timeout.h" +/* enumeration and option variables for libROM */ +#include "mgmol_config.h" +#include "rom_Control.h" + #include #include #include @@ -220,6 +224,9 @@ class Control void printRestartLink(); + /* libROM related options */ + ROMPrivateOptions rom_pri_option; + public: static Control* instance() { @@ -707,6 +714,11 @@ class Control } bool AtomsMove() { return (atoms_dyn_ != 0); } + + /* ROM-related options */ + void setROMOptions(const boost::program_options::variables_map& vm); + void syncROMOptions(); + const ROMPrivateOptions getROMOptions() { return rom_pri_option; } }; #endif diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index 8c47c388..207201d7 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -10,9 +10,11 @@ #ifndef MGMOL_PROTOTYPES_H #define MGMOL_PROTOTYPES_H +#include "mgmol_config.h" #include "global.h" #include +namespace po = boost::program_options; class Ions; class KBPsiMatrixSparse; @@ -24,4 +26,8 @@ int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin, bool& tcheck); +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg); +#endif + #endif diff --git a/src/mgmol_config.h.in b/src/mgmol_config.h.in new file mode 100644 index 00000000..0b099f6c --- /dev/null +++ b/src/mgmol_config.h.in @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +#cmakedefine MGMOL_HAS_LIBROM + +#endif diff --git a/src/read_config.cc b/src/read_config.cc index 534981d8..5a1437df 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -14,6 +14,7 @@ #include #include #include +#include "MGmol_prototypes.h" #include namespace po = boost::program_options; @@ -201,6 +202,10 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(""), "Output file for dumping cluster information in vtk format"); +#ifdef MGMOL_HAS_LIBROM + setupROMConfigOption(config); +#endif + // Hidden options, will be allowed in config file, but will not be // shown to the user. po::options_description hidden("Hidden options"); @@ -409,3 +414,22 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } + +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg) +{ + rom_cfg.add_options() + ("ROM.stage", po::value()->default_value("none"), + "ROM workflow stage: offline; build; online; none.") + ("ROM.offline.restart_filefmt", po::value(), + "File name format to read for snapshots.") + ("ROM.offline.restart_min_idx", po::value(), + "Minimum index for snapshot file format.") + ("ROM.offline.restart_max_idx", po::value(), + "Maximum index for snapshot file format.") + ("ROM.offline.basis_file", po::value(), + "File name for libROM snapshot/POD matrices.") + ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), + "Save libROM snapshot file at FOM simulation."); +} +#endif \ No newline at end of file diff --git a/src/rom_Control.h b/src/rom_Control.h new file mode 100644 index 00000000..d1fd6854 --- /dev/null +++ b/src/rom_Control.h @@ -0,0 +1,42 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_CONTROL_H +#define ROM_CONTROL_H + +#include +#include +#include +#include +#include + +enum class ROMStage +{ + OFFLINE, + ONLINE, + RESTORE, // TODO(kevin): what stage is this? + BUILD, + UNSUPPORTED +}; + +/* Stored as a private member variable of Control class */ +struct ROMPrivateOptions +{ + ROMStage rom_stage = ROMStage::UNSUPPORTED; + + std::string restart_file_fmt = ""; + int restart_file_minidx = -1; + int restart_file_maxidx = -1; + std::string basis_file = ""; + + /* save librom snapshot matrix at FOM simulation. */ + bool save_librom_snapshot = false; +}; + +#endif // ROM_CONTROL_H From 14941c53be182fc0d4830a0212a354c8328f7b00 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 1 May 2024 16:27:16 -0700 Subject: [PATCH 06/50] FOM orbital snapshot saving workflow --- src/CMakeLists.txt | 1 + src/MGmol.cc | 14 ++++++++++- src/MGmol.h | 5 ++++ src/md.cc | 12 +++++++++ src/rom.cc | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/rom.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 962c163c..1bad52e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -159,6 +159,7 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc + rom.cc ) add_library(mgmol_src ${SOURCES}) diff --git a/src/MGmol.cc b/src/MGmol.cc index 0af62ebc..501578b8 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1148,7 +1148,19 @@ void MGmol::cleanup() if (ierr < 0) os_ << "WARNING: writing restart data failed!!!" << std::endl; - } + +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 && ct.AtomsDynamic() == AtomsDynamicType::Quench) + { + ierr = save_orbital_snapshot( + filename, *current_orbitals_); + + if (ierr < 0) + os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; + } +#endif + } // if (ct.out_restart_info > 0 && !ct.AtomsMove()) MPI_Barrier(comm_); closing_tm_.stop(); diff --git a/src/MGmol.h b/src/MGmol.h index e63e7069..2a0d0783 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -10,6 +10,8 @@ #ifndef MGMOL_H #define MGMOL_H +#include "mgmol_config.h" + #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -303,6 +305,9 @@ class MGmol : public MGmolInterface } OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); +#ifdef MGMOL_HAS_LIBROM + int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); +#endif }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/md.cc b/src/md.cc index ec288839..4cb1ae10 100644 --- a/src/md.cc +++ b/src/md.cc @@ -641,6 +641,18 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } +#endif + printWithTimeStamp("dumped restart file...", std::cout); } diff --git a/src/rom.cc b/src/rom.cc new file mode 100644 index 00000000..f269f4b8 --- /dev/null +++ b/src/rom.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "mgmol_config.h" +#ifdef MGMOL_HAS_LIBROM + +#include "LocGridOrbitals.h" +#include "MGmol.h" + +#include "librom.h" + +#include +#include +#include +#include + +// Save the wavefunction snapshots +template +int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals) +{ + std::string snapshot_filename = file_path; + struct stat s; + if (stat(file_path.c_str(), &s) == 0) + { + if (s.st_mode & S_IFDIR) + snapshot_filename = file_path + "/orbital"; + else if (s.st_mode & S_IFREG) + { + snapshot_filename = file_path + "_orbital"; + } + else + { + std::cout << file_path << " exists but is not a directory or a file." << std::endl; + return 1; + } + } + + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, snapshot_filename); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + + basis_generator.writeSnapshot(); + + return 0; +} + +template class MGmol; +template class MGmol; + +#endif // MGMOL_HAS_LIBROM From a36d2c7016a5eaa49ee7cf4c7cf6ae89dcd97da0 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 2 May 2024 15:03:07 -0700 Subject: [PATCH 07/50] added braces --- src/rom.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rom.cc b/src/rom.cc index f269f4b8..a554d2cf 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -29,7 +29,9 @@ int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsTy if (stat(file_path.c_str(), &s) == 0) { if (s.st_mode & S_IFDIR) + { snapshot_filename = file_path + "/orbital"; + } else if (s.st_mode & S_IFREG) { snapshot_filename = file_path + "_orbital"; From 4d9f5f3d2fd70161a1f71a3a93f8fa5faa1322cc Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Fri, 3 May 2024 10:28:51 -0700 Subject: [PATCH 08/50] ROM workflow for restart file collection and POD basis training (#230) * rom load restart file and POD basis training. * example rom config file for carbyne. * update docker container with hdf5-tools --- docker/Dockerfile | 2 +- examples/Carbyne/carbyne.rom.cfg | 34 ++++++++ src/CMakeLists.txt | 5 ++ src/rom_main.cc | 145 ++++++++++++++++++++----------- src/rom_workflows.cc | 71 +++++++++++++++ src/rom_workflows.h | 41 +++++++++ 6 files changed, 244 insertions(+), 54 deletions(-) create mode 100644 examples/Carbyne/carbyne.rom.cfg create mode 100644 src/rom_workflows.cc create mode 100644 src/rom_workflows.h diff --git a/docker/Dockerfile b/docker/Dockerfile index a097df9b..8b1ffa94 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,7 @@ RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack- RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs -RUN sudo apt-get install -yq valgrind +RUN sudo apt-get install -yq valgrind hdf5-tools RUN sudo apt-get install -yq wget ### clang-format seems to be updated to 14.0. Not using it for now. # RUN sudo apt-get install -yq clang-format diff --git a/examples/Carbyne/carbyne.rom.cfg b/examples/Carbyne/carbyne.rom.cfg new file mode 100644 index 00000000..cb0cd295 --- /dev/null +++ b/examples/Carbyne/carbyne.rom.cfg @@ -0,0 +1,34 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=5 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 +input_level=4 +input_filename=snapshot0_000 + +[ROM.offline] +restart_filefmt=snapshot0_%03d +restart_min_idx=0 +restart_max_idx=1 +basis_file=carom diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1bad52e7..e33d03d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -162,6 +162,11 @@ set(SOURCES rom.cc ) +if(USE_LIBROM) + list(APPEND SOURCES + rom_workflows.cc) +endif(USE_LIBROM) + add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) diff --git a/src/rom_main.cc b/src/rom_main.cc index c7ec278e..dc877a58 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -23,76 +23,115 @@ // Potentials, eigenvalues and operators in Rydberg // Energies in Hartree // -#include -#include -#include -#include -using namespace std; - -#ifdef _OPENMP -#include -#endif - -#ifdef USE_CNR -#include -#endif - -#include - -#include "Control.h" -#include "DistMatrix.h" -#include "ExtendedGridOrbitals.h" -#include "LocGridOrbitals.h" -#include "MGmol.h" -#include "MGmol_MPI.h" -#include "MPIdata.h" -#include "MatricesBlacsContext.h" -#include "Mesh.h" -#include "PackedCommunicationBuffer.h" -#include "ReplicatedWorkSpace.h" -#include "SparseDistMatrix.h" -#include "magma_singleton.h" -#include "tools.h" - -#include -#include -#include - -#include -namespace po = boost::program_options; - -#include "librom.h" - -//#include "MemTrack.h" +#include "rom_workflows.h" -int main(int argc, char** argv) +// A helper function +template +std::ostream& operator<<(std::ostream& os, const std::vector& v) { - // change handling of memory allocation errors - set_new_handler(noMoreMemory); - - cout.sync_with_stdio(); + copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); + return os; +} +int main(int argc, char** argv) +{ int mpirc = MPI_Init(&argc, &argv); if (mpirc != MPI_SUCCESS) { - cerr << "MPI Initialization failed!!!" << endl; + std::cerr << "MPI Initialization failed!!!" << std::endl; MPI_Abort(MPI_COMM_WORLD, 0); } - MPI_Comm_rank(MPI_COMM_WORLD, &mype); - assert(mype > -1); - onpe0 = (mype == 0); - CAROM::Vector librom_vector(10, false); + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + bool tcheck = false; + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // use configure file if it can be found + // std::string config_filename("mgmol.cfg"); + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin, tcheck); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout); + else + mgmol + = new MGmol(global_comm, *MPIdata::sout); + + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + mgmol->setupFromInput(input_filename); + + if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + + mgmol->setupConstraintsFromInput(constraints_filename); + + mgmol_setup(); + + if (!tcheck) + { + mgmol->setup(); + + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + } + else + { + *MPIdata::sout << " Input parameters OK\n"; + } + delete mgmol; + + } // close main scope + + mgmol_finalize(); mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { - cerr << "MPI Finalize failed!!!" << endl; + std::cerr << "MPI Finalize failed!!!" << std::endl; } time_t tt; time(&tt); - if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; // MemTrack::TrackDumpBlocks(); diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc new file mode 100644 index 00000000..4594dd3a --- /dev/null +++ b/src/rom_workflows.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "rom_workflows.h" +#include +#include +#include + +template +std::string string_format( const std::string& format, Args ... args ) +{ + int size_s = std::snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0' + if( size_s <= 0 ){ throw std::runtime_error( "Error during formatting." ); } + auto size = static_cast( size_s ); + std::unique_ptr buf( new char[ size ] ); + std::snprintf( buf.get(), size, format.c_str(), args ... ); + return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside +} + +template +void readRestartFiles(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + ROMPrivateOptions rom_options = ct.getROMOptions(); + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + MGmol *mgmol = static_cast *>(mgmol_); + OrbitalsType *orbitals = nullptr; + std::string filename; + + /* Read the first snapshot to determin dimension and number of snapshots */ + filename = string_format(rom_options.restart_file_fmt, minidx); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + const int dim = orbitals->getLocNumpt(); + const int chrom_num = orbitals->chromatic_number(); + const int totalSamples = orbitals->chromatic_number() * num_restart; + delete orbitals; + + /* Initialize libROM classes */ + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + + /* Collect the restart files */ + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + assert(dim == orbitals->getLocNumpt()); + assert(chrom_num == orbitals->chromatic_number()); + + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + + delete orbitals; + } + basis_generator.writeSnapshot(); + basis_generator.endSamples(); +} + +template void readRestartFiles(MGmolInterface *mgmol_); +template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h new file mode 100644 index 00000000..dfb22b79 --- /dev/null +++ b/src/rom_workflows.h @@ -0,0 +1,41 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_WORKFLOWS_H +#define ROM_WORKFLOWS_H + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Mesh.h" +#include "mgmol_run.h" +#include "tools.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +template +void readRestartFiles(MGmolInterface *mgmol_); + +#endif // ROM_WORKFLOWS_H From ec0cb30d91ad4b51c80aeed88d4228c25fde6b04 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 20 May 2024 11:57:57 -0700 Subject: [PATCH 09/50] set default values for ROM config options. (#235) --- src/read_config.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/read_config.cc b/src/read_config.cc index 5a1437df..6ce97648 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -421,13 +421,13 @@ void setupROMConfigOption(po::options_description &rom_cfg) rom_cfg.add_options() ("ROM.stage", po::value()->default_value("none"), "ROM workflow stage: offline; build; online; none.") - ("ROM.offline.restart_filefmt", po::value(), + ("ROM.offline.restart_filefmt", po::value()->default_value(""), "File name format to read for snapshots.") - ("ROM.offline.restart_min_idx", po::value(), + ("ROM.offline.restart_min_idx", po::value()->default_value(-1), "Minimum index for snapshot file format.") - ("ROM.offline.restart_max_idx", po::value(), + ("ROM.offline.restart_max_idx", po::value()->default_value(-1), "Maximum index for snapshot file format.") - ("ROM.offline.basis_file", po::value(), + ("ROM.offline.basis_file", po::value()->default_value(""), "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation."); From 29c250254c80eb6b3188437b5a6b7568ef9dc6d5 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Wed, 22 May 2024 11:39:21 -0700 Subject: [PATCH 10/50] Merging PR #234 and #236 to ROMFPMD (#237) * Clean up class Rho (#234) * Clean up class MVPSolver (#236) --------- Co-authored-by: Jean-Luc Fattebert --- src/DavidsonSolver.cc | 2 - src/MGmol.h | 1 + src/MVPSolver.cc | 87 ++++------ src/MVPSolver.h | 15 +- src/Rho.cc | 371 +++--------------------------------------- src/Rho.h | 29 +--- 6 files changed, 67 insertions(+), 438 deletions(-) diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index f21cf7fa..c53e11c0 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -598,7 +598,6 @@ int DavidsonSolver::solve( rho_->computeRho( orbitals, work_orbitals, dm11, dm12, dm21, dm22); - rho_->rescaleTotalCharge(); mgmol_strategy_->update_pot(vh_init, ions_); @@ -663,7 +662,6 @@ int DavidsonSolver::solve( rho_->computeRho( orbitals, work_orbitals, dm11, dm12, dm21, dm22); - rho_->rescaleTotalCharge(); } } // inner iterations diff --git a/src/MGmol.h b/src/MGmol.h index 2a0d0783..b9d7c854 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -305,6 +305,7 @@ class MGmol : public MGmolInterface } OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); + #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); #endif diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 2f8060b7..8ac6a0be 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -52,18 +52,22 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), - ions_(ions) + ions_(ions), + numst_(numst) { - history_length_ = 2; - eks_history_.resize(history_length_, 100000.); + Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 0) + { + os_ << "MVPSolver..." << std::endl; + if (use_old_dm_) os_ << "MVPSolver uses old DM..." << std::endl; + } rho_ = rho; energy_ = energy; electrostat_ = electrostat; mgmol_strategy_ = mgmol_strategy; - numst_ = numst; - work_ = new MatrixType("workMVP", numst_, numst_); + work_ = new MatrixType("workMVP", numst_, numst_); proj_mat_work_ = new ProjectedMatrices(numst_, false, kbT); proj_mat_work_->setup(global_indexes); @@ -123,9 +127,9 @@ void MVPSolver::buildTarget_MVP( { target_tm_.start(); - if (onpe0) std::cout << "buildTarget_MVP()..." << std::endl; - Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 1) + std::cout << "buildTarget_MVP()..." << std::endl; proj_mat_work_->assignH(h11); @@ -135,10 +139,8 @@ void MVPSolver::buildTarget_MVP( // // compute target density matrix, with occupations>0 in numst only // - // if( onpe0 )os_<<"Compute XN..."<setHB2H(); - // if( onpe0 )os_<<"Compute DM..."<updateDM(orbitals_index); target = proj_mat_work_->dm(); @@ -168,18 +170,12 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "---------------------------------------------------------------" "-" << std::endl; - os_ << "Update DM functions using MVP Solver..." << std::endl; + os_ << "Update DM using MVP Solver..." << std::endl; os_ << "---------------------------------------------------------------" "-" << std::endl; } - // step for numerical derivative - - // double nel=orbitals.projMatrices()->getNel(); - // if( onpe0 ) - // os_<<"NEW algorithm: Number of electrons at start = "<::solve(OrbitalsType& orbitals) energy_->saveVofRho(); } - ProjectedMatricesInterface* current_proj_mat - = (inner_it == 0) ? orbitals.getProjMatrices() : proj_mat_work_; + ProjectedMatrices* current_proj_mat + = (inner_it == 0) + ? dynamic_cast*>( + orbitals.getProjMatrices()) + : proj_mat_work_; - double ts0; - double e0 = 0.; const int printE = (ct.verbose > 1) ? 1 : 0; // compute h11 for the current potential by adding local part to @@ -239,49 +236,32 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType h11(h11_nl); mgmol_strategy_->addHlocal2matrix(orbitals, orbitals, h11); + current_proj_mat->assignH(h11); + current_proj_mat->setHB2H(); + if (inner_it == 0) { - // orbitals are new, so a few things need to recomputed - ProjectedMatrices* projmatrices - = dynamic_cast*>( - orbitals.getProjMatrices()); - - projmatrices->assignH(h11); - projmatrices->setHB2H(); - if (use_old_dm_) { - dmInit = projmatrices->dm(); + dmInit = current_proj_mat->dm(); } - - ts0 = evalEntropyMVP(projmatrices, true, os_); - e0 = energy_->evaluateTotal( - ts0, projmatrices, orbitals, printE, os_); } else { - dmInit = proj_mat_work_->dm(); - - proj_mat_work_->assignH(h11); - proj_mat_work_->setHB2H(); - - ts0 = evalEntropyMVP(proj_mat_work_, true, os_); - e0 = energy_->evaluateTotal( - ts0, proj_mat_work_, orbitals, printE, os_); } - // N x N target... - // proj_mat_work_->setHiterativeIndex(orbitals.getIterativeIndex(), - // pot.getIterativeIndex()); + const double ts0 = evalEntropyMVP(current_proj_mat, true, os_); + const double e0 = energy_->evaluateTotal( + ts0, current_proj_mat, orbitals, printE, os_); MatrixType target("target", numst_, numst_); - MatrixType delta_dm("delta_dm", numst_, numst_); buildTarget_MVP(h11, s11, target); if (use_old_dm_ || inner_it > 0) { + MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; @@ -291,18 +271,16 @@ int MVPSolver::solve(OrbitalsType& orbitals) // evaluate free energy at beta=1 // if (onpe0 && ct.verbose > 2) - std::cout << "Target energy..." << std::endl; + std::cout << "MVP --- Target energy..." << std::endl; proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); proj_mat_work_->computeOccupationsFromDM(); if (ct.verbose > 2) current_proj_mat->printOccupations(os_); - double nel = proj_mat_work_->getNel(); + const double nel = proj_mat_work_->getNel(); if (onpe0 && ct.verbose > 1) - os_ << "Number of electrons at beta=1 : " << nel + os_ << "MVP --- Number of electrons at beta=1 : " << nel << std::endl; - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, target); - rho_->rescaleTotalCharge(); mgmol_strategy_->update_pot(vh_init, ions_); @@ -323,13 +301,13 @@ int MVPSolver::solve(OrbitalsType& orbitals) ts1, proj_mat_work_, orbitals, ct.verbose - 1, os_); // line minimization - double beta + const double beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); if (onpe0 && ct.verbose > 0) { os_ << std::setprecision(12); - os_ << std::fixed << "Inner iteration " << inner_it + os_ << std::fixed << "MVP inner iteration " << inner_it << ", E0=" << e0 << ", E1=" << e1; os_ << std::scientific << ", E0'=" << de0 << " -> beta=" << beta; @@ -355,14 +333,13 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "Number of electrons for interpolated DM = " << pnel << std::endl; } - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, *work_); - rho_->rescaleTotalCharge(); } } // inner iterations + // set DM to latest iteration value ProjectedMatrices* projmatrices = dynamic_cast*>( orbitals.getProjMatrices()); @@ -375,11 +352,9 @@ int MVPSolver::solve(OrbitalsType& orbitals) if (onpe0 && ct.verbose > 1) { os_ << "---------------------------------------------------------------" - "-" << std::endl; os_ << "End MVP Solver..." << std::endl; os_ << "---------------------------------------------------------------" - "-" << std::endl; } solve_tm_.stop(); diff --git a/src/MVPSolver.h b/src/MVPSolver.h index a3c29d1d..8e7a7c70 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -24,27 +24,22 @@ template class MVPSolver { private: - MPI_Comm comm_; + const MPI_Comm comm_; std::ostream& os_; - short n_inner_steps_; + const short n_inner_steps_; - bool use_old_dm_; + const bool use_old_dm_; Ions& ions_; + int numst_; + Rho* rho_; Energy* energy_; Electrostatic* electrostat_; - int history_length_; - std::vector eks_history_; - MGmol* mgmol_strategy_; - double de_old_; - double de_; - - int numst_; MatrixType* work_; ProjectedMatrices* proj_mat_work_; diff --git a/src/Rho.cc b/src/Rho.cc index 823e4478..b456f6c1 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -28,8 +28,6 @@ template Timer Rho::compute_tm_("Rho::compute"); template Timer Rho::compute_blas_tm_("Rho::compute_usingBlas"); -// template -// Timer Rho::compute_offdiag_tm_("Rho::compute_offdiag"); #ifdef HAVE_MAGMA template @@ -46,10 +44,6 @@ Rho::Rho() myspin_ = mmpi.myspin(); nspin_ = mmpi.nspin(); - // default values for block sizes - // block_functions_ = 8; - // block_space_ = 256; - Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); np_ = mygrid.size(); @@ -77,31 +71,6 @@ void Rho::setup(const OrthoType orbitals_type, orbitals_indexes_ = orbitals_indexes; } -template -void Rho::extrapolate() -{ - double minus = -1; - double two = 2.; - if (rho_minus1_.empty()) - { - rho_minus1_.resize(nspin_); - rho_minus1_[myspin_].resize(np_); - memcpy(&rho_minus1_[myspin_][0], &rho_[myspin_][0], - np_ * sizeof(RHODTYPE)); - return; - } - RHODTYPE* tmp = new RHODTYPE[np_]; - memcpy(tmp, &rho_[myspin_][0], np_ * sizeof(RHODTYPE)); - - LinearAlgebraUtils::MPscal(np_, two, &rho_[myspin_][0]); - LinearAlgebraUtils::MPaxpy( - np_, minus, &rho_minus1_[myspin_][0], &rho_[myspin_][0]); - - memcpy(&rho_minus1_[myspin_][0], tmp, np_ * sizeof(RHODTYPE)); - - delete[] tmp; -} - template void Rho::axpyRhoc(const double alpha, RHODTYPE* rhoc) { @@ -144,8 +113,6 @@ void Rho::update(OrbitalsType& current_orbitals) computeRho(current_orbitals); - rescaleTotalCharge(); - assert(iterative_index_ >= 0); update_tm_.stop(); @@ -234,68 +201,6 @@ void Rho::rescaleTotalCharge() #endif } -// template -// int Rho::setupSubdomainData(const int iloc, -// const vector& vorbitals, -// const ProjectedMatricesInterface* const projmatrices, -// vector& melements, vector>& vmpsi) -//{ -// // printWithTimeStamp("Rho::setupSubdomainData()...",cout); -// -// const short norb = (short)vorbitals.size(); -// vmpsi.resize(norb); -// -// const vector& loc_indexes(orbitals_indexes_[iloc]); -// const int n_colors = (int)loc_indexes.size(); -// -// if (n_colors == 0) return 0; -// -// vector mycolors; -// mycolors.reserve(n_colors); -// for (int icolor = 0; icolor < n_colors; icolor++) -// if (loc_indexes[icolor] != -1) mycolors.push_back(icolor); -// const int nmycolors = (int)mycolors.size(); -// -// for (short j = 0; j < norb; j++) -// vmpsi[j].resize(nmycolors); -// -// short j = 0; -// for (typename vector::const_iterator it = vorbitals.begin(); -// it != vorbitals.end(); ++it) -// { -// for (int color = 0; color < nmycolors; color++) -// { -// vmpsi[j][color] = (*it)->getPsi(mycolors[color], iloc); -// assert(vmpsi[j][color] != NULL); -// } -// j++; -// } -// -// SquareLocalMatrices& -// localX(projmatrices->getLocalX()); const MATDTYPE* const localX_iloc = -// localX.getSubMatrix(iloc); melements.clear(); melements.resize(nmycolors * -// nmycolors); -// -// for (int i = 0; i < nmycolors; i++) -// { -// const int icolor = mycolors[i]; -// if (norb == 1) -// { -// melements[i * nmycolors + i] -// = localX_iloc[icolor + n_colors * icolor]; -// } -// const int jmax = (norb == 1) ? i : nmycolors; -// for (int j = 0; j < jmax; j++) -// { -// int jcolor = mycolors[j]; -// melements[j * nmycolors + i] -// = localX_iloc[icolor + n_colors * jcolor]; -// } -// } -// -// return nmycolors; -//} - template void Rho::accumulateCharge(const double alpha, const short ix_max, const ORBDTYPE* const psii, const ORBDTYPE* const psij, @@ -305,235 +210,6 @@ void Rho::accumulateCharge(const double alpha, const short ix_max, plrho[ix] += (RHODTYPE)(alpha * (double)psii[ix] * (double)psij[ix]); } -// template -// void Rho::computeRhoSubdomain( -// const int iloc_init, const int iloc_end, const OrbitalsType& orbitals) -//{ -// assert(orbitals_type_ == OrthoType::Eigenfunctions -// || orbitals_type_ == OrthoType::Nonorthogonal); -// -// compute_tm_.start(); -// -// Mesh* mymesh = Mesh::instance(); -// -// const int loc_numpt = mymesh->locNumpt(); -// -// RHODTYPE* const prho = &rho_[myspin_][0]; -// -// vector vorbitals; -// vorbitals.push_back(&orbitals); -// -// const double nondiagfactor = 2.; -// -// for (int iloc = iloc_init; iloc < iloc_end; iloc++) -// { -// const int istart = iloc * loc_numpt; -// -// RHODTYPE* const lrho = &prho[istart]; -// -// vector melements; -// vector> vmpsi; -// -// const int nmycolors = setupSubdomainData( -// iloc, vorbitals, orbitals.projMatrices(), melements, vmpsi); -// assert(vmpsi.size() == 1); -// vector mpsi = vmpsi[0]; -// -// const int nblocks_color = nmycolors / block_functions_; -// const int max_icolor = (nmycolors % block_functions_ == 0) -// ? nmycolors -// : nblocks_color * block_functions_; -// const int missed_rows = nmycolors - max_icolor; -// //(*MPIdata::sout)<<"max_icolor="<::computeRhoSubdomain: -// //missed_rows="<::computeRhoSubdomainOffDiagBlock()...",cout); -// -// Mesh* mymesh = Mesh::instance(); -// -// const int loc_numpt = mymesh->locNumpt(); -// -// RHODTYPE* const prho = &rho_[myspin_][0]; -// -// for (int iloc = iloc_init; iloc < iloc_end; iloc++) -// { -// const int istart = iloc * loc_numpt; -// -// RHODTYPE* const lrho = &prho[istart]; -// -// vector melements; -// vector> vmpsi; -// -// const int nmycolors = setupSubdomainData( -// iloc, vorbitals, projmatrices, melements, vmpsi); -// const int nblocks_color = nmycolors / block_functions_; -// const int max_icolor = (nmycolors % block_functions_ == 0) -// ? nmycolors -// : nblocks_color * block_functions_; -// const int missed_rows = nmycolors - max_icolor; -// //(*MPIdata::sout)<<"max_icolor="<::computeRhoSubdomainOffDiagBlock: -// //missed_rows="<& occ); - // void computeRhoSubdomainOffDiagBlock(const int iloc_init, - // const int iloc_end, - // const std::vector& vorbitals, - // const ProjectedMatricesInterface* const); void computeRhoSubdomainUsingBlas3(const int iloc_init, const int iloc_end, const OrbitalsType& orbitals1, const OrbitalsType& orbitals2); void computeRhoSubdomainUsingBlas3( @@ -64,11 +53,6 @@ class Rho void accumulateCharge(const double alpha, const short ix_max, const ORBDTYPE* const psii, const ORBDTYPE* const psij, RHODTYPE* const plrho); - // int setupSubdomainData(const int iloc, - // const std::vector& vorbitals, - // const ProjectedMatricesInterface* const projmatrices, - // std::vector& melements, - // std::vector>& mpsi); void computeRho(OrbitalsType& orbitals); void computeRho( @@ -76,15 +60,17 @@ class Rho void gatherSpin(); + void resetValues(); + + void rescaleTotalCharge(); + public: // electronic density on grid std::vector> rho_; - std::vector> rho_minus1_; Rho(); ~Rho(){}; - void rescaleTotalCharge(); void setup( const OrthoType orbitals_type, const std::vector>&); void setVerbosityLevel(const int vlevel) { verbosity_level_ = vlevel; } @@ -105,18 +91,11 @@ class Rho void initUniform(); int getIterativeIndex() const { return iterative_index_; } int readRestart(HDFrestart& file); - void extrapolate(); void axpyRhoc(const double alpha, RHODTYPE* rhoc); template double dotWithRho(const T2* const func) const; - // void setupBlockSizes(const int block_functions, const int block_space) - //{ - // block_functions_ = block_functions; - // block_space_ = block_space; - //} - static void printTimers(std::ostream& os); }; From 24b26b6dcbea1bf337ab22299f465747ea4914fb Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Thu, 13 Jun 2024 19:01:54 -0700 Subject: [PATCH 11/50] Update ROMFPMD branch from release (#255) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile * update the rom main driver. * update rom main driver again * temporary fix before the next PR --------- Co-authored-by: Jean-Luc Fattebert --- CMakeLists.txt | 4 +- drivers/CMakeLists.txt | 11 ++ drivers/check_input.cc | 93 ++++++++++++++ drivers/example1.cc | 167 +++++++++++++++++++++++++ src/CMakeLists.txt | 16 +-- src/DFTsolver.cc | 4 +- src/Electrostatic.cc | 4 +- src/Ions.cc | 191 ++++++++++++++-------------- src/Ions.h | 28 +++-- src/MGmol.cc | 241 ++++++++++++++++++------------------ src/MGmol.h | 69 ++++++----- src/MGmolInterface.h | 6 + src/MGmol_prototypes.h | 2 +- src/OrbitalsExtrapolation.h | 2 - src/Rho.cc | 6 + src/computeHij.cc | 11 +- src/lbfgsrlx.cc | 7 +- src/main.cc | 62 +++------- src/md.cc | 93 +++----------- src/mlwf.cc | 6 +- src/quench.cc | 47 +++---- src/readInput.cc | 4 +- src/read_config.cc | 7 +- src/rom_main.cc | 44 +++---- src/rom_workflows.cc | 4 +- src/runfire.cc | 11 +- src/setup.cc | 14 ++- tests/CMakeLists.txt | 2 +- tests/SiH4/testSiH4.py | 13 +- util/xyz2in.py | 46 +++++-- 30 files changed, 729 insertions(+), 486 deletions(-) create mode 100644 drivers/CMakeLists.txt create mode 100644 drivers/check_input.cc create mode 100644 drivers/example1.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 75752a3a..6c48a2e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ if(${MGMOL_WITH_CLANG_FORMAT}) find_package(CLANG_FORMAT) if(${CLANG_FORMAT_FOUND}) message(STATUS "Indent with clang-format") - file(GLOB_RECURSE FORMAT_SOURCES src/*.cc src/*.h tests/*.cc tests/*.h) + file(GLOB_RECURSE FORMAT_SOURCES src/*.cc src/*.h tests/*.cc tests/*.h drivers/*.cc) add_custom_target(format COMMAND ${CLANG_FORMAT_EXECUTABLE} -i -style=file ${FORMAT_SOURCES} DEPENDS ${FORMAT_SOURCES}) @@ -271,5 +271,7 @@ link_libraries(${LIBROM_LIB}) # add subdirectories for source files, tests add_subdirectory(src) +add_subdirectory(drivers) + add_subdirectory(tests) diff --git a/drivers/CMakeLists.txt b/drivers/CMakeLists.txt new file mode 100644 index 00000000..4eb1c741 --- /dev/null +++ b/drivers/CMakeLists.txt @@ -0,0 +1,11 @@ +include_directories( ${CMAKE_SOURCE_DIR}/src ) + +add_executable(check_input check_input.cc) + +add_executable(example1 example1.cc) + +target_include_directories(check_input PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(example1 PRIVATE ${Boost_INCLUDE_DIRS}) + +target_link_libraries(check_input mgmol_src) +target_link_libraries(example1 mgmol_src) diff --git a/drivers/check_input.cc b/drivers/check_input.cc new file mode 100644 index 00000000..14dbaa09 --- /dev/null +++ b/drivers/check_input.cc @@ -0,0 +1,93 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + *MPIdata::sout << " Input parameters OK\n"; + + delete mgmol; + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + return 0; +} diff --git a/drivers/example1.cc b/drivers/example1.cc new file mode 100644 index 00000000..561f5e08 --- /dev/null +++ b/drivers/example1.cc @@ -0,0 +1,167 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e33d03d2..e602101f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -191,13 +191,15 @@ install(TARGETS mgmol_src DESTINATION lib) add_executable(mgmol-opt main.cc) target_include_directories (mgmol-opt PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries(mgmol-opt mgmol_src ${link_libs}) -target_link_libraries(mgmol-opt ${SCALAPACK_LIBRARIES}) -target_link_libraries(mgmol-opt ${HDF5_LIBRARIES}) -target_link_libraries(mgmol-opt ${HDF5_HL_LIBRARIES}) -target_link_libraries(mgmol-opt ${BLAS_LIBRARIES}) -target_link_libraries(mgmol-opt ${LAPACK_LIBRARIES}) -target_link_libraries(mgmol-opt ${Boost_LIBRARIES}) +target_link_libraries(mgmol_src ${link_libs}) +target_link_libraries(mgmol_src ${SCALAPACK_LIBRARIES}) +target_link_libraries(mgmol_src ${LAPACK_LIBRARIES}) +target_link_libraries(mgmol_src ${BLAS_LIBRARIES}) +target_link_libraries(mgmol_src ${HDF5_LIBRARIES}) +target_link_libraries(mgmol_src ${HDF5_HL_LIBRARIES}) +target_link_libraries(mgmol_src ${Boost_LIBRARIES}) + +target_link_libraries(mgmol-opt mgmol_src) if (${OPENMP_CXX_FOUND}) target_link_libraries(mgmol-opt OpenMP::OpenMP_CXX) endif() diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index c1484160..91c4a0f6 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -339,13 +339,15 @@ int DFTsolver::solve(OrbitalsType& orbitals, else { bool updateDM = false; + bool updatedS = false; if (!ct.fullyOccupied()) { orbitals.computeGramAndInvS(); dm_strategy_->dressDM(); updateDM = true; + updatedS = true; } - orbitals.orthonormalizeLoewdin(true, nullptr, updateDM); + orbitals.orthonormalizeLoewdin(updatedS, nullptr, updateDM); orbitals_stepper_->restartMixing(); } diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index b0afeb76..80cb754b 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -36,7 +36,7 @@ Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], const double screening_const) - : laptype_(lap_type) + : laptype_(lap_type), poisson_solver_(nullptr) { assert(bcPoisson[0] >= 0); assert(bcPoisson[1] >= 0); @@ -166,6 +166,8 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Electrostatic::~Electrostatic() { + assert(poisson_solver_ != nullptr); + delete poisson_solver_; if (grhod_ != nullptr) delete grhod_; if (grhoc_ != nullptr) delete grhoc_; diff --git a/src/Ions.cc b/src/Ions.cc index 53ee0cf4..940e12e8 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -10,6 +10,7 @@ #include "Ions.h" #include "Control.h" #include "HDFrestart.h" +#include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "MPIdata.h" #include "Mesh.h" @@ -169,12 +170,9 @@ void Ions::computeMaxNumProjs() << " initialized" << std::endl; #endif - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - max_num_proj_ = std::max(max_num_proj_, (*ion)->nProjectors()); - - ion++; + max_num_proj_ = std::max(max_num_proj_, ion->nProjectors()); } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -210,11 +208,9 @@ void Ions::setup() if (ct.verbose > 0) printWithTimeStamp("Ions::setup()... individual ions...", std::cout); - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - (*ion)->setup(); - ion++; + ion->setup(); } setMapVL(); @@ -243,7 +239,7 @@ Ions::~Ions() void Ions::setupListOverlappingIons() { Control& ct = *(Control::instance()); - if (ct.verbose > 0) + if (ct.verbose > 1) printWithTimeStamp("Ions::setupListOverlappingIons()...", std::cout); overlappingNL_ions_.clear(); @@ -269,7 +265,7 @@ void Ions::setupListOverlappingIons() void Ions::setupInteractingIons() { Control& ct = *(Control::instance()); - if (ct.verbose > 0) + if (ct.verbose > 1) printWithTimeStamp("Ions::setupInteractingIons()...", std::cout); ions_setupInteractingIons_tm.start(); @@ -1569,7 +1565,7 @@ Ion* Ions::findLocalIon(const int index) const return nullptr; } -void Ions::setPositions(const std::vector& tau) +void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); @@ -1586,69 +1582,6 @@ void Ions::setPositions(const std::vector& tau) setup_ = false; } -void Ions::get_positions(std::vector>& rr) const -{ - assert(rr.size() == local_ions_.size()); - if (local_ions_.empty()) return; - std::vector tau(3); - int i = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - tau[0] = (*ion)->position(0); - tau[1] = (*ion)->position(1); - tau[2] = (*ion)->position(2); - rr[i] = tau; - ion++; - i++; - } -} -void Ions::set_positions(const std::vector>& rr) -{ - assert(rr.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) - { - assert(rr[i].size() == 3); - (*ion)->setPosition(rr[i][0], rr[i][1], rr[i][2]); - ion++; - i++; - } -} -void Ions::get_forces(std::vector>& ff) const -{ - assert(ff.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - int i = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - ff[i][0] = (*ion)->force(0); - ff[i][1] = (*ion)->force(1); - ff[i][2] = (*ion)->force(2); - ion++; - i++; - } -} -void Ions::set_forces(const std::vector>& ff) -{ - assert(ff.size() == local_ions_.size()); - - if (local_ions_.empty()) return; - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) - { - (*ion)->setForce(ff[i][0], ff[i][1], ff[i][2]); - ion++; - i++; - } -} - int Ions::readAtoms(const std::string& filename, const bool cell_relative) { @@ -1785,6 +1718,7 @@ int Ions::readAtomsFromXYZ( } ++count; } + delete tfile; } mmpi.bcastGlobal(&count, 1); @@ -1793,6 +1727,17 @@ int Ions::readAtomsFromXYZ( mmpi.bcastGlobal(&crds[0], 3 * natoms); mmpi.bcastGlobal(&spec[0], natoms); + return setAtoms(crds, spec); +} + +int Ions::setAtoms( + const std::vector& crds, const std::vector& spec) +{ + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); + Control& ct(*(Control::instance())); + + const int natoms = crds.size() / 3; + double velocity[3] = { 0., 0., 0. }; bool locked = false; for (int ia = 0; ia < natoms; ++ia) @@ -1820,7 +1765,7 @@ int Ions::readAtomsFromXYZ( } if (spname.compare("") == 0) { - (*MPIdata::serr) << "Ions::readAtomsFromXYZ() --- ERROR: unknown " + (*MPIdata::serr) << "Ions::setAtoms() --- ERROR: unknown " "species for atomic number " << spec[ia] << std::endl; return -1; @@ -1841,7 +1786,7 @@ int Ions::readAtomsFromXYZ( // Populate list_ions_ list // std::cout<<"crds: "<set_here(true); + new_ion->set_here(true); local_ions_.push_back(new_ion); } else - (new_ion)->set_here(false); + new_ion->set_here(false); } else { @@ -2193,7 +2134,7 @@ void Ions::setVelocities(const std::vector& tau0, } } -void Ions::getPositions(std::vector& tau) const +void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); @@ -2207,6 +2148,42 @@ void Ions::getPositions(std::vector& tau) const } } +void Ions::getPositions(std::vector& tau) +{ + std::vector tau_local(3 * local_ions_.size()); + + getLocalPositions(tau_local); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(tau_local, tau); +} + +void Ions::getAtomicNumbers(std::vector& atnumbers) +{ + std::vector local_atnumbers; + + for (auto& ion : local_ions_) + { + local_atnumbers.push_back(ion->atomic_number()); + } + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(local_atnumbers, atnumbers); +} + +void Ions::getForces(std::vector& forces) +{ + std::vector forces_local(3 * local_ions_.size()); + + getLocalForces(forces_local); + + int n = getNumIons(); + forces.resize(3 * n); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(forces_local, forces); +} + void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); @@ -2236,6 +2213,20 @@ void Ions::setPositionsToTau0() } } +void Ions::setPositions( + const std::vector& tau, const std::vector& anumbers) +{ + assert(tau.size() == anumbers.size() * 3); + + // clear previous data + clearLists(); + + num_ions_ = setAtoms(tau, anumbers); + + // setup required after updating local ions positions + setup(); +} + void Ions::setVelocitiesToVel() { assert(velocity_.size() == 3 * local_ions_.size()); @@ -2249,7 +2240,7 @@ void Ions::setVelocitiesToVel() } } -void Ions::getForces(std::vector& tau) const +void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); @@ -2380,12 +2371,10 @@ double Ions::computeMaxNLprojRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& iion : local_ions_) { - double r = (*iion)->radiusNLproj(); + double r = iion->radiusNLproj(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -3061,7 +3050,7 @@ void Ions::updateForcesInteractingIons() MGmol_MPI& mmpi(*(MGmol_MPI::instance())); // get computed forces into fion_ - getForces(fion_); + getLocalForces(fion_); // initialize with local names and forces DistributedIonicData forces_data(local_names_, fion_); @@ -3136,6 +3125,18 @@ void Ions::updateTaupInteractingIons() } } +void Ions::clearLists() +{ + local_ions_.clear(); + std::vector::iterator ion = list_ions_.begin(); + while (ion != list_ions_.end()) + { + delete *ion; + ion++; + } + list_ions_.clear(); +} + // update list of local ions void Ions::updateListIons() { @@ -3160,15 +3161,7 @@ void Ions::updateListIons() // Note: this is based on data from MD std::vectors // First cleanup list_ions_ - local_ions_.clear(); - // delete current ions from list - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) - { - delete *ion; - ion++; - } - list_ions_.clear(); + clearLists(); // Update list starting with local ion data. // This enables overlapping data accumulation with communication. diff --git a/src/Ions.h b/src/Ions.h index 2351d678..772c51d1 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -16,7 +16,6 @@ #include #include -#include "DataDistribution.h" #include "DistributedIonicData.h" #include "Ion.h" #include "hdf5.h" @@ -37,7 +36,12 @@ class Ions const std::vector& species_; std::vector list_ions_; - std::vector local_ions_; // centered in local sub-domain + + /* + * ions located in local sub-domain + */ + std::vector local_ions_; + std::vector interacting_ions_; // for ion-ion interactions std::vector overlappingNL_ions_; // with projectors overlapping local sub-domain @@ -62,7 +66,6 @@ class Ions void readRestartPositions(HDFrestart& h5_file); int read1atom(std::ifstream* tfile, const bool cell_relative); - // void associate2PE(); void setupInteractingIons(); void setupListOverlappingIons(); void setMapVL(); @@ -158,6 +161,7 @@ class Ions void gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const; bool hasLockedAtoms() const; + void clearLists(); public: Ions(const double lat[3], const std::vector& sp); @@ -247,11 +251,7 @@ class Ions // check if ion is in list of local ions bool isLocal(const std::string& ion_name) const; - void setPositions(const std::vector& tau); - void get_positions(std::vector>& r) const; - void set_positions(const std::vector>& r); - void get_forces(std::vector>& f) const; - void set_forces(const std::vector>& f); + void setLocalPositions(const std::vector& tau); void lockAtom(const std::string& name); @@ -260,6 +260,8 @@ class Ions int readAtoms(const std::string& filename, const bool cell_relative); int readAtoms(std::ifstream* tfile, const bool cell_relative); void initFromRestartFile(HDFrestart& h5_file); + int setAtoms( + const std::vector& crds, const std::vector& spec); int getNValenceElectrons() const; void syncForces(); @@ -268,9 +270,15 @@ class Ions void setTau0(); void setPositionsToTau0(); void setVelocitiesToVel(); + void setPositions( + const std::vector& tau, const std::vector& anumbers); + + void getLocalPositions(std::vector& tau) const; + void getPositions(std::vector& tau); + void getAtomicNumbers(std::vector& atnumbers); - void getPositions(std::vector& tau) const; - void getForces(std::vector& tau) const; + void getForces(std::vector& forces); + void getLocalForces(std::vector& tau) const; void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/src/MGmol.cc b/src/MGmol.cc index 39f30378..c91120e9 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -113,73 +113,29 @@ template MGmol::MGmol(MPI_Comm comm, std::ostream& os, std::string input_filename, std::string lrs_filename, std::string constraints_filename) - : os_(os) + : os_(os), comm_(comm) { - comm_ = comm; - - constraints_ = new ConstraintSet(); + constraints_.reset(new ConstraintSet()); mgmol::out = &os; - geom_optimizer_ = nullptr; - local_cluster_ = nullptr; - proj_matrices_ = nullptr; - dm_strategy_ = nullptr; - - h5f_file_ = nullptr; - - aomm_ = nullptr; - - spreadf_ = nullptr; - - spread_penalty_ = nullptr; - - orbitals_precond_ = nullptr; - - forces_ = nullptr; - - energy_ = nullptr; + current_orbitals_ = nullptr; setupFromInput(input_filename); + /* + * Extra setup if using localization regions + */ setupLRs(lrs_filename); - setupConstraintsFromInput(constraints_filename); - - setup(); + if (!constraints_filename.empty()) + setupConstraintsFromInput(constraints_filename); } template MGmol::~MGmol() { - delete electrostat_; - delete rho_; - delete constraints_; - delete xcongrid_; - delete energy_; - if (hamiltonian_ != nullptr) delete hamiltonian_; - if (geom_optimizer_ != nullptr) delete geom_optimizer_; - - if (currentMasks_ != nullptr) delete currentMasks_; - if (corrMasks_ != nullptr) delete corrMasks_; - - if (aomm_ != nullptr) delete aomm_; - - delete current_orbitals_; - delete ions_; - delete g_kbpsi_; - - delete proj_matrices_; - if (local_cluster_ != nullptr) delete local_cluster_; - - if (h5f_file_ != nullptr) delete h5f_file_; - - if (spreadf_ != nullptr) delete spreadf_; - - if (spread_penalty_ != nullptr) delete spread_penalty_; - - delete forces_; - if (dm_strategy_ != nullptr) delete dm_strategy_; + if (current_orbitals_) delete current_orbitals_; } template <> @@ -192,18 +148,17 @@ void MGmol::initialMasks() if (ct.verbose > 0) printWithTimeStamp("MGmol::initialMasks()...", os_); - currentMasks_ = new MasksSet(false, ct.getMGlevels()); + currentMasks_ + = std::shared_ptr(new MasksSet(false, ct.getMGlevels())); currentMasks_->setup(lrs_); - corrMasks_ = new MasksSet(true, 0); + corrMasks_ = std::shared_ptr(new MasksSet(true, 0)); corrMasks_->setup(lrs_); } template <> void MGmol::initialMasks() { - currentMasks_ = nullptr; - corrMasks_ = nullptr; } template @@ -227,7 +182,7 @@ int MGmol::initial() pb::Lap* lapop = ct.Mehrstellen() ? hamiltonian_->lapOper() : nullptr; - g_kbpsi_ = new KBPsiMatrixSparse(lapop); + g_kbpsi_.reset(new KBPsiMatrixSparse(lapop)); check_anisotropy(); @@ -269,7 +224,7 @@ int MGmol::initial() // set number of iterations to 10. if (ct.load_balancing_alpha > 0.0) { - local_cluster_ = new ClusterOrbitals(lrs_); + local_cluster_.reset(new ClusterOrbitals(lrs_)); local_cluster_->setup(); local_cluster_->computeClusters(ct.load_balancing_max_iterations); } @@ -289,29 +244,31 @@ int MGmol::initial() { #ifdef HAVE_MAGMA if (use_replicated_matrix) - proj_matrices_ = new ProjectedMatricesMehrstellen( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset( + new ProjectedMatricesMehrstellen( + ct.numst, with_spin, ct.occ_width)); else #endif - proj_matrices_ = new ProjectedMatricesMehrstellen< + proj_matrices_.reset(new ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>( - ct.numst, with_spin, ct.occ_width); + ct.numst, with_spin, ct.occ_width)); } else if (ct.short_sighted) - proj_matrices_ = new ProjectedMatricesSparse( - ct.numst, ct.occ_width, lrs_, local_cluster_); + proj_matrices_.reset(new ProjectedMatricesSparse( + ct.numst, ct.occ_width, lrs_, local_cluster_.get())); else #ifdef HAVE_MAGMA if (use_replicated_matrix) - proj_matrices_ = new ProjectedMatrices( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset(new ProjectedMatrices( + ct.numst, with_spin, ct.occ_width)); else #endif - proj_matrices_ - = new ProjectedMatrices>( - ct.numst, with_spin, ct.occ_width); + proj_matrices_.reset( + new ProjectedMatrices>( + ct.numst, with_spin, ct.occ_width)); - forces_ = new Forces(hamiltonian_, rho_, proj_matrices_); + forces_.reset(new Forces( + hamiltonian_.get(), rho_.get(), proj_matrices_.get())); if (ct.verbose > 0) printWithTimeStamp( @@ -326,8 +283,8 @@ int MGmol::initial() printWithTimeStamp("MGmol::initial(), create T...", os_); current_orbitals_ = new OrbitalsType("Primary", mygrid, mymesh->subdivx(), - ct.numst, ct.bcWF, proj_matrices_, lrs_, currentMasks_, corrMasks_, - local_cluster_, true); + ct.numst, ct.bcWF, proj_matrices_.get(), lrs_, currentMasks_.get(), + corrMasks_.get(), local_cluster_.get(), true); increaseMemorySlotsForOrbitals(); @@ -372,7 +329,7 @@ int MGmol::initial() "MGmol::initial(), init wf and masks...", os_); // Make temp mask for initial random wave functions - if (ct.init_loc == 1 && currentMasks_ != nullptr) + if (ct.init_loc == 1 && currentMasks_) { float cut_init = ct.initRadius(); assert(cut_init > 0.); @@ -383,7 +340,7 @@ int MGmol::initial() current_orbitals_->initWF(lrs_); // initialize masks again - if (ct.init_loc == 1 && currentMasks_ != nullptr) + if (ct.init_loc == 1 && currentMasks_) { currentMasks_->initialize(lrs_, 0); } @@ -427,9 +384,10 @@ int MGmol::initial() } if (ct.verbose > 0) printWithTimeStamp("Initialize XC functional...", os_); - xcongrid_ = XCfunctionalFactory::create( - ct.xctype, mmpi.nspin(), *rho_, pot); - assert(xcongrid_ != nullptr); + xcongrid_ + = std::shared_ptr(XCfunctionalFactory::create( + ct.xctype, mmpi.nspin(), *rho_, pot)); + assert(xcongrid_); // initialize nl potentials with restart values if possible // if( ct.restart_info>1 ) @@ -473,7 +431,7 @@ int MGmol::initial() { Vector3D origin(mygrid.origin(0), mygrid.origin(1), mygrid.origin(2)); Vector3D ll(mygrid.ll(0), mygrid.ll(1), mygrid.ll(2)); - spreadf_ = new SpreadsAndCenters(origin, ll); + spreadf_.reset(new SpreadsAndCenters(origin, ll)); } bool energy_with_spread_penalty = false; @@ -481,26 +439,30 @@ int MGmol::initial() { if (ct.isSpreadFunctionalVolume()) { - spread_penalty_ = new SpreadPenaltyVolume(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), - ct.spreadPenaltyDampingFactor()); + spread_penalty_.reset( + new SpreadPenaltyVolume(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), + ct.spreadPenaltyDampingFactor())); } else if (ct.isSpreadFunctionalEnergy()) { energy_with_spread_penalty = true; - spread_penalty_ = new EnergySpreadPenalty(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor()); + spread_penalty_.reset( + new EnergySpreadPenalty(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor())); } else - spread_penalty_ = new SpreadPenalty(spreadf_, - ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), - ct.spreadPenaltyDampingFactor()); + spread_penalty_.reset( + new SpreadPenalty(spreadf_.get(), + ct.spreadPenaltyTarget(), ct.spreadPenaltyAlphaFactor(), + ct.spreadPenaltyDampingFactor())); } - SpreadPenaltyInterface* spread_penalty + std::shared_ptr> spread_penalty = energy_with_spread_penalty ? spread_penalty_ : nullptr; - energy_ = new Energy( - mygrid, *ions_, pot, *electrostat_, *rho_, *xcongrid_, spread_penalty); + energy_ = std::shared_ptr>( + new Energy(mygrid, *ions_, pot, *electrostat_, *rho_, + *xcongrid_, spread_penalty.get())); if (ct.verbose > 0) printWithTimeStamp("Setup matrices...", os_); @@ -509,15 +471,16 @@ int MGmol::initial() // HMVP algorithm requires that H is initialized #ifdef HAVE_MAGMA if (use_replicated_matrix) - dm_strategy_ - = DMStrategyFactory::create(comm_, - os_, *ions_, rho_, energy_, electrostat_, this, proj_matrices_, - current_orbitals_); + dm_strategy_.reset( + DMStrategyFactory::create(comm_, + os_, *ions_, rho_.get(), energy_.get(), electrostat_.get(), + this, proj_matrices_.get(), current_orbitals_)); else #endif - dm_strategy_ = DMStrategyFactory>::create(comm_, os_, *ions_, rho_, - energy_, electrostat_, this, proj_matrices_, current_orbitals_); + dm_strategy_.reset(DMStrategyFactory>::create(comm_, os_, *ions_, + rho_.get(), energy_.get(), electrostat_.get(), this, + proj_matrices_.get(), current_orbitals_)); // theta = invB * Hij proj_matrices_->updateThetaAndHB(); @@ -609,7 +572,7 @@ void MGmol::finalEnergy() // Get the total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] total_energy_ = energy_->evaluateTotal( - ts, proj_matrices_, *current_orbitals_, 2, os_); + ts, proj_matrices_.get(), *current_orbitals_, 2, os_); } template @@ -624,7 +587,7 @@ void MGmol::printMM() ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( - proj_matrices_); + proj_matrices_.get()); assert(projmatrices != nullptr); projmatrices->printHamiltonianMM(tfileh); } @@ -697,7 +660,7 @@ void MGmol::write_header() ct.printPoissonOptions(os_); } // onpe0 - if (current_orbitals_ != nullptr && ct.verbose > 0) + if (current_orbitals_ && ct.verbose > 0) { current_orbitals_->printNumst(os_); current_orbitals_->printChromaticNumber(os_); @@ -802,9 +765,9 @@ void MGmol::printEigAndOcc() #ifdef HAVE_MAGMA // try with ReplicatedMatrix first { - ProjectedMatrices* projmatrices - = dynamic_cast*>( - proj_matrices_); + std::shared_ptr> projmatrices + = std::dynamic_pointer_cast< + ProjectedMatrices>(proj_matrices_); if (projmatrices) { projmatrices->printEigenvalues(os_); @@ -815,10 +778,10 @@ void MGmol::printEigAndOcc() #endif if (!printflag) { - ProjectedMatrices>* - projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr< + ProjectedMatrices>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); @@ -1062,21 +1025,23 @@ double MGmol::get_evnl(const Ions& ions) double val; if (ct.short_sighted) { - ProjectedMatricesSparse* projmatrices - = dynamic_cast(proj_matrices_); + std::shared_ptr projmatrices + = std::dynamic_pointer_cast( + proj_matrices_); assert(projmatrices); - val = g_kbpsi_->getEvnl(ions, projmatrices); + val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } else { - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr< + ProjectedMatrices>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); - val = g_kbpsi_->getEvnl(ions, projmatrices); + val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } evnl_tm_.stop(); @@ -1102,11 +1067,11 @@ void MGmol::setup() printWithTimeStamp("MGmol::setup()...", os_); if (ct.verbose > 0) printWithTimeStamp("Setup VH...", os_); - electrostat_ = new Electrostatic( - ct.getPoissonFDtype(), ct.bcPoisson, ct.screening_const); + electrostat_ = std::shared_ptr(new Electrostatic( + ct.getPoissonFDtype(), ct.bcPoisson, ct.screening_const)); electrostat_->setup(ct.vh_init); - rho_ = new Rho(); + rho_ = std::shared_ptr>(new Rho()); rho_->setVerbosityLevel(ct.verbose); #ifdef HAVE_MAGMA @@ -1205,17 +1170,18 @@ template void MGmol::setGamma( const pb::Lap& lapOper, const Potentials& pot) { - assert(orbitals_precond_ != nullptr); + assert(orbitals_precond_); Control& ct = *(Control::instance()); - orbitals_precond_->setGamma(lapOper, pot, ct.getMGlevels(), proj_matrices_); + orbitals_precond_->setGamma( + lapOper, pot, ct.getMGlevels(), proj_matrices_.get()); } template void MGmol::precond_mg(OrbitalsType& phi) { - assert(orbitals_precond_ != nullptr); + assert(orbitals_precond_); orbitals_precond_->precond_mg(phi); } @@ -1446,11 +1412,44 @@ template void MGmol::addResidualSpreadPenalty( OrbitalsType& phi, OrbitalsType& res) { - assert(spread_penalty_ != nullptr); + assert(spread_penalty_); spread_penalty_->addResidual(phi, res); } +template +void MGmol::getAtomicPositions(std::vector& tau) +{ + ions_->getPositions(tau); +} + +template +void MGmol::getAtomicNumbers(std::vector& an) +{ + ions_->getAtomicNumbers(an); +} + +template +double MGmol::evaluateEnergyAndForces( + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) +{ + assert(tau.size() == 3 * atnumbers.size()); + + Control& ct = *(Control::instance()); + + ions_->setPositions(tau, atnumbers); + + double eks = 0.; + quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + + force(*current_orbitals_, *ions_); + + ions_->getForces(forces); + + return eks; +} + template class MGmol; template class MGmol; template int MGmol::initial(); diff --git a/src/MGmol.h b/src/MGmol.h index 1ed04041..ef7a020b 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -44,6 +44,7 @@ class IonicAlgorithm; #include "AOMMprojector.h" #include "ClusterOrbitals.h" #include "DMStrategy.h" +#include "Energy.h" #include "ExtendedGridOrbitals.h" #include "Forces.h" #include "Ions.h" @@ -55,6 +56,8 @@ class IonicAlgorithm; #include "SpreadPenaltyInterface.h" #include "SpreadsAndCenters.h" +#include + template class MGmol : public MGmolInterface { @@ -63,48 +66,49 @@ class MGmol : public MGmolInterface MPI_Comm comm_; - XConGrid* xcongrid_; + std::shared_ptr xcongrid_; OrbitalsType* current_orbitals_; - AOMMprojector* aomm_; + std::shared_ptr aomm_; - Ions* ions_; + std::shared_ptr ions_; - Rho* rho_; + std::shared_ptr> rho_; - Energy* energy_; + std::shared_ptr> energy_; - Hamiltonian* hamiltonian_; + std::shared_ptr> hamiltonian_; - Forces* forces_; + std::shared_ptr> forces_; - MasksSet* currentMasks_; - MasksSet* corrMasks_; + std::shared_ptr currentMasks_; + std::shared_ptr corrMasks_; - // ProjectedMatrices* proj_matrices_; - ProjectedMatricesInterface* proj_matrices_; + std::shared_ptr proj_matrices_; - IonicAlgorithm* geom_optimizer_; + std::shared_ptr> geom_optimizer_; std::shared_ptr lrs_; - ClusterOrbitals* local_cluster_; + std::shared_ptr local_cluster_; + + std::shared_ptr g_kbpsi_; - KBPsiMatrixSparse* g_kbpsi_; + std::shared_ptr> spreadf_; - SpreadsAndCenters* spreadf_; + std::shared_ptr> spread_penalty_; - SpreadPenaltyInterface* spread_penalty_; + std::shared_ptr> dm_strategy_; - DMStrategy* dm_strategy_; + std::shared_ptr h5f_file_; - HDFrestart* h5f_file_; + std::shared_ptr> orbitals_precond_; double total_energy_; - ConstraintSet* constraints_; + std::shared_ptr constraints_; - OrbitalsExtrapolation* orbitals_extrapol_ = nullptr; + std::shared_ptr> orbitals_extrapol_; float md_time_; int md_iteration_; @@ -168,17 +172,30 @@ class MGmol : public MGmolInterface static Timer comp_res_tm_; static Timer init_nuc_tm_; - OrbitalsPreconditioning* orbitals_precond_; - public: - Electrostatic* electrostat_; + std::shared_ptr electrostat_; MGmol(MPI_Comm comm, std::ostream& os, std::string input_filename, std::string lrs_filename, std::string constraints_filename); ~MGmol() override; + /* access functions */ + OrbitalsType* getOrbitals() { return current_orbitals_; } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + void run() override; + + double evaluateEnergyAndForces(const std::vector& tau, + std::vector& atnumbers, std::vector& forces); + + /* + * get internal atomic positions + */ + void getAtomicPositions(std::vector& tau); + + void getAtomicNumbers(std::vector& an); + void initNuc(Ions& ions); void initKBR(); @@ -276,10 +293,6 @@ class MGmol : public MGmolInterface double get_evnl(const Ions& ions); void sebprintPositions(); void sebprintForces(); - void get_positions(std::vector>& r); - void set_positions(std::vector>& r); - void get_forces(std::vector>& f); - void set_forces(std::vector>& f); int nions() { return ions_->getNumIons(); } double getTotalEnergy(); void cleanup(); @@ -306,7 +319,7 @@ class MGmol : public MGmolInterface forces_->force(orbitals, ions); } - OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); + void loadRestartFile(const std::string filename); #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9932dd9f..046459d6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -11,6 +11,7 @@ #define MGMOLINTERFACE_H #include +#include class MGmolInterface { @@ -24,6 +25,11 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual double evaluateEnergyAndForces(const std::vector& tau, + std::vector& atnumbers, std::vector& forces) + = 0; + virtual void getAtomicPositions(std::vector& tau) = 0; + virtual void getAtomicNumbers(std::vector& an) = 0; }; #endif diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index 207201d7..02e37890 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -25,7 +25,7 @@ double getLAeigen(const double tol, const int maxit, Ions& ions); int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, - float& total_spin, bool& with_spin, bool& tcheck); + float& total_spin, bool& with_spin); #ifdef MGMOL_HAS_LIBROM void setupROMConfigOption(po::options_description &rom_cfg); #endif diff --git a/src/OrbitalsExtrapolation.h b/src/OrbitalsExtrapolation.h index 47c1a21b..e2d0ff0c 100644 --- a/src/OrbitalsExtrapolation.h +++ b/src/OrbitalsExtrapolation.h @@ -26,8 +26,6 @@ class OrbitalsExtrapolation virtual ~OrbitalsExtrapolation(); - virtual bool extrapolatedH() const { return false; } - virtual void clearOldOrbitals(); bool getRestartData(OrbitalsType& orbitals); virtual void setupPreviousOrbitals(OrbitalsType** orbitals, diff --git a/src/Rho.cc b/src/Rho.cc index b456f6c1..ca0f20ea 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -175,6 +175,12 @@ void Rho::rescaleTotalCharge() (*MPIdata::sout) << " Rescaling factor: " << t1 << std::endl; (*MPIdata::sout) << " Num. electrons: " << nel << std::endl; } + if (fabs(t1 - 1.) > 0.02) + { + std::cerr << "Error on density too large to continue. Abort." + << std::endl; + mmpi.abort(); + } for (int ispin = 0; ispin < nspin; ispin++) LinearAlgebraUtils::MPscal( diff --git a/src/computeHij.cc b/src/computeHij.cc index 4942959a..75b8ef86 100644 --- a/src/computeHij.cc +++ b/src/computeHij.cc @@ -280,13 +280,14 @@ template void MGmol::getKBPsiAndHij(OrbitalsType& orbitals, Ions& ions, KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij) { - getKBPsiAndHij(orbitals, orbitals, ions, kbpsi, proj_matrices_, hij); + getKBPsiAndHij(orbitals, orbitals, ions, kbpsi, proj_matrices_.get(), hij); } template void MGmol::getKBPsiAndHij(OrbitalsType& orbitals, Ions& ions) { - getKBPsiAndHij(orbitals, orbitals, ions, g_kbpsi_, proj_matrices_); + getKBPsiAndHij( + orbitals, orbitals, ions, g_kbpsi_.get(), proj_matrices_.get()); } template @@ -396,7 +397,7 @@ template void MGmol::getHpsiAndTheta( Ions& ions, OrbitalsType& phi, OrbitalsType& hphi) { - getHpsiAndTheta(ions, phi, hphi, g_kbpsi_); + getHpsiAndTheta(ions, phi, hphi, g_kbpsi_.get()); } template @@ -434,10 +435,10 @@ void MGmol::getHpsiAndTheta(Ions& ions, OrbitalsType& phi, proj_matrices_->clearSparseH(); // Compute the contribution of the non-local potential into sh - kbpsi->computeHvnlMatrix(ions, proj_matrices_); + kbpsi->computeHvnlMatrix(ions, proj_matrices_.get()); // add local part of H to sh - hamiltonian_->addHlocalij(phi, proj_matrices_); + hamiltonian_->addHlocalij(phi, proj_matrices_.get()); energy_->saveVofRho(); diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index 3881abab..b2aff795 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -36,15 +36,14 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) Control& ct = *(Control::instance()); LBFGS lbfgs(orbitals, ions, *rho_, *constraints_, lrs_, - local_cluster_, *currentMasks_, *corrMasks_, *electrostat_, ct.dt, + local_cluster_.get(), *currentMasks_, *corrMasks_, *electrostat_, ct.dt, *this); DFTsolver::resetItCount(); - lbfgs.init(h5f_file_); + lbfgs.init(h5f_file_.get()); - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); // additional quench to compensate random start if (ct.restart_info < 3) diff --git a/src/main.cc b/src/main.cc index f6f396d2..3fcbf791 100644 --- a/src/main.cc +++ b/src/main.cc @@ -29,39 +29,16 @@ #include "MGmol.h" #include "MGmol_MPI.h" #include "MPIdata.h" -#include "Mesh.h" #include "mgmol_run.h" -#include "tools.h" #include -#include #include -#include -#include #include #include -#include - #include namespace po = boost::program_options; -//#include "MemTrack.h" - -/* -void trapfpe () { - feenableexcept(FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW); -} -*/ - -// A helper function -template -std::ostream& operator<<(std::ostream& os, const std::vector& v) -{ - copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); - return os; -} - int main(int argc, char** argv) { int mpirc = MPI_Init(&argc, &argv); @@ -73,33 +50,37 @@ int main(int argc, char** argv) MPI_Comm comm = MPI_COMM_WORLD; + /* + * Initialize general things, like magma, openmp, IO, ... + */ mgmol_init(comm); - // read runtime parameters + /* + * read runtime parameters + */ std::string input_filename(""); std::string lrs_filename; std::string constraints_filename(""); - bool tcheck = false; float total_spin = 0.; bool with_spin = false; po::variables_map vm; - // use configure file if it can be found - // std::string config_filename("mgmol.cfg"); - - // read options from PE0 only + // read from PE0 only if (MPIdata::onpe0) { read_config(argc, argv, vm, input_filename, lrs_filename, - constraints_filename, total_spin, with_spin, tcheck); + constraints_filename, total_spin, with_spin); } MGmol_MPI::setup(comm, std::cout, with_spin); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); MPI_Comm global_comm = mmpi.commGlobal(); + /* + * Setup control struct with run time parameters + */ Control::setup(global_comm, with_spin, total_spin); Control& ct = *(Control::instance()); @@ -108,11 +89,6 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; - unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; - double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; - const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; - Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); - mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); @@ -126,14 +102,10 @@ int main(int argc, char** argv) mgmol = new MGmol(global_comm, *MPIdata::sout, input_filename, lrs_filename, constraints_filename); - if (!tcheck) - { - mgmol->run(); - } - else - { - *MPIdata::sout << " Input parameters OK\n"; - } + mgmol->setup(); + + mgmol->run(); + delete mgmol; } // close main scope @@ -150,9 +122,5 @@ int main(int argc, char** argv) time(&tt); if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; - // MemTrack::TrackDumpBlocks(); - - // MemTrack::TrackListMemoryUsage(); - return 0; } diff --git a/src/md.cc b/src/md.cc index 089deb82..939c6e62 100644 --- a/src/md.cc +++ b/src/md.cc @@ -91,11 +91,6 @@ void MGmol::postWFextrapolation(OrbitalsType* orbitals) else proj_matrices_->setDMto2InvS(); } - else - { - if (orbitals_extrapol_->extrapolatedH()) - dm_strategy_->update(*orbitals); - } } template @@ -137,8 +132,8 @@ OrbitalsType* MGmol::new_orbitals_with_current_LRs(bool setup) // need to build new orbitals as masks have changed OrbitalsType* new_orbitals = new OrbitalsType("NewMasks", mygrid, - mymesh->subdivx(), ct.numst, ct.bcWF, proj_matrices_, lrs_, - currentMasks_, corrMasks_, local_cluster_, setup); + mymesh->subdivx(), ct.numst, ct.bcWF, proj_matrices_.get(), lrs_, + currentMasks_.get(), corrMasks_.get(), local_cluster_.get(), setup); return new_orbitals; } @@ -317,8 +312,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) int size_tau = (int)tau0.size(); DFTsolver::resetItCount(); - orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( - ct.WFExtrapolation()); + orbitals_extrapol_.reset(OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation())); MD_IonicStepper* stepper = new MD_IonicStepper( ct.dt, atmove, tau0, taup, taum, fion, pmass, rand_states); @@ -349,8 +344,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) constraints_->enforceConstraints(20); stepper->updateTau(); - ions.setPositions(tau0); + ions.setLocalPositions(tau0); + // setup required after updating local ions positions ions.setup(); } @@ -376,8 +372,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (onpe0) os_ << "Create new orbitals_minus1..." << std::endl; orbitals_extrapol_->setupPreviousOrbitals(¤t_orbitals_, - proj_matrices_, lrs_, local_cluster_, currentMasks_, - corrMasks_, *h5f_file_); + proj_matrices_.get(), lrs_, local_cluster_.get(), + currentMasks_.get(), corrMasks_.get(), *h5f_file_); // need to reset a few things as we just read new orbitals (*orbitals)->computeGramAndInvS(); @@ -395,8 +391,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) moveVnuc(ions); } - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); } // additional SC steps to compensate random start @@ -437,7 +432,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.adaptiveLRs()) { assert(lrs_); - adaptLR(spreadf_, nullptr); + adaptLR(spreadf_.get(), nullptr); last_move_is_small = lrs_->moveIsSmall(); @@ -495,7 +490,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) force(**orbitals, ions); // set fion - ions.getForces(fion); + ions.getLocalForces(fion); // constraints need to be added again and setup as atoms // may have moved and local atoms are not the same anymore @@ -682,11 +677,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) } delete stepper; - delete orbitals_extrapol_; + orbitals_extrapol_.reset(); } template -OrbitalsType* MGmol::loadOrbitalFromRestartFile( +void MGmol::loadRestartFile( const std::string filename) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -700,53 +695,15 @@ OrbitalsType* MGmol::loadOrbitalFromRestartFile( || (ct.AtomsDynamic() == AtomsDynamicType::Quench)); HDFrestart h5file(filename, myPEenv, ct.out_restart_file_type); - int ierr; - - OrbitalsType* restart_orbitals = nullptr; - - /* This corresponds to MGmol::initial */ - { - // Copy from current orbital, instead of constructing brand-new one - restart_orbitals - = new OrbitalsType("ForLoading", *current_orbitals_, false); - /* This corresponds to MGmol::read_restart_data */ - { - ierr = restart_orbitals->read_func_hdf5(h5file); - } // read_restart_data - } // initial() - - /* This corresponds to MGmol::md */ + int ierr = read_restart_data(h5file, *rho_, *current_orbitals_); + if (ierr < 0) { - int flag_extrapolated_data = 0; if (onpe0) - { - flag_extrapolated_data - = h5file.dset_exists("ExtrapolatedFunction0000"); - if (flag_extrapolated_data == 0) - flag_extrapolated_data - = h5file.dset_exists("ExtrapolatedFunction0"); - } - MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); - - /* - If extrapolated function exists, - then function is set as previous orbitals, - while the extrapolated function is set as the current orbitals. - This is how the restart file is saved via dumprestartFile. + (*MPIdata::serr) << "loadRestartFile: failed to read the restart file." << std::endl; - For now, we just enforce to not use the restart files with extrapolation. - */ - if (flag_extrapolated_data) - { - if (onpe0) - (*MPIdata::serr) << "loadRestartFile: does not support restart files with extrapolation." << std::endl; - - global_exit(0); - } - - /* main workflow delete h5f_file_ here, meaning the loading is over. */ - } // md() + global_exit(0); + } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); @@ -755,20 +712,10 @@ OrbitalsType* MGmol::loadOrbitalFromRestartFile( if (onpe0) (*MPIdata::serr) << "loadRestartFile: cannot close file..." << std::endl; - return nullptr; + return; } - /* - In returning restart_orbitals, - we hope that the wavefunctions in restart_orbitals are all set. - At least the following functions should return proper data loaded from - the file: - - restart_orbitals->getLocNumpt() - restart_orbitals->chromatic_number() - restart_orbitals->getPsi(idx) (for int idx) - */ - return restart_orbitals; + return; } template class MGmol; diff --git a/src/mlwf.cc b/src/mlwf.cc index cc4fdeb8..3efdfeb2 100644 --- a/src/mlwf.cc +++ b/src/mlwf.cc @@ -279,9 +279,9 @@ int MGmol::get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, } sincos.clear(); - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( + std::shared_ptr>> + projmatrices = std::dynamic_pointer_cast< + ProjectedMatrices>>( proj_matrices_); assert(projmatrices); diff --git a/src/quench.cc b/src/quench.cc index de01dad3..614aa636 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -326,8 +326,9 @@ bool MGmol::rotateStatesPairsOverlap( spreadf2st.print(os_); orbitals.computeGramAndInvS(); - ProjectedMatricesSparse* projmatrices - = dynamic_cast(proj_matrices_); + std::shared_ptr projmatrices + = std::dynamic_pointer_cast( + proj_matrices_); if (onpe0) std::cout << "Gram Matrix for pair of states after transformation:" << std::endl; @@ -398,7 +399,7 @@ void MGmol::disentangleOrbitals(OrbitalsType& orbitals, template <> void MGmol::applyAOMMprojection(LocGridOrbitals& orbitals) { - aomm_ = new AOMMprojector(orbitals, lrs_); + aomm_.reset(new AOMMprojector(orbitals, lrs_)); aomm_->projectOut(orbitals); } @@ -422,8 +423,9 @@ int MGmol::outerSolve(LocGridOrbitals& orbitals, case OuterSolverType::ABPG: case OuterSolverType::NLCG: { - DFTsolver solver(hamiltonian_, proj_matrices_, - energy_, electrostat_, this, ions, rho_, dm_strategy_, os_); + DFTsolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -433,9 +435,9 @@ int MGmol::outerSolve(LocGridOrbitals& orbitals, case OuterSolverType::PolakRibiere: { - PolakRibiereSolver solver(hamiltonian_, - proj_matrices_, energy_, electrostat_, this, ions, rho_, - dm_strategy_, os_); + PolakRibiereSolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -466,8 +468,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, case OuterSolverType::ABPG: case OuterSolverType::NLCG: { - DFTsolver solver(hamiltonian_, proj_matrices_, - energy_, electrostat_, this, ions, rho_, dm_strategy_, os_); + DFTsolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -477,9 +480,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, case OuterSolverType::PolakRibiere: { - PolakRibiereSolver solver(hamiltonian_, - proj_matrices_, energy_, electrostat_, this, ions, rho_, - dm_strategy_, os_); + PolakRibiereSolver solver(hamiltonian_.get(), + proj_matrices_.get(), energy_.get(), electrostat_.get(), this, + ions, rho_.get(), dm_strategy_.get(), os_); retval = solver.solve( orbitals, work_orbitals, ions, max_steps, iprint, last_eks); @@ -499,8 +502,9 @@ int MGmol::outerSolve(OrbitalsType& orbitals, #else DavidsonSolver> #endif - solver(os_, *ions_, hamiltonian_, rho_, energy_, electrostat_, - this, gids, ct.dm_mix, with_spin); + solver(os_, *ions_, hamiltonian_.get(), rho_.get(), + energy_.get(), electrostat_.get(), this, gids, ct.dm_mix, + with_spin); retval = solver.solve(orbitals, work_orbitals); break; @@ -558,9 +562,9 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, applyAOMMprojection(*orbitals); } - orbitals_precond_ = new OrbitalsPreconditioning(); + orbitals_precond_.reset(new OrbitalsPreconditioning()); orbitals_precond_->setup( - *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_, lrs_); + *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); // solve electronic structure problem // (inner iterations) @@ -570,11 +574,9 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, if (ct.use_kernel_functions) { - delete aomm_; - aomm_ = nullptr; + aomm_.reset(); } - delete orbitals_precond_; - orbitals_precond_ = nullptr; + orbitals_precond_.reset(); // Get the n.l. energy // TODO: Fix bug where energy vs. time output is incorrect if get_evnl is @@ -599,7 +601,8 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, << " TS [Ha] = " << ts << std::endl; } } - last_eks = energy_->evaluateTotal(ts, proj_matrices_, *orbitals, 2, os_); + last_eks + = energy_->evaluateTotal(ts, proj_matrices_.get(), *orbitals, 2, os_); if (ct.computeCondGramMD()) { diff --git a/src/readInput.cc b/src/readInput.cc index b0a47411..79fbecf1 100644 --- a/src/readInput.cc +++ b/src/readInput.cc @@ -175,7 +175,7 @@ int MGmol::readCoordinates( // setup ions const std::vector& sp(ct.getSpecies()); - ions_ = new Ions(lattice, sp); + ions_.reset(new Ions(lattice, sp)); if (ct.restart_info > 0 && ct.override_restart == 0) // read restart ionic positions @@ -217,7 +217,7 @@ int MGmol::readCoordinates( // setup ions const std::vector& sp(ct.getSpecies()); - ions_ = new Ions(lattice, sp); + ions_.reset(new Ions(lattice, sp)); if (ct.restart_info > 0 && ct.override_restart == 0) // read restart ionic positions diff --git a/src/read_config.cc b/src/read_config.cc index 6ce97648..74533dfb 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -21,8 +21,7 @@ namespace po = boost::program_options; int read_config(int argc, char** argv, po::variables_map& vm, std::string& input_file, std::string& lrs_filename, - std::string& constraints_filename, float& total_spin, bool& with_spin, - bool& tcheck) + std::string& constraints_filename, float& total_spin, bool& with_spin) { // use configure file if it can be found // std::string config_filename("mgmol.cfg"); @@ -378,10 +377,6 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } - if (vm.count("check")) - { - tcheck = true; - } if (vm.count("spin")) { total_spin = vm["spin"].as(); diff --git a/src/rom_main.cc b/src/rom_main.cc index dc877a58..3718078f 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -50,7 +50,6 @@ int main(int argc, char** argv) std::string input_filename(""); std::string lrs_filename; std::string constraints_filename(""); - bool tcheck = false; float total_spin = 0.; bool with_spin = false; @@ -64,7 +63,7 @@ int main(int argc, char** argv) if (MPIdata::onpe0) { read_config(argc, argv, vm, input_filename, lrs_filename, - constraints_filename, total_spin, with_spin, tcheck); + constraints_filename, total_spin, with_spin); } MGmol_MPI::setup(comm, std::cout, with_spin); @@ -79,6 +78,11 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); @@ -86,37 +90,19 @@ int main(int argc, char** argv) { MGmolInterface* mgmol; if (ct.isLocMode()) - mgmol = new MGmol(global_comm, *MPIdata::sout); + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); else - mgmol - = new MGmol(global_comm, *MPIdata::sout); - - unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; - double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; - const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; - Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); - - mgmol->setupFromInput(input_filename); - - if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); - mgmol->setupConstraintsFromInput(constraints_filename); + mgmol->setup(); - mgmol_setup(); - - if (!tcheck) - { - mgmol->setup(); - - if (ct.isLocMode()) - readRestartFiles(mgmol); - else - readRestartFiles(mgmol); - } + if (ct.isLocMode()) + readRestartFiles(mgmol); else - { - *MPIdata::sout << " Input parameters OK\n"; - } + readRestartFiles(mgmol); + delete mgmol; } // close main scope diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 4594dd3a..1788281d 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -40,7 +40,7 @@ void readRestartFiles(MGmolInterface *mgmol_) /* Read the first snapshot to determin dimension and number of snapshots */ filename = string_format(rom_options.restart_file_fmt, minidx); - orbitals = mgmol->loadOrbitalFromRestartFile(filename); + mgmol->loadRestartFile(filename); const int dim = orbitals->getLocNumpt(); const int chrom_num = orbitals->chromatic_number(); const int totalSamples = orbitals->chromatic_number() * num_restart; @@ -54,7 +54,7 @@ void readRestartFiles(MGmolInterface *mgmol_) for (int k = minidx; k <= maxidx; k++) { filename = string_format(rom_options.restart_file_fmt, k); - orbitals = mgmol->loadOrbitalFromRestartFile(filename); + mgmol->loadRestartFile(filename); assert(dim == orbitals->getLocNumpt()); assert(chrom_num == orbitals->chromatic_number()); diff --git a/src/runfire.cc b/src/runfire.cc index d71c52a8..3239e1b1 100644 --- a/src/runfire.cc +++ b/src/runfire.cc @@ -33,13 +33,12 @@ void MGmol::runfire(OrbitalsType** orbitals, Ions& ions) DFTsolver::resetItCount(); - orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( - ct.WFExtrapolation()); + orbitals_extrapol_.reset(OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation())); - fire.init(h5f_file_); + fire.init(h5f_file_.get()); - delete h5f_file_; - h5f_file_ = nullptr; + h5f_file_.reset(); // additional quench to compensate random start if (ct.restart_info < 3) @@ -129,7 +128,7 @@ void MGmol::runfire(OrbitalsType** orbitals, Ions& ions) } // end for steps - delete orbitals_extrapol_; + orbitals_extrapol_.reset(); // final dump if (ct.out_restart_info > 0) diff --git a/src/setup.cc b/src/setup.cc index 62d9f835..f022678c 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -28,7 +28,15 @@ int MGmol::setupFromInput(const std::string filename) MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - hamiltonian_ = new Hamiltonian(); + /* + * Setup global mesh for calculations + */ + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + hamiltonian_.reset(new Hamiltonian()); Potentials& pot = hamiltonian_->potential(); ct.registerPotentials(pot); @@ -43,8 +51,8 @@ int MGmol::setupFromInput(const std::string filename) const pb::PEenv& myPEenv = mymesh->peenv(); if (ct.restart_info > 0) - h5f_file_ - = new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type); + h5f_file_.reset( + new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type)); int status = readCoordinates(filename, false); if (status == -1) return -1; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4c68adb7..fe7b571b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -482,7 +482,7 @@ add_test(NAME ChebyshevMVP ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -if(NOT ${MAGMA_FOUND}) +if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} diff --git a/tests/SiH4/testSiH4.py b/tests/SiH4/testSiH4.py index 1a0965ac..c7f81bdc 100755 --- a/tests/SiH4/testSiH4.py +++ b/tests/SiH4/testSiH4.py @@ -44,16 +44,21 @@ lines=output.split(b'\n') tol = 5.e-3 +found_forces = False for line in lines: - num_matches = line.count(b'%%') - if num_matches: + if line.count(b'%%'): print(line) - num_matches = line.count(b'##') - if num_matches: + if line.count(b'##'): words=line.split() if len(words)==8: print(line) + found_forces = True for i in range(5,8): if abs(eval(words[i]))>tol: sys.exit(1) +if (not found_forces): + print("no forces found") + sys.exit(1) + +sys.exit(0) diff --git a/util/xyz2in.py b/util/xyz2in.py index 3555e138..bb2e022d 100644 --- a/util/xyz2in.py +++ b/util/xyz2in.py @@ -5,9 +5,11 @@ # This file is part of MGmol. For details, see https://github.com/llnl/mgmol. # Please also read this link https://github.com/llnl/mgmol/LICENSE # -# Python program to convert .xyz file into mgmol input +# Python program to convert .xyz file into mgmol input coordinates file +# Optional arguments: [lx ly lz] to define box size and map all atoms into +# box (0,0,0)-(lx,ly,lz) using periodic boundary conditions # -# use: python coords.xyz > coords.in +# use: python coords.xyz [lx ly lz] > coords.in #------------------------------------------------------------------------------- import sys, string @@ -15,6 +17,16 @@ #read file ifile=open(sys.argv[1],'r') + +lx = 0. +ly = 0. +lz = 0. +if( len(sys.argv) > 2 ): + lx = eval(sys.argv[2]) + ly = eval(sys.argv[3]) + lz = eval(sys.argv[4]) + + lines=ifile.readlines() count=0 @@ -22,13 +34,31 @@ dummy=0 #unused flag set to 0 for line in lines: ## loop over lines of file - words=line.split() - if len(words)>1: - if words[0][0:1]!='#': - name=words[0]+str(count) + if count>1: + words=line.split() + if len(words)>1: + name=words[0]+str(count-2) x=eval(words[1])*ang2bohr y=eval(words[2])*ang2bohr z=eval(words[3])*ang2bohr - + + if lx > 0.: + if x<0: + x = x +lx + if x>lx: + x = x -lx + + if ly > 0.: + if y<0: + y = y +ly + if y>ly: + y = y -ly + + if lz > 0.: + if z<0: + z = z +lz + if z>lz: + z = z -lz + print(name,'\t',dummy,'\t',x,'\t',y,'\t',z,'\t',movable) - count=count+1 + count=count+1 From 63587f6cd5080a99286b757471ae4b1fa95bc54f Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 17 Jun 2024 09:44:09 -0700 Subject: [PATCH 12/50] Snapshot collection/POD training for potential (#254) * offline workflow with potential POD training * update rom main driver again * ci workflow include librom in compilation * add config option --- .github/workflows/ci.yml | 2 +- src/Control.cc | 13 ++++++++++ src/read_config.cc | 6 +++-- src/rom_Control.h | 8 ++++++ src/rom_workflows.cc | 55 ++++++++++++++++++++++++++++++++-------- src/rom_workflows.h | 1 + 6 files changed, 71 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05942636..beec14ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM - name: make run: | cd build && make -j 4 diff --git a/src/Control.cc b/src/Control.cc index ce3023e3..ae8caf4e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2077,6 +2077,14 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + str = vm["ROM.offline.variable"].as(); + if (str.compare("orbitals") == 0) + rom_pri_option.variable = ROMVariable::ORBITALS; + else if (str.compare("potential") == 0) + rom_pri_option.variable = ROMVariable::POTENTIAL; + else + rom_pri_option.variable = ROMVariable::NONE; + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); } // onpe0 @@ -2116,5 +2124,10 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); bcast_check(mpirc); + short rom_var = (short)static_cast(rom_pri_option.variable); + mpirc = MPI_Bcast(&rom_var, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + rom_pri_option.rom_stage = static_cast(rom_stage); + rom_pri_option.variable = static_cast(rom_var); } \ No newline at end of file diff --git a/src/read_config.cc b/src/read_config.cc index 74533dfb..159e8668 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -425,6 +425,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.basis_file", po::value()->default_value(""), "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), - "Save libROM snapshot file at FOM simulation."); + "Save libROM snapshot file at FOM simulation.") + ("ROM.offline.variable", po::value()->default_value(""), + "FOM variable to perform POD: either orbitals or potential."); } -#endif \ No newline at end of file +#endif diff --git a/src/rom_Control.h b/src/rom_Control.h index d1fd6854..753a2e68 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -25,6 +25,13 @@ enum class ROMStage UNSUPPORTED }; +enum class ROMVariable +{ + ORBITALS, + POTENTIAL, + NONE +}; + /* Stored as a private member variable of Control class */ struct ROMPrivateOptions { @@ -34,6 +41,7 @@ struct ROMPrivateOptions int restart_file_minidx = -1; int restart_file_maxidx = -1; std::string basis_file = ""; + ROMVariable variable=ROMVariable::NONE; /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 1788281d..b73f1898 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -27,7 +27,14 @@ template void readRestartFiles(MGmolInterface *mgmol_) { Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + + /* number of restart files, start/end indices */ assert(rom_options.restart_file_minidx >= 0); assert(rom_options.restart_file_maxidx >= 0); const int minidx = rom_options.restart_file_minidx; @@ -35,20 +42,37 @@ void readRestartFiles(MGmolInterface *mgmol_) const int num_restart = maxidx - minidx + 1; MGmol *mgmol = static_cast *>(mgmol_); - OrbitalsType *orbitals = nullptr; + OrbitalsType *orbitals = mgmol->getOrbitals(); + Potentials& pot = mgmol->getHamiltonian()->potential(); std::string filename; - /* Read the first snapshot to determin dimension and number of snapshots */ - filename = string_format(rom_options.restart_file_fmt, minidx); - mgmol->loadRestartFile(filename); - const int dim = orbitals->getLocNumpt(); + /* Determine basis prefix, dimension, and sample size */ + std::string basis_prefix = rom_options.basis_file; + int dim; + int totalSamples = num_restart; const int chrom_num = orbitals->chromatic_number(); - const int totalSamples = orbitals->chromatic_number() * num_restart; - delete orbitals; + switch (rom_var) + { + case ROMVariable::ORBITALS: + basis_prefix += "_orbitals"; + dim = orbitals->getLocNumpt(); + /* if orbitals, each sample have chromatic number of wave functions */ + totalSamples *= orbitals->chromatic_number(); + break; + + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + dim = pot.size(); + break; + + default: + ct.global_exit(-1); + break; + } /* Initialize libROM classes */ CAROM::Options svd_options(dim, totalSamples, 1); - CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); /* Collect the restart files */ for (int k = minidx; k <= maxidx; k++) @@ -58,10 +82,19 @@ void readRestartFiles(MGmolInterface *mgmol_) assert(dim == orbitals->getLocNumpt()); assert(chrom_num == orbitals->chromatic_number()); - for (int i = 0; i < chrom_num; ++i) - basis_generator.takeSample(orbitals->getPsi(i)); + switch (rom_var) + { + case ROMVariable::ORBITALS: + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + break; - delete orbitals; + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + /* we save total potential for now */ + basis_generator.takeSample(pot.vtot()); + break; + } } basis_generator.writeSnapshot(); basis_generator.endSamples(); diff --git a/src/rom_workflows.h b/src/rom_workflows.h index dfb22b79..d45abb92 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -13,6 +13,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" +#include "Potentials.h" #include "MGmol.h" #include "MGmol_MPI.h" #include "MPIdata.h" From 58b9abe293a082893d7180c4f7826dcf1dc7ab51 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Fri, 19 Jul 2024 11:02:53 -0700 Subject: [PATCH 13/50] Poisson rom (#257) * rom workflow for buildROMPoissonOperator * change from total to hartree potential * test routine for Poisson ROM operator. * POD reconstruction * test Poisson ROM operator verified. * buildROMPoissonOperator updated. * ci test for Poisson ROM operator --- .github/workflows/ci.yml | 14 +- src/Control.cc | 7 + src/Electrostatic.h | 2 + src/Hartree_CG.h | 7 + src/PCGSolver.h | 2 + src/Poisson.h | 6 + src/read_config.cc | 4 +- src/rom_Control.h | 4 + src/rom_main.cc | 38 ++- src/rom_workflows.cc | 341 ++++++++++++++++++++++++- src/rom_workflows.h | 7 + tests/ROM/test_rom_poisson/carbyne.cfg | 63 +++++ tests/ROM/test_rom_poisson/carbyne.in | 14 + 13 files changed, 497 insertions(+), 12 deletions(-) create mode 100644 tests/ROM/test_rom_poisson/carbyne.cfg create mode 100644 tests/ROM/test_rom_poisson/carbyne.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beec14ef..5e94113f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,15 +35,21 @@ jobs: submodules: 'true' - name: cmake run: | - mkdir build - cd build + mkdir ${GITHUB_WORKSPACE}/build + cd ${GITHUB_WORKSPACE}/build cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM - name: make run: | - cd build && make -j 4 + cd ${GITHUB_WORKSPACE}/build && make -j 4 - name: test run: | - cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + cd ${GITHUB_WORKSPACE}/build && ctest --no-compress-output -V -T Test -I 1,20,1 + - name: test ROM Poisson operator + run: | + cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson + ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom . + ln -s ${GITHUB_WORKSPACE}/potentials/* . + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.cfg -i carbyne.in # code-style: # runs-on: ubuntu-latest # needs: [docker-image] diff --git a/src/Control.cc b/src/Control.cc index ae8caf4e..d1c4c672 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2069,6 +2069,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::ONLINE; else if (str.compare("build") == 0) rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("test_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_POISSON; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; @@ -2086,6 +2088,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.variable = ROMVariable::NONE; rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + + rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 // synchronize all processors @@ -2130,4 +2134,7 @@ void Control::syncROMOptions() rom_pri_option.rom_stage = static_cast(rom_stage); rom_pri_option.variable = static_cast(rom_var); + + mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); } \ No newline at end of file diff --git a/src/Electrostatic.h b/src/Electrostatic.h index a3b86879..785bc909 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -46,6 +46,8 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + Poisson* getPoissonSolver() { return poisson_solver_; } + void setup(const short max_sweeps); void setupPB(const double e0, const double rho0, const double drho0, Potentials& pot); diff --git a/src/Hartree_CG.h b/src/Hartree_CG.h index 78e9da65..efa50919 100644 --- a/src/Hartree_CG.h +++ b/src/Hartree_CG.h @@ -40,6 +40,13 @@ class Hartree_CG : public Poisson void solve(const pb::GridFunc& rho, const pb::GridFunc& rhoc) override; + + void applyOperator(pb::GridFunc &vh, + pb::GridFunc &lhs) override + { + T *oper = poisson_solver_->getOperator(); + oper->apply(vh, lhs); + } }; #endif diff --git a/src/PCGSolver.h b/src/PCGSolver.h index 8ba76158..d38787f1 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -97,6 +97,8 @@ class PCGSolver double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } + T* getOperator() { return &oper_; } + // Destructor ~PCGSolver() { clear(); } }; diff --git a/src/Poisson.h b/src/Poisson.h index 87107542..14716cda 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -88,6 +88,12 @@ class Poisson : public PoissonInterface Int_vhrho_ = vel * vh_->gdot(rho); Int_vhrhoc_ = vel * vh_->gdot(rhoc); } + + virtual void applyOperator(pb::GridFunc &vh, pb::GridFunc &lhs) + { + std::cerr << "ERROR: Abstract method Poisson::applyOperator()" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } }; #endif diff --git a/src/read_config.cc b/src/read_config.cc index 159e8668..14057553 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -427,6 +427,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), - "FOM variable to perform POD: either orbitals or potential."); + "FOM variable to perform POD: either orbitals or potential.") + ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), + "Number of potential POD basis to build Hartree potential ROM operator."); } #endif diff --git a/src/rom_Control.h b/src/rom_Control.h index 753a2e68..2e4a1c71 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -22,6 +22,7 @@ enum class ROMStage ONLINE, RESTORE, // TODO(kevin): what stage is this? BUILD, + TEST_POISSON, UNSUPPORTED }; @@ -45,6 +46,9 @@ struct ROMPrivateOptions /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; + + /* options for ROM building */ + int num_potbasis = -1; }; #endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc index 3718078f..c54bcd37 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -86,6 +86,10 @@ int main(int argc, char** argv) mmpi.bcastGlobal(input_filename); mmpi.bcastGlobal(lrs_filename); + /* Get ROM driver mode */ + ROMPrivateOptions rom_options = ct.getROMOptions(); + ROMStage rom_stage = rom_options.rom_stage; + // Enter main scope { MGmolInterface* mgmol; @@ -98,10 +102,36 @@ int main(int argc, char** argv) mgmol->setup(); - if (ct.isLocMode()) - readRestartFiles(mgmol); - else - readRestartFiles(mgmol); + switch (rom_stage) + { + case (ROMStage::OFFLINE): + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + break; + + case (ROMStage::BUILD): + if (ct.isLocMode()) + buildROMPoissonOperator(mgmol); + else + buildROMPoissonOperator(mgmol); + break; + + case (ROMStage::TEST_POISSON): + if (ct.isLocMode()) + testROMPoissonOperator(mgmol); + else + testROMPoissonOperator(mgmol); + break; + + default: + std::cerr << "rom_main error: Unknown ROM stage" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + break; + } + + delete mgmol; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index b73f1898..d9629c07 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -8,6 +8,7 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "rom_workflows.h" +#include "Electrostatic.h" #include #include #include @@ -91,8 +92,8 @@ void readRestartFiles(MGmolInterface *mgmol_) case ROMVariable::POTENTIAL: basis_prefix += "_potential"; - /* we save total potential for now */ - basis_generator.takeSample(pot.vtot()); + /* we save hartree potential */ + basis_generator.takeSample(pot.vh_rho()); break; } } @@ -100,5 +101,339 @@ void readRestartFiles(MGmolInterface *mgmol_) basis_generator.endSamples(); } +template +void buildROMPoissonOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "buildROMPoissonOperator error: ROM variable must be POTENTIAL to run this stage!\n" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix *pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* Load PoissonSolver pointer */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + + /* GridFunc initialization inputs */ + const pb::Grid &grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* Initialize ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + CAROM::Vector rom_col(num_pot_basis, false); + for (int c = 0; c < num_pot_basis; c++) + { + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector *col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + pot_basis->transposeMult(op_col, rom_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < num_pot_basis; r++) + pot_rom(r, c) = rom_col(r); + + delete col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* Save ROM operator */ + // write the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = "pot_rom_oper.h5"; + CAROM::HDFDatabase h5_helper; + h5_helper.create(rom_oper); + h5_helper.putInteger("number_of_potential_basis", num_pot_basis); + h5_helper.putDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save the inverse as well */ + pot_rom.inverse(); + h5_helper.putDoubleArray("potential_rom_inverse", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + h5_helper.close(); + } +} + +template +void testROMPoissonOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + printf("pot size: %d\n", dim); + + /* GridFunc initialization inputs */ + const pb::Grid &grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* fictitious snapshot numbers */ + const int nsnapshot = 3; + + /* Set compensating charges to zero now */ + pb::GridFunc rhoc(grid, bc[0], bc[1], bc[2]); + rhoc = 0.0; + + /* Generate fictitious right-hand sides and snapshots */ + std::vector> rhs(nsnapshot), fom_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + rhs[s].resize(dim); + for (int d = 0; d < dim; d++) + rhs[s][d] = ran0(); + + /* average out for periodic bc */ + pb::GridFunc rhs_gf(grid, bc[0], bc[1], bc[2]); + rhs_gf.assign(rhs[s].data(), 'd'); + double avg = rhs_gf.get_average(); + rhs_gf -= avg; + + /* copy back to rhs */ + rhs_gf.init_vect(rhs[s].data(), 'd'); + + poisson->solve(rhs_gf, rhoc); + + fom_sol[s].resize(dim); + poisson->vh().init_vect(fom_sol[s].data(), 'd'); + + /* check if the solution is correct */ + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc sol_gf(grid, bc[0], bc[1], bc[2]); + sol_gf.assign(fom_sol[s].data()); + /* apply Laplace operator */ + poisson->applyOperator(sol_gf, res); + /* FD operator scales rhs by 4pi */ + res.axpy(- 4. * M_PI, rhs_gf); + printf("FOM res norm: %.3e\n", res.norm2()); + } + + /* Initialize libROM classes */ + std::string basis_prefix = "test_poisson"; + CAROM::Options svd_options(dim, nsnapshot, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); + + /* Collect snapshots and train POD basis */ + for (int s = 0; s < nsnapshot; s++) + basis_generator.takeSample(fom_sol[s].data()); + basis_generator.endSamples(); + + /* Load POD basis. We use the maximum number of basis vectors. */ + const CAROM::Matrix *pot_basis = basis_generator.getSpatialBasis(); + + /* Check if full projection preserves FOM solution */ + for (int c = 0; c < nsnapshot; c++) + { + CAROM::Vector *fom_sol_vec = nullptr; + /* get librom view-vector of fom_sol */ + fom_sol_vec = new CAROM::Vector(fom_sol[c].data(), pot_basis->numRows(), true, false); + + CAROM::Vector *rom_proj = pot_basis->transposeMult(*fom_sol_vec); + CAROM::Vector *reconstruct = pot_basis->mult(*rom_proj); + + /* error on libROM side */ + CAROM::Vector *librom_error = reconstruct->minus(fom_sol_vec); + printf("librom reconstruction error: %.3e\n", librom_error->norm()); + + /* error on mgmol side */ + pb::GridFunc recon_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fom_gf(grid, bc[0], bc[1], bc[2]); + recon_gf.assign(reconstruct->getData(), 'd'); + fom_gf.assign(fom_sol[c].data(), 'd'); + recon_gf -= fom_gf; + printf("mgmol reconstruction error: %.3e\n", recon_gf.norm2()); + + delete fom_sol_vec; + delete rom_proj; + delete reconstruct; + delete librom_error; + } + + /* Check FOM axpy is equivalent to ROM axpy */ + for (int s = 0; s < nsnapshot; s++) + { + CAROM::Vector fom_res(pot_basis->numRows(), true); + CAROM::Vector rom_res(nsnapshot, false); + CAROM::Vector fom_rhs(pot_basis->numRows(), true); + CAROM::Vector rom_rhs(nsnapshot, false); + + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc mgmol_rhs(grid, bc[0], bc[1], bc[2]); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + mgmol_rhs.assign(rhs[s].data(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(fomsol_gf, res); + + /* get librom view-vector of fom_res */ + res.init_vect(fom_res.getData(), 'd'); + pot_basis->transposeMult(fom_res, rom_res); + + /* get librom view-vector of fom_rhs */ + mgmol_rhs.init_vect(fom_rhs.getData(), 'd'); + pot_basis->transposeMult(fom_rhs, rom_rhs); + + /* ROM residual: FD operator scales rhs by 4pi */ + rom_rhs *= 4. * M_PI; + rom_res -= rom_rhs; + printf("ROM res norm: %.3e\n", rom_res.norm()); + + /* FOM residual: FD operator scales rhs by 4pi */ + res.axpy(- 4. * M_PI, mgmol_rhs); + printf("FOM res norm: %.3e\n", res.norm2()); + + /* projection of the residual */ + res.init_vect(fom_res.getData(), 'd'); + CAROM::Vector *res_proj = pot_basis->transposeMult(fom_res); + printf("FOM res projection norm: %.3e\n", res_proj->norm()); + + delete res_proj; + } + + /* Initialize Projection ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(nsnapshot, nsnapshot, false); + + /* Build Projection of Poisson operator */ + for (int c = 0; c < nsnapshot; c++) + { + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector *col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + CAROM::Vector *rom_col = pot_basis->transposeMult(op_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < nsnapshot; r++) + pot_rom(r, c) = (*rom_col)(r); + + delete col; + delete rom_col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* Inverse of the projection ROM matrix */ + CAROM::Matrix pot_rom_inv(pot_rom); + pot_rom_inv.inverse(); + + /* Check the inverse */ + CAROM::Matrix *identity = pot_rom_inv.mult(pot_rom); + printf("pot_rom_inv * pot_rom = identity\n"); + for (int i = 0; i < nsnapshot; i++) + { + for (int j = 0; j < nsnapshot; j++) + printf("%.3e\t", identity->item(i, j)); + printf("\n"); + } + delete identity; + + /* Test with sample RHS. ROM must be able to 100% reproduce the FOM solution. */ + std::vector rom_sol(0), rom_rhs(0); + std::vector> test_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + /* get librom view-vector of rhs[s] */ + CAROM::Vector fom_rhs(rhs[s].data(), dim, true, false); + + /* project onto POD basis */ + rom_rhs.push_back(pot_basis->transposeMult(fom_rhs)); + + /* FOM FD operator scales rhs by 4pi */ + *rom_rhs.back() *= 4. * M_PI; + + /* solve ROM */ + rom_sol.push_back(pot_rom_inv.mult(*rom_rhs.back())); + + /* check ROM solution */ + CAROM::Vector &res(*pot_rom.mult(*rom_sol.back())); + res -= *rom_rhs.back(); + printf("rom res norm: %.3e\n", res.norm()); + + /* initialize lift-up FOM solution */ + test_sol[s].resize(dim); + /* get librom view-vector of test_sol[s] */ + CAROM::Vector test_sol_vec(test_sol[s].data(), dim, true, false); + pot_basis->mult(*rom_sol.back(), test_sol_vec); + } + + /* Compute relative errors */ + for (int s = 0; s < nsnapshot; s++) + { + pb::GridFunc testsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + + testsol_gf.assign(test_sol[s].data(), 'd'); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + + testsol_gf -= fomsol_gf; + double rel_error = testsol_gf.norm2() / fomsol_gf.norm2(); + printf("%d-th sample relative error: %.3e\n", s, rel_error); + + if (rel_error > 1.0e-9) + abort(); + } + + /* clean up pointers */ + for (int s = 0; s < nsnapshot; s++) + { + delete rom_sol[s]; + delete rom_rhs[s]; + } +} + template void readRestartFiles(MGmolInterface *mgmol_); -template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file +template void readRestartFiles(MGmolInterface *mgmol_); + +template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template void buildROMPoissonOperator(MGmolInterface *mgmol_); + +template void testROMPoissonOperator(MGmolInterface *mgmol_); +template void testROMPoissonOperator(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h index d45abb92..2399d065 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -35,8 +35,15 @@ namespace po = boost::program_options; #include "librom.h" +#include "utils/HDFDatabase.h" template void readRestartFiles(MGmolInterface *mgmol_); +template +void buildROMPoissonOperator(MGmolInterface *mgmol_); + +template +void testROMPoissonOperator(MGmolInterface *mgmol_); + #endif // ROM_WORKFLOWS_H diff --git a/tests/ROM/test_rom_poisson/carbyne.cfg b/tests/ROM/test_rom_poisson/carbyne.cfg new file mode 100644 index 00000000..6aac926c --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_poisson + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/ROM/test_rom_poisson/carbyne.in b/tests/ROM/test_rom_poisson/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 From 75b41753a092281eecec38468f93064e47c01d4e Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Wed, 14 Aug 2024 04:43:14 -0700 Subject: [PATCH 14/50] ROM Pinned H2O (#271) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Kevin" Seung Whan Chung --- examples/2-pyridone/2-pyridone.cfg | 26 +- examples/2-pyridone/README | 10 +- examples/2-pyridone/coords.in | 25 +- examples/Li2GTH/README | 11 + examples/Li2GTH/davidson.cfg | 46 + examples/Li2GTH/li2.xyz | 4 + examples/PinnedH2O/README | 3 + examples/PinnedH2O/coords.in | 3 + examples/PinnedH2O/mgmol.cfg | 35 + potentials/pseudo.Li_GTH_PBE | 332 +++ potentials/pseudo.N_ONCV_PBE_SG15 | 1863 +++++++++++++++++ src/Control.cc | 36 - src/Control.h | 1 - src/DavidsonSolver.cc | 12 +- src/DensityMatrix.cc | 4 +- src/Hartree_CG.cc | 4 +- src/IonicAlgorithm.cc | 2 +- src/LBFGS.cc | 2 +- src/MGmol.cc | 53 +- src/MGmol.h | 22 +- src/MGmolInterface.h | 7 + src/MVPSolver.cc | 9 +- src/Orbitals.h | 2 + src/ProjectedMatrices.h | 6 + src/ProjectedMatricesInterface.h | 2 +- src/lbfgsrlx.cc | 12 +- src/md.cc | 37 +- src/quench.cc | 24 +- tests/CMakeLists.txt | 21 + tests/EnergyAndForces/coords.in | 2 + tests/EnergyAndForces/lrs.in | 6 + tests/EnergyAndForces/mgmol.cfg | 33 + tests/EnergyAndForces/test.py | 87 + tests/EnergyAndForces/testEnergyAndForces.cc | 198 ++ tests/MVP/test.py | 24 + tests/WFEnergyAndForces/mgmol.cfg | 28 + tests/WFEnergyAndForces/sih4.xyz | 8 + tests/WFEnergyAndForces/test.py | 91 + .../testWFEnergyAndForces.cc | 210 ++ util/generateLiLocalGTHpseudo.py | 65 + 40 files changed, 3221 insertions(+), 145 deletions(-) create mode 100644 examples/Li2GTH/README create mode 100644 examples/Li2GTH/davidson.cfg create mode 100644 examples/Li2GTH/li2.xyz create mode 100644 examples/PinnedH2O/README create mode 100644 examples/PinnedH2O/coords.in create mode 100644 examples/PinnedH2O/mgmol.cfg create mode 100644 potentials/pseudo.Li_GTH_PBE create mode 100644 potentials/pseudo.N_ONCV_PBE_SG15 create mode 100644 tests/EnergyAndForces/coords.in create mode 100644 tests/EnergyAndForces/lrs.in create mode 100644 tests/EnergyAndForces/mgmol.cfg create mode 100755 tests/EnergyAndForces/test.py create mode 100644 tests/EnergyAndForces/testEnergyAndForces.cc create mode 100644 tests/WFEnergyAndForces/mgmol.cfg create mode 100644 tests/WFEnergyAndForces/sih4.xyz create mode 100755 tests/WFEnergyAndForces/test.py create mode 100644 tests/WFEnergyAndForces/testWFEnergyAndForces.cc create mode 100644 util/generateLiLocalGTHpseudo.py diff --git a/examples/2-pyridone/2-pyridone.cfg b/examples/2-pyridone/2-pyridone.cfg index 82457d9b..23f53496 100644 --- a/examples/2-pyridone/2-pyridone.cfg +++ b/examples/2-pyridone/2-pyridone.cfg @@ -1,5 +1,6 @@ verbosity=1 xcFunctional=PBE +FDtype=4th [Mesh] nx=128 ny=128 @@ -12,30 +13,17 @@ lx=24. ly=24. lz=12. [Potentials] -pseudopotential=pseudo.N_soft_pbe -pseudopotential=pseudo.O_pbe -pseudopotential=pseudo.C_pbe -pseudopotential=pseudo.H_pbe -[Poisson] -solver=MG -max_steps_initial=10 -max_steps=10 -bcx=periodic -bcy=periodic -bcz=periodic +pseudopotential=pseudo.N_ONCV_PBE_SG15 +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=QUENCH [Quench] -step_length=1.5 max_steps=200 atol=1.e-7 -num_lin_iterations=2 -ortho_freq=20 -interval_print_residual=10 [Orbitals] initial_type=Gaussian initial_width=1.5 -[ProjectedMatrices] -solver=exact -[Restart] -output_type=single_file +[Poisson] +FDtype=4th diff --git a/examples/2-pyridone/README b/examples/2-pyridone/README index 351b7012..663f5855 100644 --- a/examples/2-pyridone/README +++ b/examples/2-pyridone/README @@ -1,6 +1,6 @@ -ln -s ../../potentials/pseudo.N_soft_pbe -ln -s ../../potentials/pseudo.O_pbe -ln -s ../../potentials/pseudo.C_pbe -ln -s ../../potentials/pseudo.H_pbe +ln -s ../../potentials/pseudo.N_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.O_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.C_ONCV_PBE_SG15 +ln -s ../../potentials/pseudo.H_ONCV_PBE_SG15 -srun -ppdebug -n32 /nfs/tmp2/jeanluc/SVN/MGmol/mgmol/branches/bin/mgmol-pel -c 2-pyridone.cfg -l lrs.in -i coords.in +srun -ppdebug -n32 mgmol-opt -c 2-pyridone.cfg -l lrs.in -i coords.in diff --git a/examples/2-pyridone/coords.in b/examples/2-pyridone/coords.in index 911b5cdf..390ff902 100644 --- a/examples/2-pyridone/coords.in +++ b/examples/2-pyridone/coords.in @@ -1,13 +1,12 @@ -N1 1 10.0818 7.9149 6. 1 -O1 2 14.2787 7.9126 6. 1 -C1 3 7.9625 9.1867 6. 0 -C2 3 7.8388 11.8070 6. 0 -C3 3 10.1050 13.1610 6. 0 -C4 3 12.3830 11.8680 6. 0 -C5 3 12.2810 9.2255 6. 0 -H1 4 6.2219 8.0878 6. 1 -H2 4 6.0028 12.7142 6. 1 -H3 4 10.0754 15.2192 6. 1 -H4 4 14.2295 12.7482 6. 1 -H5 4 10.2332 6.0038 6. 1 - +N1 1 10.0818 7.9149 6. +O1 2 14.2787 7.9126 6. +C1 3 7.9625 9.1867 6. +C2 3 7.8388 11.8070 6. +C3 3 10.1050 13.1610 6. +C4 3 12.3830 11.8680 6. +C5 3 12.2810 9.2255 6. +H1 4 6.2219 8.0878 6. +H2 4 6.0028 12.7142 6. +H3 4 10.0754 15.2192 6. +H4 4 14.2295 12.7482 6. +H5 4 10.2332 6.0038 6. diff --git a/examples/Li2GTH/README b/examples/Li2GTH/README new file mode 100644 index 00000000..f411a01e --- /dev/null +++ b/examples/Li2GTH/README @@ -0,0 +1,11 @@ +#Run MGmol +mpirun -np 4 mgmol-opt -c davidson.cfg -i li2.xyz > davidson.out + +#extract visit files from HDF5 file +python read_hdf5.py -bov li2.h5 Vtotal + +python read_hdf5.py -bov li2.h50 Function0002 + +Note: the eigenfunctions corresponding to the 3 occupied states are stored +in functions 2, 3 and 4 because they are the result of diagonalizing the DM +in Davidson algorithm, not H. diff --git a/examples/Li2GTH/davidson.cfg b/examples/Li2GTH/davidson.cfg new file mode 100644 index 00000000..d211b812 --- /dev/null +++ b/examples/Li2GTH/davidson.cfg @@ -0,0 +1,46 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=96 +ny=96 +nz=96 +[Domain] +ox=-9. +oy=-9. +oz=-9. +lx=18. +ly=18. +lz=18. +[Potentials] +pseudopotential=pseudo.Li_GTH_PBE +[Poisson] +solver=CG +bcx=0 +bcy=0 +bcz=0 +[Run] +type=QUENCH +[Quench] +solver=Davidson +max_steps=200 +atol=1.e-8 +preconditioner_num_levels=3 +[Orbitals] +nempty=2 +initial_type=random +temperature=300. +bcx=0 +bcy=0 +bcz=0 +[ProjectedMatrices] +solver=exact +[DensityMatrix] +nb_inner_it=1 +mixing=1. +[Restart] +output_level=3 +output_filename=li2.h5 +output_type=single_file +[Coloring] +scope=global diff --git a/examples/Li2GTH/li2.xyz b/examples/Li2GTH/li2.xyz new file mode 100644 index 00000000..ccf3a3f1 --- /dev/null +++ b/examples/Li2GTH/li2.xyz @@ -0,0 +1,4 @@ +2 +Experimental geometry (Menconi and Tozer, Chem. Phys. Lett. 2002) +Li 1.3365 0.0 0.0 +Li -1.3365 0.0 0.0 diff --git a/examples/PinnedH2O/README b/examples/PinnedH2O/README new file mode 100644 index 00000000..12ca9138 --- /dev/null +++ b/examples/PinnedH2O/README @@ -0,0 +1,3 @@ +Single water molecule MD, with Oxigen atom pinned to (0,0,0) coordinate: + +mpirun -np 4 mgmol-opt -c mgmol.cfg -i coords.in diff --git a/examples/PinnedH2O/coords.in b/examples/PinnedH2O/coords.in new file mode 100644 index 00000000..307e170c --- /dev/null +++ b/examples/PinnedH2O/coords.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.42 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/examples/PinnedH2O/mgmol.cfg b/examples/PinnedH2O/mgmol.cfg new file mode 100644 index 00000000..32325e47 --- /dev/null +++ b/examples/PinnedH2O/mgmol.cfg @@ -0,0 +1,35 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=50 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=0 diff --git a/potentials/pseudo.Li_GTH_PBE b/potentials/pseudo.Li_GTH_PBE new file mode 100644 index 00000000..7da944e0 --- /dev/null +++ b/potentials/pseudo.Li_GTH_PBE @@ -0,0 +1,332 @@ +# Goedecker, Teter, Hutter, Phys. rev. B 54 (3), 1996 +# Parameters from M. Krack, Theor. Chem. Acc. 114, 2005 +# Short description of the species type. One line only! +LithiumGTH_PBE +# +White +#radii of balls and covalent bonds +0.4 1.0 +# Nlcc flag +0 +# Atomic number +3 +# Atomic mass +3.0 +# Number of valence electrons +3.0 +# Gaussian core charge parameter rc +1. +# Number of potentials +1 +# l-value for state which is local +0 0 +# Local potential radius +3. +# Non-local potential radius +3. +# number of points in radial grid +301 +# log mesh parameter +0. +# radial grid, reference state, and potential for l=0 +0.0 -20.06528920602149 +0.01 -20.054252432503986 +0.02 -20.021181949571293 +0.03 -19.966197051499293 +0.04 -19.889495835016746 +0.05 -19.791354114111485 +0.06 -19.672123911517346 +0.07 -19.53223153752974 +0.08 -19.372175269684018 +0.09 -19.192522649601603 +0.1 -18.993907415941983 +0.11 -18.777026094871008 +0.12 -18.54263427174855 +0.13 -18.291542569831762 +0.14 -18.02461236366824 +0.15 -17.742751256501393 +0.16 -17.446908352415903 +0.17 -17.138069355105003 +0.18 -16.81725152603506 +0.19 -16.485498535412276 +0.2 -16.14387523971814 +0.21 -15.7934624196748 +0.22 -15.435351512330925 +0.23 -15.07063937052802 +0.24 -14.700423082322942 +0.25 -14.325794882014955 +0.26 -13.947837183265412 +0.27 -13.567617763419314 +0.28 -13.186185126555326 +0.29 -12.80456407102134 +0.3 -12.423751485274781 +0.31 -12.0447123937598 +0.32 -11.668376272337989 +0.33 -11.295633650466765 +0.34 -10.927333014911554 +0.35 -10.564278027307214 +0.36 -10.207225065372501 +0.37 -9.856881095051628 +0.38 -9.513901878330472 +0.39 -9.178890518973706 +0.4 -8.852396345973204 +0.41 -8.534914132107685 +0.42 -8.226883642706916 +0.43 -7.928689507508834 +0.44 -7.64066140641021 +0.45 -7.363074557955612 +0.46 -7.096150497598659 +0.47 -6.840058131114424 +0.48 -6.594915047052536 +0.49 -6.3607890708040955 +0.5 -6.1377000417180545 +0.51 -5.925621793748402 +0.52 -5.724484319343899 +0.53 -5.534176095708496 +0.54 -5.354546552160849 +0.55 -5.1854086571031885 +0.56 -5.026541603068161 +0.57 -4.877693568441651 +0.58 -4.73858453475222 +0.59 -4.60890913886544 +0.6 -4.488339540014026 +0.61 -4.376528282322026 +0.62 -4.273111134331415 +0.63 -4.17770988800058 +0.64 -4.089935100703225 +0.65 -4.009388764900398 +0.66 -3.935666891373983 +0.67 -3.868361993183916 +0.68 -3.807065458829906 +0.69 -3.751369804448444 +0.7 -3.7008707962442955 +0.71 -3.655169435730016 +0.72 -3.6138738017150143 +0.73 -3.5766007443361536 +0.74 -3.5429774277437494 +0.75 -3.512642719340314 +0.76 -3.4852484247051203 +0.77 -3.4604603685176145 +0.78 -3.4379593229091037 +0.79 -3.4174417857190456 +0.8 -3.3986206121036857 +0.81 -3.38122550383682 +0.82 -3.3650033614510035 +0.83 -3.3497185050904883 +0.84 -3.335152770582715 +0.85 -3.3211054877824484 +0.86 -3.307393348702202 +0.87 -3.293850173314738 +0.88 -3.2803265812005225 +0.89 -3.2666895774169027 +0.9 -3.2528220610898493 +0.91 -3.2386222652767795 +0.92 -3.224003136624447 +0.93 -3.208891663253622 +0.94 -3.1932281591472704 +0.95 -3.1769655131062664 +0.96 -3.160068410071886 +0.97 -3.142512532302873 +0.98 -3.1242837475424827 +0.99 -3.10537729092335 +1.0 -3.085796946940759 +1.01 -3.065554237383791 +1.02 -3.0446676206540504 +1.03 -3.0231617074287267 +1.04 -3.001066497143526 +1.05 -2.978416639286402 +1.06 -2.9552507230094722 +1.07 -2.931610598088238 +1.08 -2.9075407297881393 +1.09 -2.8830875897419252 +1.1 -2.8582990845006977 +1.11 -2.833224022999286 +1.12 -2.8079116237754826 +1.13 -2.7824110624044724 +1.14 -2.7567710592563195 +1.15 -2.7310395073568614 +1.16 -2.705263139831839 +1.17 -2.6794872361411723 +1.18 -2.653755366065358 +1.19 -2.628109170188997 +1.2 -2.60258817543732 +1.21 -2.57722964405972 +1.22 -2.552068454319101 +1.23 -2.527137011036365 +1.24 -2.50246518405455 +1.25 -2.4780802726257742 +1.26 -2.4540069936848927 +1.27 -2.430267491955173 +1.28 -2.4068813698318525 +1.29 -2.383865735007489 +1.3 -2.361235263837057 +1.31 -2.339002278488971 +1.32 -2.31717683598915 +1.33 -2.2957668273371157 +1.34 -2.274778084954434 +1.35 -2.2542144968149556 +1.36 -2.2340781257018256 +1.37 -2.2143693321366866 +1.38 -2.1950868996304913 +1.39 -2.176228161011706 +1.4 -2.1577891246951135 +1.41 -2.1397645998619574 +1.42 -2.1221483196287054 +1.43 -2.1049330613864585 +1.44 -2.0881107635950986 +1.45 -2.07167263841506 +1.46 -2.055609279654423 +1.47 -2.03991076559945 +1.48 -2.0245667563822187 +1.49 -2.0095665856193814 +1.5 -1.9948993461310054 +1.51 -1.9805539696177794 +1.52 -1.96651930023847 +1.53 -1.95278416208739 +1.54 -1.9393374206237712 +1.55 -1.9261680381514323 +1.56 -1.9132651234880866 +1.57 -1.900617975999247 +1.58 -1.8882161242021371 +1.59 -1.876049359170528 +1.6 -1.8641077629922906 +1.61 -1.8523817325479044 +1.62 -1.8408619988905475 +1.63 -1.8295396425169552 +1.64 -1.8184061048233324 +1.65 -1.807453196042526 +1.66 -1.796673099957719 +1.67 -1.786058375684427 +1.68 -1.7756019568068189 +1.69 -1.7652971481466952 +1.7 -1.7551376204340732 +1.71 -1.7451174031375227 +1.72 -1.7352308757004553 +1.73 -1.7254727574166897 +1.74 -1.7158380961650346 +1.75 -1.7063222562085625 +1.76 -1.696920905249867 +1.77 -1.6876300009190652 +1.78 -1.6784457768568064 +1.79 -1.6693647285401818 +1.8 -1.6603835989853395 +1.81 -1.6514993644468863 +1.82 -1.6427092202208944 +1.83 -1.6340105666456095 +1.84 -1.625400995381811 +1.85 -1.6168782760432898 +1.86 -1.608440343237089 +1.87 -1.6000852840630355 +1.88 -1.5918113261126972 +1.89 -1.5836168259992183 +1.9 -1.5755002584415392 +1.91 -1.567460205919256 +1.92 -1.5594953489078422 +1.93 -1.5516044566980827 +1.94 -1.54378637879836 +1.95 -1.5360400369138487 +1.96 -1.5283644174926794 +1.97 -1.5207585648257078 +1.98 -1.513221574683613 +1.99 -1.5057525884726553 +2.0 -1.4983507878884341 +2.01 -1.4910153900454863 +2.02 -1.4837456430593685 +2.03 -1.476540822057088 +2.04 -1.4694002255912106 +2.05 -1.4623231724327952 +2.06 -1.4553089987182684 +2.07 -1.4483570554256544 +2.08 -1.441466706155941 +2.09 -1.4346373251959794 +2.1 -1.4278682958400157 +2.11 -1.4211590089477848 +2.12 -1.4145088617179962 +2.13 -1.407917256657042 +2.14 -1.4013836007237732 +2.15 -1.3949073046322673 +2.16 -1.38848778229559 +2.17 -1.3821244503946528 +2.18 -1.3758167280573612 +2.19 -1.3695640366343098 +2.2 -1.3633657995583632 +2.21 -1.3572214422764575 +2.22 -1.3511303922429727 +2.23 -1.3450920789649687 +2.24 -1.3391059340904876 +2.25 -1.3331713915319947 +2.26 -1.3272878876178418 +2.27 -1.321454861265409 +2.28 -1.3156717541702987 +2.29 -1.3099380110066239 +2.3 -1.304253079634056 +2.31 -1.2986164113078702 +2.32 -1.2930274608887506 +2.33 -1.2874856870495992 +2.34 -1.2819905524770356 +2.35 -1.2765415240656612 +2.36 -1.271138073103527 +2.37 -1.2657796754475625 +2.38 -1.260465811688003 +2.39 -1.2551959673011142 +2.4 -1.2499696327897274 +2.41 -1.244786303811294 +2.42 -1.2396454812933442 +2.43 -1.2345466715363633 +2.44 -1.229489386304243 +2.45 -1.224473142902546 +2.46 -1.219497464244928 +2.47 -1.214561878908113 +2.48 -1.2096659211758896 +2.49 -1.204809131072622 +2.5 -1.1999910543868184 +2.51 -1.1952112426853008 +2.52 -1.1904692533185557 +2.53 -1.1857646494178307 +2.54 -1.1810969998845513 +2.55 -1.1764658793726301 +2.56 -1.1718708682642154 +2.57 -1.1673115526394273 +2.58 -1.1627875242406034 +2.59 -1.158298380431558 +2.6 -1.1538437241523383 +2.61 -1.149423163869939 +2.62 -1.1450363135254051 +2.63 -1.1406827924777474 +2.64 -1.1363622254450418 +2.65 -1.1320742424430907 +2.66 -1.1278184787219725 +2.67 -1.1235945747008054 +2.68 -1.1194021759010047 +2.69 -1.1152409328783188 +2.7 -1.1111105011538749 +2.71 -1.107010541144477 +2.72 -1.1029407180923525 +2.73 -1.0989007019945478 +2.74 -1.094890167532131 +2.75 -1.0909087939993736 +2.76 -1.0869562652330373 +2.77 -1.0830322695418986 +2.78 -1.079136499636622 +2.79 -1.0752686525600812 +2.8 -1.0714284296182166 +2.81 -1.067615536311507 +2.82 -1.0638296822671227 +2.83 -1.0600705811718185 +2.84 -1.0563379507056208 +2.85 -1.0526315124763437 +2.86 -1.0489509919549826 +2.87 -1.0452961184120013 +2.88 -1.0416666248545523 +2.89 -1.0380622479646329 +2.9 -1.0344827280382114 +2.91 -1.0309278089253149 +2.92 -1.0273972379711012 +2.93 -1.0238907659579113 +2.94 -1.0204081470483046 +2.95 -1.0169491387290766 +2.96 -1.013513501756257 +2.97 -1.0101010001010782 +2.98 -1.0067114008969125 +2.99 -1.0033444743871645 +3.0 -0.9999999938741118 diff --git a/potentials/pseudo.N_ONCV_PBE_SG15 b/potentials/pseudo.N_ONCV_PBE_SG15 new file mode 100644 index 00000000..958c0c8b --- /dev/null +++ b/potentials/pseudo.N_ONCV_PBE_SG15 @@ -0,0 +1,1863 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# as distributed by Mat-Sim Research at www.mat-simresearch.com. +# Reference: +# D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# DOI:https://doi.org/10.1103/PhysRevB.88.085117 +# +# Pseudopotential generation was done by M. Schlipf and F. Gygi using +# the input parameters obtained from their optimization algorithm. +# Reference: +# M. Schlipf and F. Gygi, Computer Physics Communications (2015) +# DOI: 10.1016/j.cpc.2015.05.011 +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# http://www.quantum-simulation.org +# License: +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# Format: +# The format of this file has been adapted to match MGmol pseudopotential +# parser using the script convertONCVPSP_SG15.py +# +N_ONCV_PBE-1 +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +7 +# Atomic mass +14.00699997 +# Number of valence electrons +5 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +3 +# l-value for state which is local, then type of potential format +2 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 0.7070940539E+01 0.9781080649E+00 +1 2 -0.4781783452E+01 -0.1328201236E+01 +# l= 0 +0.0 -8.3833937254 3.4232195615 +0.01 -8.3799978758 3.4202071705 +0.02 -8.3697860653 3.4111530231 +0.03 -8.3527825341 3.3960738696 +0.04 -8.3290115228 3.3749864603 +0.05 -8.2985069887 3.3479144681 +0.06 -8.261312626 3.3148887067 +0.07 -8.2174818893 3.2759474053 +0.08 -8.1670780196 3.231136536 +0.09 -8.1101740718 3.1805101875 +0.1 -8.0468529415 3.1241309807 +0.11 -7.9772073885 3.0620705184 +0.12 -7.9013400567 2.9944098623 +0.13 -7.8193634857 2.9212400287 +0.14 -7.7314001139 2.8426624957 +0.15 -7.6375822685 2.7587897114 +0.16 -7.5380521411 2.6697455952 +0.17 -7.4329617463 2.5756660223 +0.18 -7.3224728588 2.4766992817 +0.19 -7.2067569298 2.3730065 +0.2 -7.0859949741 2.264762015 +0.21 -6.9603774349 2.1521537041 +0.22 -6.8301040119 2.0353832393 +0.23 -6.6953834613 1.9146662776 +0.24 -6.5564333561 1.7902325648 +0.25 -6.4134798193 1.6623259662 +0.26 -6.2667572033 1.5312043795 +0.27 -6.1165077488 1.3971395792 +0.28 -5.962981185 1.2604169264 +0.29 -5.8064342997 1.1213349889 +0.3 -5.647130465 0.98020504493 +0.31 -5.4853391206 0.83735047259 +0.32 -5.321335212 0.69310601673 +0.33 -5.1553985987 0.54781696226 +0.34 -4.9878134137 0.4018381796 +0.35 -4.8188673807 0.2555330423 +0.36 -4.6488511141 0.10927228913 +0.37 -4.4780573629 -0.036567271748 +0.38 -4.3067802363 -0.18160413005 +0.39 -4.1353144013 -0.32545351173 +0.4 -3.9639542476 -0.46772893557 +0.41 -3.7929930387 -0.60804381665 +0.42 -3.6227220421 -0.74601313801 +0.43 -3.4534296479 -0.88125517478 +0.44 -3.2854004768 -1.0133932584 +0.45 -3.1189144859 -1.1420575692 +0.46 -2.9542460734 -1.2668869442 +0.47 -2.7916631913 -1.387530686 +0.48 -2.6314265018 -1.5036503015 +0.49 -2.47378844 -1.6149214048 +0.5 -2.3189924173 -1.7210353819 +0.51 -2.1672720468 -1.8217009988 +0.52 -2.0188503481 -1.9166460389 +0.53 -1.8739388537 -2.0056191124 +0.54 -1.732737225 -2.0883905138 +0.55 -1.5954322069 -2.1647542719 +0.56 -1.4621973571 -2.2345287551 +0.57 -1.333192259 -2.2975581981 +0.58 -1.2085621978 -2.353713346 +0.59 -1.0884375079 -2.4028926755 +0.6 -0.97293360827 -2.4450223075 +0.61 -0.86215002573 -2.4800577568 +0.62 -0.75617104038 -2.5079826487 +0.63 -0.65506469693 -2.5288104095 +0.64 -0.55888310603 -2.5425835395 +0.65 -0.46766273408 -2.5493728888 +0.66 -0.38142374876 -2.5492785866 +0.67 -0.30017057762 -2.5424287393 +0.68 -0.22389217436 -2.5289786283 +0.69 -0.15256244154 -2.5091096125 +0.7 -0.086139788103 -2.4830294406 +0.71 -0.024568342949 -2.4509697004 +0.72 0.032221809773 -2.413184916 +0.73 0.084314093363 -2.3699510915 +0.74 0.1318047363 -2.321563976 +0.75 0.17480239707 -2.2683377698 +0.76 0.2134272741 -2.2106030175 +0.77 0.24781028338 -2.1487046133 +0.78 0.27809232097 -2.0829999018 +0.79 0.30442342743 -2.0138566245 +0.8 0.32696191973 -1.9416508145 +0.81 0.34587349606 -1.8667646538 +0.82 0.36133031921 -1.7895843109 +0.83 0.37351008427 -1.7104977709 +0.84 0.38259507609 -1.6298926758 +0.85 0.38877122239 -1.5481541871 +0.86 0.39222714765 -1.4656628877 +0.87 0.39315323345 -1.3827927341 +0.88 0.39174069021 -1.2999090738 +0.89 0.3881806455 -1.21736674 +0.9 0.38266325363 -1.1355082343 +0.91 0.3753768312 -1.05466201 +0.92 0.36650702291 -0.97514086392 +0.93 0.35623600185 -0.8972404476 +0.94 0.34474170806 -0.82123790516 +0.95 0.33219712893 -0.74739064427 +0.96 0.31876980308 -0.67593583254 +0.97 0.30462077846 -0.60708800794 +0.98 0.28990423397 -0.54103907779 +0.99 0.27476701087 -0.47795782396 +1.0 0.25934807756 -0.41798919045 +1.01 0.24377810285 -0.3612539062 +1.02 0.22817911217 -0.30784832508 +1.03 0.21266470226 -0.2578456722 +1.04 0.19733797262 -0.21129138178 +1.05 0.18229298665 -0.1682077754 +1.06 0.16761409975 -0.12859294839 +1.07 0.15337584366 -0.092421151151 +1.08 0.1396438166 -0.059645387463 +1.09 0.12647159361 -0.030191037026 +1.1 0.11390353081 -0.0039630063463 +1.11 0.10197406871 0.019155281934 +1.12 0.090708799358 0.039298886262 +1.13 0.080123593815 0.056619471672 +1.14 0.070227673946 0.071275245192 +1.15 0.061024372085 0.083429790064 +1.16 0.052511962988 0.093250798533 +1.17 0.044682028129 0.10090978584 +1.18 0.037521888086 0.10657912289 +1.19 0.031015059574 0.11043138827 +1.2 0.025141213318 0.11263866157 +1.21 0.019876005667 0.11336980971 +1.22 0.015192955473 0.11279033889 +1.23 0.011063595893 0.11106306695 +1.24 0.007456655257 0.10834247812 +1.25 0.0043400745021 0.10477759314 +1.26 0.0016808247737 0.10051281117 +1.27 -0.00055510433714 0.095680812463 +1.28 -0.0024016580764 0.090406927903 +1.29 -0.0038929622779 0.084811273797 +1.3 -0.0050625477429 0.078998968474 +1.31 -0.0059432416054 0.073068149935 +1.32 -0.0065674921377 0.067110734764 +1.33 -0.0069657771649 0.061201995595 +1.34 -0.007167828516 0.055412646763 +1.35 -0.0072020326121 0.049803640184 +1.36 -0.0070943360586 0.044422493293 +1.37 -0.0068701508663 0.039313829408 +1.38 -0.0065520891543 0.034508390981 +1.39 -0.0061611134842 0.030030366818 +1.4 -0.005717227564 0.025899372872 +1.41 -0.005236852799 0.022121815245 +1.42 -0.0047362333134 0.018703666441 +1.43 -0.0042285949508 0.01564086524 +1.44 -0.0037256393417 0.012925597893 +1.45 -0.003238024767 0.010546723262 +1.46 -0.0027732187555 0.0084862043738 +1.47 -0.0023389767364 0.0067264437607 +1.48 -0.0019396582406 0.005243996278 +1.49 -0.0015795366936 0.0040158727896 +1.5 -0.0012608949043 0.0030165453042 +1.51 -0.00098475359848 0.0022205393899 +1.52 -0.00075153947202 0.0016016330403 +1.53 -0.00055956428127 0.0011349570223 +1.54 -0.000407575033 0.00079484813589 +1.55 -0.00029180979095 0.00055922039698 +1.56 -0.00020917873683 0.00040490419124 +1.57 -0.00015459824577 0.00031332706041 +1.58 -0.00012325344328 0.00026554043958 +1.59 -0.00011211107894 0.00025459512511 +1.6 -0.00011091815786 0.00025685343044 +1.61 -8.0065645976e-05 0.00018636643746 +1.62 -2.207959279e-05 5.1598480417e-05 +1.63 6.3416160474e-06 -1.4718026015e-05 +1.64 3.4369124819e-06 -7.9766051653e-06 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 0.31691115296 0.12833055002 +0.02 0.6327506292 0.25591217486 +0.03 0.94645136007 0.38199950664 +0.04 1.2569554737 0.50585428246 +0.05 1.5632188455 0.62674887265 +0.06 1.8642155902 0.74396977966 +0.07 2.1589424785 0.85682109725 +0.08 2.4464232565 0.96462791932 +0.09 2.7257128515 1.0667396871 +0.1 2.9959014448 1.1625334629 +0.11 3.2561183922 1.2514171183 +0.12 3.5055359765 1.3328324232 +0.13 3.7433729717 1.4062580228 +0.14 3.9688980035 1.4712122871 +0.15 4.1814326898 1.5272560198 +0.16 4.3803545423 1.5739950083 +0.17 4.5650996186 1.6110824023 +0.18 4.7351649045 1.6382209014 +0.19 4.8901104195 1.6551647388 +0.2 5.0295610238 1.6617214382 +0.21 5.1532079278 1.6577533385 +0.22 5.2608098792 1.643178856 +0.23 5.3521940304 1.6179734804 +0.24 5.4272564708 1.5821704817 +0.25 5.4859624266 1.5358613221 +0.26 5.5283461091 1.4791957464 +0.27 5.5545102276 1.4123815589 +0.28 5.5646251474 1.3356840555 +0.29 5.5589277078 1.2494251212 +0.3 5.5377196963 1.1539819761 +0.31 5.5013659832 1.0497855704 +0.32 5.4502923332 0.93731863025 +0.33 5.3849828862 0.81711334783 +0.34 5.3059773248 0.68974871947 +0.35 5.213867786 0.55584758189 +0.36 5.1092954099 0.41607324677 +0.37 4.9929467086 0.27112589469 +0.38 4.8655496976 0.12173868067 +0.39 4.7278696661 -0.031326572994 +0.4 4.5807049337 -0.18728376199 +0.41 4.4248823343 -0.34532720879 +0.42 4.261252569 -0.50463655721 +0.43 4.0906854444 -0.66438183174 +0.44 3.9140650229 -0.82372863041 +0.45 3.732284716 -0.98184341479 +0.46 3.5462423515 -1.137898858 +0.47 3.3568352506 -1.2910792079 +0.48 3.1649554901 -1.4405854704 +0.49 2.9714847265 -1.5856410667 +0.5 2.777289713 -1.7254967608 +0.51 2.5832178663 -1.8594355206 +0.52 2.3900927615 -1.9867774392 +0.53 2.1987094003 -2.1068848753 +0.54 2.009831375 -2.2191655283 +0.55 1.8241857462 -2.323077914 +0.56 1.6424607668 -2.4181337401 +0.57 1.4653019142 -2.5039020266 +0.58 1.293309626 -2.5800113322 +0.59 1.1270361291 -2.6461528615 +0.6 0.96698445288 -2.702081178 +0.61 0.81360483138 -2.7476175365 +0.62 0.66729559136 -2.782648464 +0.63 0.52840007985 -2.8071282798 +0.64 0.39720703887 -2.8210779659 +0.65 0.27395097135 -2.8245839931 +0.66 0.15881083858 -2.8177985492 +0.67 0.051911531399 -2.8009369687 +0.68 -0.04667512666 -2.774275508 +0.69 -0.13692897946 -2.7381486826 +0.7 -0.21888098639 -2.6929468757 +0.71 -0.29261008059 -2.6391119188 +0.72 -0.35824128404 -2.5771334414 +0.73 -0.4159431628 -2.5075445643 +0.74 -0.46592500191 -2.4309173651 +0.75 -0.50843386132 -2.3478577659 +0.76 -0.54375116939 -2.2590003427 +0.77 -0.57218933031 -2.165003103 +0.78 -0.59408818976 -2.0665419618 +0.79 -0.60981137125 -1.9643051699 +0.8 -0.61974254036 -1.8589877089 +0.81 -0.62428163146 -1.7512857073 +0.82 -0.62384107136 -1.6418909303 +0.83 -0.61884203343 -1.5314853966 +0.84 -0.60971075517 -1.4207361709 +0.85 -0.59687495066 -1.3102903826 +0.86 -0.58076034815 -1.2007705148 +0.87 -0.56178738113 -1.0927700081 +0.88 -0.54036805961 -0.98684921825 +0.89 -0.51690304588 -0.88353176365 +0.9 -0.49177895717 -0.78330129473 +0.91 -0.46536591463 -0.68659871302 +0.92 -0.43801535576 -0.59381986204 +0.93 -0.41005812411 -0.50531370759 +0.94 -0.38180284687 -0.42138101873 +0.95 -0.3535346078 -0.3422735548 +0.96 -0.32551478108 -0.26819491238 +0.97 -0.29797801258 -0.1992975509 +0.98 -0.27113326694 -0.13568562865 +0.99 -0.24516387242 -0.077416374488 +1.0 -0.22022736766 -0.024501296954 +1.01 -0.19645591928 0.023091770031 +1.02 -0.17395706242 0.065436168456 +1.03 -0.15281569619 0.10264273095 +1.04 -0.13309095544 0.13486073225 +1.05 -0.11482209959 0.16226970049 +1.06 -0.098028572297 0.18507736826 +1.07 -0.082711657304 0.20351587633 +1.08 -0.068857068133 0.21783821631 +1.09 -0.056433565659 0.22831418337 +1.1 -0.04539878345 0.23522480425 +1.11 -0.035699969459 0.23885904541 +1.12 -0.027275757447 0.23951189031 +1.13 -0.020055775351 0.23748125528 +1.14 -0.013962010699 0.23306834239 +1.15 -0.0089106101841 0.2265749932 +1.16 -0.0048126887966 0.21830334176 +1.17 -0.0015766618338 0.20854486815 +1.18 0.00088988846604 0.19758215521 +1.19 0.0026797111161 0.18568844316 +1.2 0.0038846691896 0.17312405795 +1.21 0.0045919134973 0.16012585041 +1.22 0.0048855679028 0.14691636088 +1.23 0.004847386319 0.13370571947 +1.24 0.0045491243956 0.12066933844 +1.25 0.0040581363722 0.10796761423 +1.26 0.0034376915115 0.095746225461 +1.27 0.0027393344079 0.084115610895 +1.28 0.0020090206457 0.073169616761 +1.29 0.001288444668 0.062988588425 +1.3 0.00060672379237 0.053618373655 +1.31 -1.1366147491e-05 0.045092609726 +1.32 -0.00054582135205 0.037431155887 +1.33 -0.00098759768257 0.030625723663 +1.34 -0.0013297945509 0.024661714734 +1.35 -0.001571493513 0.019508874959 +1.36 -0.0017177187464 0.015122411378 +1.37 -0.0017749428805 0.011452922158 +1.38 -0.0017547314206 0.0084389831708 +1.39 -0.0016697251667 0.0060163835385 +1.4 -0.0015344078581 0.0041157761059 +1.41 -0.0013638321981 0.0026676915805 +1.42 -0.0011732258973 0.0016011633474 +1.43 -0.00097689571102 0.00084843776735 +1.44 -0.00078726131467 0.00034630915422 +1.45 -0.00061636763112 3.2136810107e-05 +1.46 -0.00047079465995 -0.00014219650548 +1.47 -0.00035797587788 -0.00022811628966 +1.48 -0.00027793709055 -0.00025754617817 +1.49 -0.00023007362243 -0.00026174974903 +1.5 -0.00021697247884 -0.00026976292795 +1.51 -0.00020908394519 -0.00026344496824 +1.52 -0.00011615291035 -0.0001468686914 +1.53 -2.9555906996e-06 -3.7564455382e-06 +1.54 1.2960730815e-05 1.6280198529e-05 +1.55 0.0 0.0 +1.56 0.0 0.0 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -9.7953079069E+00 +0.01 -1.0400311409E+01 +0.02 -1.0888406632E+01 +0.03 -1.1200435759E+01 +0.04 -1.1277240974E+01 +0.05 -1.1209255048E+01 +0.06 -1.1082805427E+01 +0.07 -1.0947620831E+01 +0.08 -1.0824723658E+01 +0.09 -1.0719296734E+01 +0.1 -1.0629697231E+01 +0.11 -1.0552200672E+01 +0.12 -1.0483027171E+01 +0.13 -1.0419016416E+01 +0.14 -1.0357741770E+01 +0.15 -1.0297434680E+01 +0.16 -1.0236868006E+01 +0.17 -1.0175247464E+01 +0.18 -1.0112121292E+01 +0.19 -1.0047305980E+01 +0.2 -9.9808234246E+00 +0.21 -9.9128463822E+00 +0.22 -9.8436503701E+00 +0.23 -9.7735719697E+00 +0.24 -9.7029738376E+00 +0.25 -9.6322169180E+00 +0.26 -9.5616399874E+00 +0.27 -9.4915460360E+00 +0.28 -9.4221946690E+00 +0.29 -9.3537991265E+00 +0.3 -9.2865265731E+00 +0.31 -9.2205003077E+00 +0.32 -9.1558027948E+00 +0.33 -9.0924785976E+00 +0.34 -9.0305368263E+00 +0.35 -8.9699529387E+00 +0.36 -8.9106696470E+00 +0.37 -8.8525976536E+00 +0.38 -8.7956163315E+00 +0.39 -8.7395746729E+00 +0.4 -8.6842935256E+00 +0.41 -8.6295690309E+00 +0.42 -8.5751777523E+00 +0.43 -8.5208835233E+00 +0.44 -8.4664458092E+00 +0.45 -8.4116291209E+00 +0.46 -8.3562127953E+00 +0.47 -8.3000003168E+00 +0.48 -8.2428271911E+00 +0.49 -8.1845673552E+00 +0.5 -8.1251364881E+00 +0.51 -8.0644926540E+00 +0.52 -8.0026346953E+00 +0.53 -7.9395985751E+00 +0.54 -7.8754509157E+00 +0.55 -7.8102832830E+00 +0.56 -7.7442045501E+00 +0.57 -7.6773346132E+00 +0.58 -7.6097981896E+00 +0.59 -7.5417198577E+00 +0.6 -7.4732201798E+00 +0.61 -7.4044126046E+00 +0.62 -7.3354020632E+00 +0.63 -7.2662835375E+00 +0.64 -7.1971420148E+00 +0.65 -7.1280529085E+00 +0.66 -7.0590822441E+00 +0.67 -6.9902877566E+00 +0.68 -6.9217198555E+00 +0.69 -6.8534226258E+00 +0.7 -6.7854345116E+00 +0.71 -6.7177895103E+00 +0.72 -6.6505178739E+00 +0.73 -6.5836468129E+00 +0.74 -6.5172011142E+00 +0.75 -6.4512036077E+00 +0.76 -6.3856756197E+00 +0.77 -6.3206373121E+00 +0.78 -6.2561079364E+00 +0.79 -6.1921060357E+00 +0.8 -6.1286495929E+00 +0.81 -6.0657561346E+00 +0.82 -6.0034427989E+00 +0.83 -5.9417263711E+00 +0.84 -5.8806232961E+00 +0.85 -5.8201496694E+00 +0.86 -5.7603212132E+00 +0.87 -5.7011532385E+00 +0.88 -5.6426605986E+00 +0.89 -5.5848576333E+00 +0.9 -5.5277581083E+00 +0.91 -5.4713751498E+00 +0.92 -5.4157211748E+00 +0.93 -5.3608078208E+00 +0.94 -5.3066458721E+00 +0.95 -5.2532451864E+00 +0.96 -5.2006146500E+00 +0.97 -5.1487620332E+00 +0.98 -5.0976939521E+00 +0.99 -5.0474158034E+00 +1.0 -4.9979316802E+00 +1.01 -4.9492442981E+00 +1.02 -4.9013549271E+00 +1.03 -4.8542634146E+00 +1.04 -4.8079678026E+00 +1.05 -4.7624645403E+00 +1.06 -4.7177483280E+00 +1.07 -4.6738120477E+00 +1.08 -4.6306468941E+00 +1.09 -4.5882416969E+00 +1.1 -4.5465833716E+00 +1.11 -4.5056567463E+00 +1.12 -4.4654448539E+00 +1.13 -4.4259298481E+00 +1.14 -4.3870946833E+00 +1.15 -4.3489228924E+00 +1.16 -4.3113985506E+00 +1.17 -4.2745064577E+00 +1.18 -4.2382318993E+00 +1.19 -4.2025606063E+00 +1.2 -4.1674787716E+00 +1.21 -4.1329731249E+00 +1.22 -4.0990307195E+00 +1.23 -4.0656388980E+00 +1.24 -4.0327855198E+00 +1.25 -4.0004586688E+00 +1.26 -3.9686466511E+00 +1.27 -3.9373381834E+00 +1.28 -3.9065221765E+00 +1.29 -3.8761876992E+00 +1.3 -3.8463241824E+00 +1.31 -3.8169211999E+00 +1.32 -3.7879684651E+00 +1.33 -3.7594560262E+00 +1.34 -3.7313740213E+00 +1.35 -3.7037127811E+00 +1.36 -3.6764629142E+00 +1.37 -3.6496150905E+00 +1.38 -3.6231602853E+00 +1.39 -3.5970896520E+00 +1.4 -3.5713944446E+00 +1.41 -3.5460663178E+00 +1.42 -3.5210969494E+00 +1.43 -3.4964783706E+00 +1.44 -3.4722028189E+00 +1.45 -3.4482626548E+00 +1.46 -3.4246506993E+00 +1.47 -3.4013597263E+00 +1.48 -3.3783830415E+00 +1.49 -3.3557139795E+00 +1.5 -3.3333461989E+00 +1.51 -3.3112736518E+00 +1.52 -3.2894903707E+00 +1.53 -3.2679910124E+00 +1.54 -3.2467701693E+00 +1.55 -3.2258229108E+00 +1.56 -3.2051442058E+00 +1.57 -3.1847290067E+00 +1.58 -3.1645723052E+00 +1.59 -3.1446693226E+00 +1.6 -3.1250152668E+00 +1.61 -3.1056053810E+00 +1.62 -3.0864350882E+00 +1.63 -3.0674999620E+00 +1.64 -3.0487957324E+00 +1.65 -3.0303182431E+00 +1.66 -3.0120634156E+00 +1.67 -2.9940271521E+00 +1.68 -2.9762055892E+00 +1.69 -2.9585948974E+00 +1.7 -2.9411913974E+00 +1.71 -2.9239914167E+00 +1.72 -2.9069914394E+00 +1.73 -2.8901879713E+00 +1.74 -2.8735776456E+00 +1.75 -2.8571571408E+00 +1.76 -2.8409232236E+00 +1.77 -2.8248727423E+00 +1.78 -2.8090025834E+00 +1.79 -2.7933097620E+00 +1.8 -2.7777912717E+00 +1.81 -2.7624442908E+00 +1.82 -2.7472659091E+00 +1.83 -2.7322534622E+00 +1.84 -2.7174041320E+00 +1.85 -2.7027153831E+00 +1.86 -2.6881845381E+00 +1.87 -2.6738091286E+00 +1.88 -2.6595866351E+00 +1.89 -2.6455146440E+00 +1.9 -2.6315907931E+00 +1.91 -2.6178127142E+00 +1.92 -2.6041781993E+00 +1.93 -2.5906849203E+00 +1.94 -2.5773307925E+00 +1.95 -2.5641135986E+00 +1.96 -2.5510312912E+00 +1.97 -2.5380818020E+00 +1.98 -2.5252630984E+00 +1.99 -2.5125732639E+00 +2.0 -2.5000102769E+00 +2.01 -2.4875723360E+00 +2.02 -2.4752575202E+00 +2.03 -2.4630640399E+00 +2.04 -2.4509901208E+00 +2.05 -2.4390339638E+00 +2.06 -2.4271939251E+00 +2.07 -2.4154682479E+00 +2.08 -2.4038553365E+00 +2.09 -2.3923535617E+00 +2.1 -2.3809613030E+00 +2.11 -2.3696770644E+00 +2.12 -2.3584992450E+00 +2.13 -2.3474264006E+00 +2.14 -2.3364570512E+00 +2.15 -2.3255897148E+00 +2.16 -2.3148230361E+00 +2.17 -2.3041555626E+00 +2.18 -2.2935859647E+00 +2.19 -2.2831129150E+00 +2.2 -2.2727350358E+00 +2.21 -2.2624511056E+00 +2.22 -2.2522598146E+00 +2.23 -2.2421599154E+00 +2.24 -2.2321502244E+00 +2.25 -2.2222294734E+00 +2.26 -2.2123965330E+00 +2.27 -2.2026502440E+00 +2.28 -2.1929894132E+00 +2.29 -2.1834129847E+00 +2.3 -2.1739198255E+00 +2.31 -2.1645088428E+00 +2.32 -2.1551790170E+00 +2.33 -2.1459292535E+00 +2.34 -2.1367585500E+00 +2.35 -2.1276659213E+00 +2.36 -2.1186503127E+00 +2.37 -2.1097108018E+00 +2.38 -2.1008464331E+00 +2.39 -2.0920562046E+00 +2.4 -2.0833392519E+00 +2.41 -2.0746946489E+00 +2.42 -2.0661214596E+00 +2.43 -2.0576188552E+00 +2.44 -2.0491859464E+00 +2.45 -2.0408218556E+00 +2.46 -2.0325257888E+00 +2.47 -2.0242968934E+00 +2.48 -2.0161343429E+00 +2.49 -2.0080373776E+00 +2.5 -2.0000051821E+00 +2.51 -1.9920369736E+00 +2.52 -1.9841320267E+00 +2.53 -1.9762895631E+00 +2.54 -1.9685088371E+00 +2.55 -1.9607891574E+00 +2.56 -1.9531297830E+00 +2.57 -1.9455299990E+00 +2.58 -1.9379891482E+00 +2.59 -1.9305065270E+00 +2.6 -1.9230814452E+00 +2.61 -1.9157132799E+00 +2.62 -1.9084013649E+00 +2.63 -1.9011450292E+00 +2.64 -1.8939436842E+00 +2.65 -1.8867967011E+00 +2.66 -1.8797034234E+00 +2.67 -1.8726632960E+00 +2.68 -1.8656757242E+00 +2.69 -1.8587400729E+00 +2.7 -1.8518558088E+00 +2.71 -1.8450223684E+00 +2.72 -1.8382391535E+00 +2.73 -1.8315056327E+00 +2.74 -1.8248212788E+00 +2.75 -1.8181855313E+00 +2.76 -1.8115978559E+00 +2.77 -1.8050577615E+00 +2.78 -1.7985647255E+00 +2.79 -1.7921182080E+00 +2.8 -1.7857177520E+00 +2.81 -1.7793628661E+00 +2.82 -1.7730530256E+00 +2.83 -1.7667877827E+00 +2.84 -1.7605666767E+00 +2.85 -1.7543892206E+00 +2.86 -1.7482549508E+00 +2.87 -1.7421634433E+00 +2.88 -1.7361142500E+00 +2.89 -1.7301068892E+00 +2.9 -1.7241409715E+00 +2.91 -1.7182160709E+00 +2.92 -1.7123317389E+00 +2.93 -1.7064875664E+00 +2.94 -1.7006831640E+00 +2.95 -1.6949181227E+00 +2.96 -1.6891920072E+00 +2.97 -1.6835044635E+00 +2.98 -1.6778551046E+00 +2.99 -1.6722435230E+00 +3.0 -1.6666693437E+00 +3.01 -1.6611322150E+00 +3.02 -1.6556317677E+00 +3.03 -1.6501675996E+00 +3.04 -1.6447393891E+00 +3.05 -1.6393467861E+00 +3.06 -1.6339894265E+00 +3.07 -1.6286669522E+00 +3.08 -1.6233790509E+00 +3.09 -1.6181253872E+00 +3.1 -1.6129056026E+00 +3.11 -1.6077193828E+00 +3.12 -1.6025664196E+00 +3.13 -1.5974463932E+00 +3.14 -1.5923589527E+00 +3.15 -1.5873038213E+00 +3.16 -1.5822806958E+00 +3.17 -1.5772892645E+00 +3.18 -1.5723292057E+00 +3.19 -1.5674002544E+00 +3.2 -1.5625021199E+00 +3.21 -1.5576344984E+00 +3.22 -1.5527970977E+00 +3.23 -1.5479896601E+00 +3.24 -1.5432119082E+00 +3.25 -1.5384635475E+00 +3.26 -1.5337443106E+00 +3.27 -1.5290539473E+00 +3.28 -1.5243921936E+00 +3.29 -1.5197587649E+00 +3.3 -1.5151534139E+00 +3.31 -1.5105758996E+00 +3.32 -1.5060259698E+00 +3.33 -1.5015033514E+00 +3.34 -1.4970078128E+00 +3.35 -1.4925391227E+00 +3.36 -1.4880970407E+00 +3.37 -1.4836813055E+00 +3.38 -1.4792916976E+00 +3.39 -1.4749279962E+00 +3.4 -1.4705899722E+00 +3.41 -1.4662773767E+00 +3.42 -1.4619899986E+00 +3.43 -1.4577276285E+00 +3.44 -1.4534900480E+00 +3.45 -1.4492770216E+00 +3.46 -1.4450883429E+00 +3.47 -1.4409238146E+00 +3.48 -1.4367832287E+00 +3.49 -1.4326663637E+00 +3.5 -1.4285730146E+00 +3.51 -1.4245029971E+00 +3.52 -1.4204561129E+00 +3.53 -1.4164321554E+00 +3.54 -1.4124309178E+00 +3.55 -1.4084522296E+00 +3.56 -1.4044959016E+00 +3.57 -1.4005617428E+00 +3.58 -1.3966495421E+00 +3.59 -1.3927591429E+00 +3.6 -1.3888903647E+00 +3.61 -1.3850430270E+00 +3.62 -1.3812169299E+00 +3.63 -1.3774119129E+00 +3.64 -1.3736278098E+00 +3.65 -1.3698644486E+00 +3.66 -1.3661216462E+00 +3.67 -1.3623992310E+00 +3.68 -1.3586970529E+00 +3.69 -1.3550149479E+00 +3.7 -1.3513527506E+00 +3.71 -1.3477102763E+00 +3.72 -1.3440873909E+00 +3.73 -1.3404839380E+00 +3.74 -1.3368997612E+00 +3.75 -1.3333346896E+00 +3.76 -1.3297885764E+00 +3.77 -1.3262612817E+00 +3.78 -1.3227526565E+00 +3.79 -1.3192625488E+00 +3.8 -1.3157907932E+00 +3.81 -1.3123372674E+00 +3.82 -1.3089018292E+00 +3.83 -1.3054843366E+00 +3.84 -1.3020846342E+00 +3.85 -1.2987025869E+00 +3.86 -1.2953380691E+00 +3.87 -1.2919909450E+00 +3.88 -1.2886610792E+00 +3.89 -1.2853483151E+00 +3.9 -1.2820525436E+00 +3.91 -1.2787736362E+00 +3.92 -1.2755114636E+00 +3.93 -1.2722658895E+00 +3.94 -1.2690367790E+00 +3.95 -1.2658240235E+00 +3.96 -1.2626274997E+00 +3.97 -1.2594470844E+00 +3.98 -1.2562826417E+00 +3.99 -1.2531340562E+00 +4.0 -1.2500012187E+00 +4.01 -1.2468840119E+00 +4.02 -1.2437823182E+00 +4.03 -1.2406960031E+00 +4.04 -1.2376249674E+00 +4.05 -1.2345691024E+00 +4.06 -1.2315282961E+00 +4.07 -1.2285024364E+00 +4.08 -1.2254913921E+00 +4.09 -1.2224950757E+00 +4.1 -1.2195133805E+00 +4.11 -1.2165461998E+00 +4.12 -1.2135934242E+00 +4.13 -1.2106549331E+00 +4.14 -1.2077306417E+00 +4.15 -1.2048204481E+00 +4.16 -1.2019242507E+00 +4.17 -1.1990419438E+00 +4.18 -1.1961734148E+00 +4.19 -1.1933185819E+00 +4.2 -1.1904773482E+00 +4.21 -1.1876496167E+00 +4.22 -1.1848352867E+00 +4.23 -1.1820342502E+00 +4.24 -1.1792464301E+00 +4.25 -1.1764717336E+00 +4.26 -1.1737100682E+00 +4.27 -1.1709613393E+00 +4.28 -1.1682254410E+00 +4.29 -1.1655023010E+00 +4.3 -1.1627918312E+00 +4.31 -1.1600939433E+00 +4.32 -1.1574085490E+00 +4.33 -1.1547355427E+00 +4.34 -1.1520748576E+00 +4.35 -1.1494264097E+00 +4.36 -1.1467901148E+00 +4.37 -1.1441658889E+00 +4.38 -1.1415536337E+00 +4.39 -1.1389532790E+00 +4.4 -1.1363647480E+00 +4.41 -1.1337879605E+00 +4.42 -1.1312228362E+00 +4.43 -1.1286692854E+00 +4.44 -1.1261272320E+00 +4.45 -1.1235966071E+00 +4.46 -1.1210773343E+00 +4.47 -1.1185693371E+00 +4.48 -1.1160725347E+00 +4.49 -1.1135868436E+00 +4.5 -1.1111122031E+00 +4.51 -1.1086485404E+00 +4.52 -1.1061957826E+00 +4.53 -1.1037538568E+00 +4.54 -1.1013226758E+00 +4.55 -1.0989021820E+00 +4.56 -1.0964923080E+00 +4.57 -1.0940929841E+00 +4.58 -1.0917041410E+00 +4.59 -1.0893257018E+00 +4.6 -1.0869575968E+00 +4.61 -1.0845997688E+00 +4.62 -1.0822521513E+00 +4.63 -1.0799146781E+00 +4.64 -1.0775872830E+00 +4.65 -1.0752698851E+00 +4.66 -1.0729624349E+00 +4.67 -1.0706648699E+00 +4.68 -1.0683771270E+00 +4.69 -1.0660991430E+00 +4.7 -1.0638308489E+00 +4.71 -1.0615721790E+00 +4.72 -1.0593230825E+00 +4.73 -1.0570834992E+00 +4.74 -1.0548533689E+00 +4.75 -1.0526326312E+00 +4.76 -1.0504212151E+00 +4.77 -1.0482190695E+00 +4.78 -1.0460261407E+00 +4.79 -1.0438423714E+00 +4.8 -1.0416677041E+00 +4.81 -1.0395020809E+00 +4.82 -1.0373454313E+00 +4.83 -1.0351977142E+00 +4.84 -1.0330588748E+00 +4.85 -1.0309288585E+00 +4.86 -1.0288076105E+00 +4.87 -1.0266950722E+00 +4.88 -1.0245911830E+00 +4.89 -1.0224959010E+00 +4.9 -1.0204091741E+00 +4.91 -1.0183309501E+00 +4.92 -1.0162611767E+00 +4.93 -1.0141997958E+00 +4.94 -1.0121467543E+00 +4.95 -1.0101020102E+00 +4.96 -1.0080655139E+00 +4.97 -1.0060372156E+00 +4.98 -1.0040170655E+00 +4.99 -1.0020050068E+00 +5.0 -1.0000009915E+00 +5.01 -9.9800497853E-01 +5.02 -9.9601692053E-01 +5.03 -9.9403677005E-01 +5.04 -9.9206447965E-01 +5.05 -9.9009999481E-01 +5.06 -9.8814327013E-01 +5.07 -9.8619426653E-01 +5.08 -9.8425293881E-01 +5.09 -9.8231924171E-01 +5.1 -9.8039313002E-01 +5.11 -9.7847455227E-01 +5.12 -9.7656346372E-01 +5.13 -9.7465982794E-01 +5.14 -9.7276360183E-01 +5.15 -9.7087474224E-01 +5.16 -9.6899320607E-01 +5.17 -9.6711894568E-01 +5.18 -9.6525191502E-01 +5.19 -9.6339208106E-01 +5.2 -9.6153940271E-01 +5.21 -9.5969383884E-01 +5.22 -9.5785534834E-01 +5.23 -9.5602388818E-01 +5.24 -9.5419940913E-01 +5.25 -9.5238188227E-01 +5.26 -9.5057126840E-01 +5.27 -9.4876752834E-01 +5.28 -9.4697062287E-01 +5.29 -9.4518051281E-01 +5.3 -9.4339714846E-01 +5.31 -9.4162050123E-01 +5.32 -9.3985053520E-01 +5.33 -9.3808721299E-01 +5.34 -9.3633049724E-01 +5.35 -9.3458035056E-01 +5.36 -9.3283672935E-01 +5.37 -9.3109959783E-01 +5.38 -9.2936892590E-01 +5.39 -9.2764467793E-01 +5.4 -9.2592681829E-01 +5.41 -9.2421531134E-01 +5.42 -9.2251012024E-01 +5.43 -9.2081120058E-01 +5.44 -9.1911852851E-01 +5.45 -9.1743207008E-01 +5.46 -9.1575179130E-01 +5.47 -9.1407765821E-01 +5.48 -9.1240963683E-01 +5.49 -9.1074768644E-01 +5.5 -9.0909177628E-01 +5.51 -9.0744187842E-01 +5.52 -9.0579796048E-01 +5.53 -9.0415999006E-01 +5.54 -9.0252793478E-01 +5.55 -9.0090176200E-01 +5.56 -8.9928142906E-01 +5.57 -8.9766691560E-01 +5.58 -8.9605819074E-01 +5.59 -8.9445522360E-01 +5.6 -8.9285798330E-01 +5.61 -8.9126643897E-01 +5.62 -8.8968055584E-01 +5.63 -8.8810030060E-01 +5.64 -8.8652565061E-01 +5.65 -8.8495657644E-01 +5.66 -8.8339304863E-01 +5.67 -8.8183503776E-01 +5.68 -8.8028251439E-01 +5.69 -8.7873544257E-01 +5.7 -8.7719379623E-01 +5.71 -8.7565755125E-01 +5.72 -8.7412667957E-01 +5.73 -8.7260115311E-01 +5.74 -8.7108094382E-01 +5.75 -8.6956602361E-01 +5.76 -8.6805635630E-01 +5.77 -8.6655192092E-01 +5.78 -8.6505269276E-01 +5.79 -8.6355864504E-01 +5.8 -8.6206975101E-01 +5.81 -8.6058598390E-01 +5.82 -8.5910731697E-01 +5.83 -8.5763371456E-01 +5.84 -8.5616515888E-01 +5.85 -8.5470162543E-01 +5.86 -8.5324308868E-01 +5.87 -8.5178952313E-01 +5.88 -8.5034090325E-01 +5.89 -8.4889720355E-01 +5.9 -8.4745838969E-01 +5.91 -8.4602444519E-01 +5.92 -8.4459534656E-01 +5.93 -8.4317106947E-01 +5.94 -8.4175158961E-01 +5.95 -8.4033688264E-01 +5.96 -8.3892692424E-01 +5.97 -8.3752168212E-01 +5.98 -8.3612113938E-01 +5.99 -8.3472527429E-01 +6.0 -8.3333406366E-01 +6.01 -8.3194748432E-01 diff --git a/src/Control.cc b/src/Control.cc index d1c4c672..a798b5db 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -997,42 +997,6 @@ void Control::readRestartInfo(std::ifstream* tfile) } } -void Control::readRestartOutputInfo(std::ifstream* tfile) -{ - const std::string zero = "0"; - const std::string one = "1"; - if (tfile != nullptr) - { - // Read in the output restart filename - std::string filename; - (*tfile) >> filename; - // int dpcs_chkpoint=0; - if (zero.compare(filename) == 0) // no restart dump - out_restart_info = 0; - else - { - if (one.compare(filename) == 0) - { // automatic naming of dump - filename = "snapshot"; - out_restart_file_naming_strategy = 1; - } - (*tfile) >> out_restart_info; - (*tfile) >> out_restart_file_type; - //(*tfile)>>dpcs_chkpoint; - // timeout_.set(dpcs_chkpoint); - } - - out_restart_file.assign(run_directory_); - out_restart_file.append("/"); - out_restart_file.append(filename); - (*MPIdata::sout) << "Output restart file: " << out_restart_file - << " with info level " << out_restart_info - << std::endl; - //(*MPIdata::sout)<<"Time for DPCS checkpoint: - //"<::solve( // orthonormalization over outer iterations (MD or geometry optimization) // jlf, 01/02/2021 if (mmpi.PE0() && ct.verbose > 1) - os_ << "Orthonormalize wavefunctions ate start of DavidsonSolver" + os_ << "Orthonormalize wavefunctions at start of DavidsonSolver" << std::endl; orbitals.orthonormalizeLoewdin(false, nullptr, false); @@ -418,7 +418,7 @@ int DavidsonSolver::solve( ProjectedMatrices* proj_matN = dynamic_cast*>( orbitals.getProjMatrices()); - if (mmpi.PE0() && ct.verbose > 1) + if (mmpi.PE0() && ct.verbose > 0) { os_ << "###########################" << std::endl; os_ << "DavidsonSolver -> Iteration " << outer_it << std::endl; @@ -498,7 +498,7 @@ int DavidsonSolver::solve( // if( onpe0 )os_<<"Matrices..."<printMatrices(os_); - ts0 = evalEntropy(projmatrices, true, os_); + ts0 = evalEntropy(projmatrices, (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( ts0, projmatrices, orbitals, printE, os_); @@ -547,7 +547,7 @@ int DavidsonSolver::solve( proj_mat2N_->assignBlocksH(h11, h12, h21, h22); proj_mat2N_->setHB2H(); - ts0 = evalEntropy(proj_mat2N_.get(), true, os_); + ts0 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( ts0, proj_mat2N_.get(), orbitals, printE, os_); } @@ -627,7 +627,7 @@ int DavidsonSolver::solve( // line minimization beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); - if (mmpi.PE0() && ct.verbose > 1) + if (mmpi.PE0() && ct.verbose > 0) { os_ << std::setprecision(12); os_ << "ts1=" << ts1 << std::endl; @@ -690,7 +690,7 @@ int DavidsonSolver::solve( os_ << "Total occupations for top half states=" << std::setprecision(15) << tot << std::endl; } - if (mmpi.PE0() && ct.verbose > 0) + if (mmpi.PE0() && ct.verbose > 1) { os_ << std::setprecision(15) << "Last level occupancy = " << eval[numst_] << std::endl; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 32877d01..87b30224 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -118,13 +118,13 @@ void DensityMatrix::build( template void DensityMatrix::build(const int new_orbitals_index) { - //#ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); +#ifdef PRINT_OPERATIONS if (mmpi.PE0()) std::cout << "DensityMatrix::build() for diagonal occupation..." << std::endl; - //#endif +#endif if (!occ_uptodate_ && mmpi.instancePE0()) std::cout << "Warning: occupations not up to date to build DM!!!" << std::endl; diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index db156a9f..f9bf3860 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -90,8 +90,10 @@ void Hartree_CG::solve( double residual_reduction = poisson_solver_->getResidualReduction(); double final_residual = poisson_solver_->getFinalResidual(); + const bool large_residual + = (residual_reduction > 1.e-3 || final_residual > 1.e-3); - if (onpe0) + if (onpe0 && (large_residual || ct.verbose > 1)) (*MPIdata::sout) << setprecision(2) << scientific << "Hartree_CG: residual reduction = " << residual_reduction diff --git a/src/IonicAlgorithm.cc b/src/IonicAlgorithm.cc index d2cea35a..37a077e4 100644 --- a/src/IonicAlgorithm.cc +++ b/src/IonicAlgorithm.cc @@ -98,7 +98,7 @@ void IonicAlgorithm::init(HDFrestart* h5f_file) template int IonicAlgorithm::quenchElectrons(const int itmax, double& etot) { - int ret = mgmol_strategy_.quench(*orbitals_, ions_, itmax, 0, etot); + int ret = mgmol_strategy_.quench(**orbitals_, ions_, itmax, 0, etot); return ret; } diff --git a/src/LBFGS.cc b/src/LBFGS.cc index 0837a0e1..9154b7b3 100644 --- a/src/LBFGS.cc +++ b/src/LBFGS.cc @@ -97,7 +97,7 @@ int LBFGS::quenchElectrons(const int itmax, double& etot) { etot_i_[0] = etot_i_[1]; etot_i_[1] = etot_i_[2]; - int ret = mgmol_strategy_.quench(*orbitals_, ions_, itmax, 0, etot_i_[2]); + int ret = mgmol_strategy_.quench(**orbitals_, ions_, itmax, 0, etot_i_[2]); etot = etot_i_[2]; return ret; diff --git a/src/MGmol.cc b/src/MGmol.cc index c91120e9..5eb37c2c 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -521,7 +521,8 @@ void MGmol::run() switch (ct.AtomsDynamic()) { case AtomsDynamicType::Quench: - quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + quench( + *current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); // Forces for the last states force(*current_orbitals_, *ions_); @@ -1097,25 +1098,21 @@ void MGmol::setup() } template -void MGmol::cleanup() +void MGmol::dumpRestart() { - closing_tm_.start(); - - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - Control& ct = *(Control::instance()); - - printTimers(); + Control& ct = *(Control::instance()); // Save data to restart file - if (ct.out_restart_info > 0 && !ct.AtomsMove()) + if (ct.out_restart_info > 0) { + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); unsigned gdim[3] = { mygrid.gdim(0), mygrid.gdim(1), mygrid.gdim(2) }; + const pb::PEenv& myPEenv = mymesh->peenv(); // create restart file std::string filename(std::string(ct.out_restart_file)); - filename += "0"; + if (ct.out_restart_file_naming_strategy) filename += "0"; HDFrestart h5restartfile( filename, myPEenv, gdim, ct.out_restart_file_type); @@ -1136,7 +1133,22 @@ void MGmol::cleanup() os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; } #endif - } // if (ct.out_restart_info > 0 && !ct.AtomsMove()) + } +} +template +void MGmol::cleanup() +{ + closing_tm_.start(); + + Control& ct = *(Control::instance()); + + printTimers(); + + // Save data to restart file + if (!ct.AtomsMove()) + { + dumpRestart(); + } MPI_Barrier(comm_); closing_tm_.stop(); @@ -1433,6 +1445,14 @@ template double MGmol::evaluateEnergyAndForces( const std::vector& tau, std::vector& atnumbers, std::vector& forces) +{ + return evaluateEnergyAndForces(current_orbitals_, tau, atnumbers, forces); +} + +template +double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) { assert(tau.size() == 3 * atnumbers.size()); @@ -1440,10 +1460,13 @@ double MGmol::evaluateEnergyAndForces( ions_->setPositions(tau, atnumbers); - double eks = 0.; - quench(current_orbitals_, *ions_, ct.max_electronic_steps, 20, eks); + moveVnuc(*ions_); + + double eks = 0.; + OrbitalsType* dorbitals = dynamic_cast(orbitals); + quench(*dorbitals, *ions_, ct.max_electronic_steps, 20, eks); - force(*current_orbitals_, *ions_); + force(*dorbitals, *ions_); ions_->getForces(forces); diff --git a/src/MGmol.h b/src/MGmol.h index ef7a020b..0e70d6b7 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -133,7 +133,7 @@ class MGmol : public MGmolInterface KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij); void computeHnlPhiAndAdd2HPhi(Ions& ions, OrbitalsType& phi, OrbitalsType& hphi, const KBPsiMatrixSparse* const kbpsi); - int dumprestartFile(OrbitalsType** orbitals, Ions& ions, + int dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count); @@ -182,13 +182,19 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getHamiltonian() + { + return hamiltonian_; + } void run() override; double evaluateEnergyAndForces(const std::vector& tau, std::vector& atnumbers, std::vector& forces); + double evaluateEnergyAndForces(Orbitals*, const std::vector& tau, + std::vector& atnumbers, std::vector& forces); + /* * get internal atomic positions */ @@ -246,7 +252,7 @@ class MGmol : public MGmolInterface void update_pot(const pb::GridFunc& vh_init, const Ions& ions); void update_pot(const Ions& ions); - int quench(OrbitalsType* orbitals, Ions& ions, const int max_steps, + int quench(OrbitalsType& orbitals, Ions& ions, const int max_steps, const int iprint, double& last_eks); int outerSolve(OrbitalsType& orbitals, OrbitalsType& work_orbitals, Ions& ions, const int max_steps, const int iprint, double& last_eks); @@ -319,8 +325,18 @@ class MGmol : public MGmolInterface forces_->force(orbitals, ions); } + /* + * simply dump current state + */ + void dumpRestart(); + void loadRestartFile(const std::string filename); + std::shared_ptr getProjectedMatrices() + { + return proj_matrices_; + } + #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); #endif diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 046459d6..af7be67b 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -28,8 +28,15 @@ class MGmolInterface virtual double evaluateEnergyAndForces(const std::vector& tau, std::vector& atnumbers, std::vector& forces) = 0; + virtual double evaluateEnergyAndForces(Orbitals*, + const std::vector& tau, std::vector& atnumbers, + std::vector& forces) + = 0; virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual std::shared_ptr getProjectedMatrices() + = 0; + virtual void dumpRestart() = 0; }; #endif diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 63ddf690..7feb564d 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -59,7 +59,8 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, if (onpe0 && ct.verbose > 0) { os_ << "MVPSolver..." << std::endl; - if (use_old_dm_) os_ << "MVPSolver uses old DM..." << std::endl; + if (use_old_dm_ && ct.verbose > 1) + os_ << "MVPSolver uses old DM..." << std::endl; } rho_ = rho; @@ -252,8 +253,9 @@ int MVPSolver::solve(OrbitalsType& orbitals) dmInit = proj_mat_work_->dm(); } - const double ts0 = evalEntropyMVP(current_proj_mat, true, os_); - const double e0 = energy_->evaluateTotal( + const double ts0 + = evalEntropyMVP(current_proj_mat, (ct.verbose > 1), os_); + const double e0 = energy_->evaluateTotal( ts0, current_proj_mat, orbitals, printE, os_); MatrixType target("target", numst_, numst_); @@ -345,6 +347,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) = dynamic_cast*>( orbitals.getProjMatrices()); projmatrices->setDM(*work_, orbitals.getIterativeIndex()); + projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); } // Generate new density diff --git a/src/Orbitals.h b/src/Orbitals.h index d58f46bb..bc524aff 100644 --- a/src/Orbitals.h +++ b/src/Orbitals.h @@ -26,6 +26,8 @@ class Orbitals Orbitals() { iterative_index_ = -10; } + virtual ~Orbitals(){}; + Orbitals(const Orbitals& A, const bool copy_data) { if (copy_data) diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 702637f4..6725b827 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -409,6 +409,12 @@ class ProjectedMatrices : public ProjectedMatricesInterface { dm_->setMatrix(mat, orbitals_index); } + void setEigenvalues(const std::vector& eigenvalues) + { + memcpy(eigenvalues_.data(), eigenvalues.data(), + eigenvalues.size() * sizeof(double)); + } + const std::vector& getEigenvalues() const { return eigenvalues_; } void computeGenEigenInterval(std::vector& interval, const int maxits, const double padding = 0.01); DensityMatrix& getDM() { return *dm_; } diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index 4041ba76..653b386d 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -71,7 +71,7 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction Control& ct = *(Control::instance()); nel_ = ct.getNelSpin(); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.PE0()) + if (mmpi.PE0() && ct.verbose > 1) std::cout << "ProjectedMatricesInterface: nel_=" << nel_ << std::endl; }; diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index b2aff795..6698eef0 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -77,6 +77,7 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) lbfgs.computeForces(); + // check for convergence int flag_convF = lbfgs.checkTolForces(ct.tol_forces); int conv = 0; @@ -87,10 +88,8 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) os_ << endl << endl << "LBFGS: convergence in forces has been achieved. " - "stopping ..." << endl; } - conv = 1; } else { @@ -116,10 +115,17 @@ void MGmol::lbfgsrlx(OrbitalsType** orbitals, Ions& ions) // Write down positions and displacements ions.printPositions(os_); + // early termination if convergence achieved + if (flag_convF) + { + if (onpe0) os_ << "LBFGS: stopping ..." << endl; + break; + } + + // reset iterations if last step failed if (conv != 0) { if (onpe0) os_ << "LBFGS Geometry optimization stopped" << endl; - // break; if (onpe0) os_ << "LBFGS Geometry optimization reset" << endl; lbfgs.reset(ct.dt); } diff --git a/src/md.cc b/src/md.cc index 939c6e62..354f3f47 100644 --- a/src/md.cc +++ b/src/md.cc @@ -226,7 +226,7 @@ void checkMaxForces(const std::vector& fion, } template -int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, +int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -255,7 +255,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, { if (onpe0) (*MPIdata::serr) - << "dumprestartFile: cannot write ...previous_orbitals..." + << "dumpMDrestartFile: cannot write ...previous_orbitals..." << std::endl; return ierr; } @@ -269,7 +269,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, if (ierr < 0) { if (onpe0) - (*MPIdata::serr) << "dumprestartFile: cannot write " + (*MPIdata::serr) << "dumpMDrestartFile: cannot write " "...ExtrapolatedFunction..." << std::endl; return ierr; @@ -282,7 +282,7 @@ int MGmol::dumprestartFile(OrbitalsType** orbitals, Ions& ions, { if (onpe0) (*MPIdata::serr) - << "dumprestartFile: cannot close file..." << std::endl; + << "dumpMDrestartFile: cannot close file..." << std::endl; return ierr; } @@ -398,7 +398,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.restart_info < 3) { double eks = 0.; - quench(*orbitals, ions, ct.max_electronic_steps, 20, eks); + quench(**orbitals, ions, ct.max_electronic_steps, 20, eks); } ct.max_changes_pot = 0; @@ -426,7 +426,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) bool last_move_is_small = true; do { - retval = quench(*orbitals, ions, ct.max_electronic_steps, 0, eks); + retval = quench(**orbitals, ions, ct.max_electronic_steps, 0, eks); // update localization regions if (ct.adaptiveLRs()) @@ -576,18 +576,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) ct.steps = md_iteration_; -#if EXTRAPOLATE_RHO - if (onpe0) os_ << "Extrapolate rho..." << std::endl; - rho_->axpyRhoc(-1., rhoc_); - rho_->extrapolate(); -#endif - moveVnuc(ions); -#if EXTRAPOLATE_RHO - rho_->axpyRhoc(1., rhoc_); -#endif - if (!small_move) { orbitals_extrapol_->clearOldOrbitals(); @@ -624,12 +614,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) while (ierr < 0 && count < DUMP_MAX_NUM_TRY) { dump_tm_.start(); - ierr = dumprestartFile( + ierr = dumpMDrestartFile( orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) std::cout - << "dumprestartFile() failed... try again..." + << "dumpMDrestartFile() failed... try again..." << std::endl; if (ierr < 0) sleep(1.); count++; @@ -662,12 +652,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) while (ierr < 0 && count < DUMP_MAX_NUM_TRY) { dump_tm_.start(); - ierr = dumprestartFile( + ierr = dumpMDrestartFile( orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) - std::cout << "dumprestartFile() failed... try again..." + std::cout << "dumpMDrestartFile() failed... try again..." << std::endl; if (ierr < 0) sleep(1.); count++; @@ -681,8 +671,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) } template -void MGmol::loadRestartFile( - const std::string filename) +void MGmol::loadRestartFile(const std::string filename) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); Control& ct = *(Control::instance()); @@ -700,7 +689,9 @@ void MGmol::loadRestartFile( if (ierr < 0) { if (onpe0) - (*MPIdata::serr) << "loadRestartFile: failed to read the restart file." << std::endl; + (*MPIdata::serr) + << "loadRestartFile: failed to read the restart file." + << std::endl; global_exit(0); } diff --git a/src/quench.cc b/src/quench.cc index 614aa636..59a16d50 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -519,7 +519,7 @@ int MGmol::outerSolve(OrbitalsType& orbitals, } template -int MGmol::quench(OrbitalsType* orbitals, Ions& ions, +int MGmol::quench(OrbitalsType& orbitals, Ions& ions, const int max_inner_steps, const int iprint, double& last_eks) { assert(max_inner_steps > -1); @@ -543,33 +543,33 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, } // get actual indexes of stored functions - const std::vector>& gids(orbitals->getOverlappingGids()); + const std::vector>& gids(orbitals.getOverlappingGids()); g_kbpsi_->setup(*ions_); electrostat_->setup(ct.vh_its); rho_->setup(ct.getOrthoType(), gids); - OrbitalsType work_orbitals("Work", *orbitals); + OrbitalsType work_orbitals("Work", orbitals); - orbitals->setDataWithGhosts(); - orbitals->trade_boundaries(); + orbitals.setDataWithGhosts(); + orbitals.trade_boundaries(); - disentangleOrbitals(*orbitals, work_orbitals, ions, max_steps); + disentangleOrbitals(orbitals, work_orbitals, ions, max_steps); // setup "kernel" functions for AOMM algorithm if (ct.use_kernel_functions) { - applyAOMMprojection(*orbitals); + applyAOMMprojection(orbitals); } orbitals_precond_.reset(new OrbitalsPreconditioning()); orbitals_precond_->setup( - *orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); + orbitals, ct.getMGlevels(), ct.lap_type, currentMasks_.get(), lrs_); // solve electronic structure problem // (inner iterations) retval = outerSolve( - *orbitals, work_orbitals, ions, max_steps, iprint, last_eks); + orbitals, work_orbitals, ions, max_steps, iprint, last_eks); if (retval == -1) return -1; if (ct.use_kernel_functions) @@ -602,7 +602,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, } } last_eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), *orbitals, 2, os_); + = energy_->evaluateTotal(ts, proj_matrices_.get(), orbitals, 2, os_); if (ct.computeCondGramMD()) { @@ -615,7 +615,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, if (ct.isLocMode() || ct.isSpreadFunctionalActive()) { // build matrices necessary to compute spreads and centers - spreadf_->computePositionMatrix(*orbitals, work_orbitals); + spreadf_->computePositionMatrix(orbitals, work_orbitals); if (ct.verbose > 0 || !ct.AtomsMove()) { @@ -628,7 +628,7 @@ int MGmol::quench(OrbitalsType* orbitals, Ions& ions, { if (ct.wannier_transform_type >= 1) { - wftransform(orbitals, &work_orbitals, ions); + wftransform(&orbitals, &work_orbitals, ions); } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe7b571b..11125081 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -246,6 +246,10 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +add_executable(testEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) +add_executable(testWFEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -334,6 +338,21 @@ add_test(NAME testGramMatrix add_test(NAME testDensityMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDensityMatrix) +add_test(NAME testEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testWFEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testWFEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/sih4.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -518,6 +537,8 @@ target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX) target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) +target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/EnergyAndForces/coords.in b/tests/EnergyAndForces/coords.in new file mode 100644 index 00000000..cda898b9 --- /dev/null +++ b/tests/EnergyAndForces/coords.in @@ -0,0 +1,2 @@ +N1 1 0. 0. -1.0345 +N2 1 0. 0. 1.0345 diff --git a/tests/EnergyAndForces/lrs.in b/tests/EnergyAndForces/lrs.in new file mode 100644 index 00000000..9136a500 --- /dev/null +++ b/tests/EnergyAndForces/lrs.in @@ -0,0 +1,6 @@ +0.00 0.7 1.4 +0.00 -0.7 -1.4 +0.00 0.7 -1.4 +0.00 -0.7 1.4 +0.46 0.0 0.0 +-0.46 0.0 0.0 diff --git a/tests/EnergyAndForces/mgmol.cfg b/tests/EnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..cf23889e --- /dev/null +++ b/tests/EnergyAndForces/mgmol.cfg @@ -0,0 +1,33 @@ +verbosity=1 +xcFunctional=LDA +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.N_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-9 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=Gaussian +initial_width=1.5 +temperature=10. +nempty=1 +[Restart] +output_level=0 +[DensityMatrix] +mixing=0.5 diff --git a/tests/EnergyAndForces/test.py b/tests/EnergyAndForces/test.py new file mode 100755 index 00000000..e066bfc6 --- /dev/null +++ b/tests/EnergyAndForces/test.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test EnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp = sys.argv[nargs-4] +coords = sys.argv[nargs-3] +print("coordinates file: %s"%coords) +lrs = sys.argv[-2] + +#create links to potentials files +dst = 'pseudo.N_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run +command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp,coords,lrs) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + +flag=0 +forces=[] +for line in lines: + if flag>0: + print(line) + words=line.split(b' ') + forces.append(words[1].decode()) + forces.append(words[2].decode()) + forces.append(words[3].decode()) + flag=flag-1 + if line.count(b'Forces:'): + flag=2 + + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Check forces...") +print(forces) +flag=0 +for i in range(6): + diff=eval(forces[i+6])-eval(forces[i]) + print(diff) + if abs(diff)>1.e-3: + print("Forces difference larger than tol") + flag=1 +if flag>0: + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/EnergyAndForces/testEnergyAndForces.cc b/tests/EnergyAndForces/testEnergyAndForces.cc new file mode 100644 index 00000000..c8f7ab95 --- /dev/null +++ b/tests/EnergyAndForces/testEnergyAndForces.cc @@ -0,0 +1,198 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // move atoms one mesh spacing in all directions + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double hspacing[3] + = { mygrid.hgrid(0), mygrid.hgrid(1), mygrid.hgrid(2) }; + int i = 0; + for (auto& pos : positions) + { + pos += hspacing[i % 3]; + i++; + } + + // evaluate energy and forces again + eks = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/tests/MVP/test.py b/tests/MVP/test.py index 708fd45c..bd5d708e 100644 --- a/tests/MVP/test.py +++ b/tests/MVP/test.py @@ -70,6 +70,30 @@ print("Found HDF5 error") sys.exit(1) +flag = 0 +eigenvalues=[] +for line in lines: + if line.count(b'FERMI'): + flag = 0 + if flag==1: + words=line.split() + for w in words: + eigenvalues.append(eval(w)) + if line.count(b'Eigenvalues'): + flag = 1 + eigenvalues=[] + +print(eigenvalues) +tol = 1.e-4 +eigenvalue0 = -0.208 +if abs(eigenvalues[0]-eigenvalue0)>tol: + print("Expected eigenvalue 0 to be {}".format(eigenvalue0)) + sys.exit(1) +eigenvalue50 = 0.208 +if abs(eigenvalues[50]-eigenvalue50)>tol: + print("Expected eigenvalue 50 to be {}".format(eigenvalue50)) + sys.exit(1) + niterations = len(energies) print("MVP solver ran for {} iterations".format(niterations)) if niterations>180: diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..d543b626 --- /dev/null +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -0,0 +1,28 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=50 +atol=1.e-9 +num_lin_iterations=2 +[Orbitals] +initial_type=Gaussian +initial_width=2. +[Restart] +output_level=3 +output_filename=WF diff --git a/tests/WFEnergyAndForces/sih4.xyz b/tests/WFEnergyAndForces/sih4.xyz new file mode 100644 index 00000000..b3f921e3 --- /dev/null +++ b/tests/WFEnergyAndForces/sih4.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.885 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/WFEnergyAndForces/test.py b/tests/WFEnergyAndForces/test.py new file mode 100755 index 00000000..45420ddf --- /dev/null +++ b/tests/WFEnergyAndForces/test.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test WFEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-4): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-4] +inp = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.Si' +dst2 = 'pseudo.H' +src1 = sys.argv[nargs-1] + '/' + dst1 +src2 = sys.argv[nargs-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + +flag=0 +forces=[] +for line in lines: + if flag>0: + print(line) + words=line.split(b' ') + forces.append(words[1].decode()) + forces.append(words[2].decode()) + forces.append(words[3].decode()) + flag=flag-1 + if line.count(b'Forces:'): + flag=2 + + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Check forces...") +print(forces) +flag=0 +for i in range(6): + diff=eval(forces[i+6])-eval(forces[i]) + print(diff) + if abs(diff)>1.e-3: + print("Forces difference larger than tol") + flag=1 +if flag>0: + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc new file mode 100644 index 00000000..c9e28d48 --- /dev/null +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -0,0 +1,210 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + mgmol->dumpRestart(); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // compute energy and forces again using wavefunctions + // from previous call + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + const pb::PEenv& myPEenv = mymesh->peenv(); + HDFrestart h5file("WF", myPEenv, ct.out_restart_file_type); + orbitals.read_hdf5(h5file); + + // + // evaluate energy and forces again + // + + // convergence should be really quick since we start with an initial + // guess which is the solution + ct.max_electronic_steps = 5; + eks = mgmol->evaluateEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/util/generateLiLocalGTHpseudo.py b/util/generateLiLocalGTHpseudo.py new file mode 100644 index 00000000..1b553370 --- /dev/null +++ b/util/generateLiLocalGTHpseudo.py @@ -0,0 +1,65 @@ +# Generate local potential according to +# Goedecker, Teter, Hutter, Phys. rev. B 54 (3), 1996 +# Parameters from M. Krack, Theor. Chem. Acc. 114, 2005 +from math import exp, erf, sqrt, pi + +#coefficients for H +rloc = 0.4 +c1 = -14.081155 +c2 = 9.626220 +c3 = -1.783616 +c4 = 0.085152 +zion = 3. +anumber = 3 +name = "LithiumGTH_PBE" +mass = 3. + +def radialfunction(r): + alpha = (r/rloc)**2 + val = exp(-0.5*alpha)*(c1+c2*alpha+c3*alpha*alpha+c4*alpha*alpha*alpha) + if r>1.e-8: + val = val - zion*erf(r/(sqrt(2.)*rloc))/r + else: + #print("special case for r = {}".format(r)) + val = val -zion*sqrt(2.)/(sqrt(pi)*rloc) + + return val + +npts = 301 + +#header +print("# Short description of the species type. One line only!") +print(name) +print("#") +print("White") +print("#radii of balls and covalent bonds") +print("0.4 1.0") +print("# Nlcc flag") +print("0") +print("# Atomic number") +print(anumber) +print("# Atomic mass") +print(mass) +print("# Number of valence electrons") +print(zion) +print("# Gaussian core charge parameter rc") +print("1.") +print("# Number of potentials") +print("1") +print("# l-value for state which is local") +print("0 0") +print("# Local potential radius") +print("3.") +print("# Non-local potential radius") +print("3.") +print("# number of points in radial grid") +print(npts) +print("# log mesh parameter") +print("0.") +print("# radial grid, reference state, and potential for l=0") + +#potential +for i in range(npts): + r = round(0.01*i,2) + f = radialfunction(r) + print("{} {}".format(r,f)) From 1140370f41408ef9dd07393cdaba25bdafaf3ce7 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 20 Aug 2024 05:25:32 -0700 Subject: [PATCH 15/50] ROM offline calculations for MD snapshots (#272) * Add libROM snapshot frequency option * make install enabled * Add scripts * Add postprocessing script --- examples/PinnedH2O/job.basis | 32 +++++++++++++++++ examples/PinnedH2O/job.rom | 35 +++++++++++++++++++ examples/PinnedH2O/mgmol_rom.cfg | 38 +++++++++++++++++++++ examples/PinnedH2O/postprocess_PinnedH2O.py | 27 +++++++++++++++ scripts/build_quartz_libROM.sh | 2 +- src/Control.cc | 6 +++- src/md.cc | 26 ++++++++------ src/read_config.cc | 2 ++ src/rom_Control.h | 1 + 9 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 examples/PinnedH2O/job.basis create mode 100644 examples/PinnedH2O/job.rom create mode 100644 examples/PinnedH2O/mgmol_rom.cfg create mode 100644 examples/PinnedH2O/postprocess_PinnedH2O.py diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis new file mode 100644 index 00000000..8f8c79f6 --- /dev/null +++ b/examples/PinnedH2O/job.basis @@ -0,0 +1,32 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20240815 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set snapshot_files = "" +set increment_md_steps = 1 +set num_md_steps = 500 + +foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`) + set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot" +end +echo "Snapshot files: $snapshot_files" + +set basis_file = "PinnedH2O_orbitals_basis" + +srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out + +date diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom new file mode 100644 index 00000000..854c56d8 --- /dev/null +++ b/examples/PinnedH2O/job.rom @@ -0,0 +1,35 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20240815 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set cfg_rom = mgmol_rom.cfg +#cp $datadir/$cfg_rom . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_rom.cfg new file mode 100644 index 00000000..332dd77d --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom.cfg @@ -0,0 +1,38 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +save_librom_snapshot=true +librom_snapshot_freq=1 diff --git a/examples/PinnedH2O/postprocess_PinnedH2O.py b/examples/PinnedH2O/postprocess_PinnedH2O.py new file mode 100644 index 00000000..142a34ac --- /dev/null +++ b/examples/PinnedH2O/postprocess_PinnedH2O.py @@ -0,0 +1,27 @@ +import subprocess +import re + +print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" +for t in range(10): + k = 50*(t+1) + snapshots = 4*k + grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index 458dd662..7092f3d5 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -55,7 +55,7 @@ make -j 16 ### Currently libROM does not have the installation procedure, ### so copying binary file to installation directory will disrupt the relative path to libROM.so, ### causing a run-time error. -#make install +make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/src/Control.cc b/src/Control.cc index a798b5db..56272996 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2052,6 +2052,7 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.variable = ROMVariable::NONE; rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 @@ -2092,6 +2093,9 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); bcast_check(mpirc); + mpirc = MPI_Bcast(&rom_pri_option.librom_snapshot_freq, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + short rom_var = (short)static_cast(rom_pri_option.variable); mpirc = MPI_Bcast(&rom_var, 1, MPI_SHORT, 0, comm_global_); bcast_check(mpirc); @@ -2101,4 +2105,4 @@ void Control::syncROMOptions() mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); bcast_check(mpirc); -} \ No newline at end of file +} diff --git a/src/md.cc b/src/md.cc index 354f3f47..ed1e5969 100644 --- a/src/md.cc +++ b/src/md.cc @@ -406,6 +406,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) bool extrapolated_flag = true; if (ct.dt <= 0.) extrapolated_flag = false; + int librom_snapshot_freq = ct.getROMOptions().librom_snapshot_freq; + if (librom_snapshot_freq == -1) librom_snapshot_freq = ct.md_print_freq; + MDfiles md_files; // main MD iteration loop @@ -625,21 +628,22 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } + printWithTimeStamp("dumped restart file...", std::cout); + } + #ifdef MGMOL_HAS_LIBROM - // Save orbital snapshots - if (ct.getROMOptions().save_librom_snapshot > 0) - { - int ierr = save_orbital_snapshot( - ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + // Save orbital snapshots + if (md_iteration_ % librom_snapshot_freq == 0 + && ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); - if (ierr < 0) - os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; - } + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } #endif - printWithTimeStamp("dumped restart file...", std::cout); - } - md_iterations_tm.stop(); } // md loop diff --git a/src/read_config.cc b/src/read_config.cc index 14057553..40194678 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -426,6 +426,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) "File name for libROM snapshot/POD matrices.") ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), "Save libROM snapshot file at FOM simulation.") + ("ROM.offline.librom_snapshot_freq", po::value()->default_value(-1), + "Frequency of saving libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), diff --git a/src/rom_Control.h b/src/rom_Control.h index 2e4a1c71..67cb85a6 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -46,6 +46,7 @@ struct ROMPrivateOptions /* save librom snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; + int librom_snapshot_freq = -1; /* options for ROM building */ int num_potbasis = -1; From c40089b1a76b244bb4d989dced9b9637ad70fb94 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Mon, 16 Sep 2024 13:49:03 -0700 Subject: [PATCH 16/50] ROM hyper reduction for density evaluation (#268) * base for rho hyperreduction test * compute rho on sample points, on carom space. * computeRhoOnSamplePts checked. does not seem to have the factor 2.. * exact equivalence cannot be accomplished due to rescaleTotalCharge. adjusting tolerance. * save hyperreduction operator in build rom phase. --- src/Control.cc | 2 + src/MGmol.h | 6 +- src/Rho.h | 2 + src/rom_Control.h | 1 + src/rom_main.cc | 7 + src/rom_workflows.cc | 297 ++++++++++++++++++++++++++++++++++++++++++- src/rom_workflows.h | 9 ++ 7 files changed, 319 insertions(+), 5 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 56272996..2231813e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2035,6 +2035,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::BUILD; else if (str.compare("test_poisson") == 0) rom_pri_option.rom_stage = ROMStage::TEST_POISSON; + else if (str.compare("test_rho") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_RHO; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; diff --git a/src/MGmol.h b/src/MGmol.h index 0e70d6b7..ab07703d 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,10 +182,8 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() - { - return hamiltonian_; - } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/Rho.h b/src/Rho.h index ea5069da..11063914 100644 --- a/src/Rho.h +++ b/src/Rho.h @@ -71,6 +71,8 @@ class Rho Rho(); ~Rho(){}; + const OrthoType getOrthoType() { return orbitals_type_; } + void setup( const OrthoType orbitals_type, const std::vector>&); void setVerbosityLevel(const int vlevel) { verbosity_level_ = vlevel; } diff --git a/src/rom_Control.h b/src/rom_Control.h index 67cb85a6..8c3b4b2c 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -23,6 +23,7 @@ enum class ROMStage RESTORE, // TODO(kevin): what stage is this? BUILD, TEST_POISSON, + TEST_RHO, UNSUPPORTED }; diff --git a/src/rom_main.cc b/src/rom_main.cc index c54bcd37..fa956221 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -125,6 +125,13 @@ int main(int argc, char** argv) testROMPoissonOperator(mgmol); break; + case (ROMStage::TEST_RHO): + if (ct.isLocMode()) + testROMRhoOperator(mgmol); + else + testROMRhoOperator(mgmol); + break; + default: std::cerr << "rom_main error: Unknown ROM stage" << std::endl; MPI_Abort(MPI_COMM_WORLD, 0); diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index d9629c07..68349e90 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -13,6 +13,42 @@ #include #include +namespace CAROM { + +/* + This is not implemented in libROM yet. + A temporary placeholder until it is merged into libROM. +*/ +void mult(const Matrix &A, const std::vector &sample_row_A, const Matrix &B, Matrix &AB) +{ + CAROM_VERIFY(!B.distributed()); + CAROM_VERIFY(A.distributed() == AB.distributed()); + CAROM_VERIFY(A.numColumns() == B.numRows()); + CAROM_VERIFY(sample_row_A.size() <= A.numRows()); + + const int num_rows = sample_row_A.size(); + const int num_cols = B.numColumns(); + AB.setSize(num_rows, num_cols); + + // Do the multiplication. + const int Acol = A.numColumns(); + for (int r = 0; r < num_rows; ++r) { + double *d_Arow = A.getData() + sample_row_A[r] * Acol; + for (int c = 0; c < num_cols; ++c) { + double *d_Aptr = d_Arow; + double *d_Bptr = B.getData() + c; + double result_val = 0.0; + for (int ac = 0; ac < Acol; ++ac, d_Aptr++) { + result_val += (*d_Aptr) * (*d_Bptr); + d_Bptr += num_cols; + } + AB.item(r, c) = result_val; + } + } +} + +} + template std::string string_format( const std::string& format, Args ... args ) { @@ -107,6 +143,9 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); ROMPrivateOptions rom_options = ct.getROMOptions(); /* type of variable we intend to run POD */ @@ -162,6 +201,18 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) delete col; } // for (int c = 0; c < num_pot_basis; c++) + + /* DEIM hyperreduction */ + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + std::vector global_sampled_row(num_pot_basis), sampled_rows_per_proc(nprocs); + DEIM(pot_basis, num_pot_basis, global_sampled_row, sampled_rows_per_proc, + pot_rhs_rom, rank, nprocs); + + /* ROM rescaleTotalCharge operator */ + CAROM::Vector fom_ones(pot_basis->numRows(), true); + CAROM::Vector rom_ones(num_pot_basis, false); + fom_ones = mymesh->grid().vel(); // volume element + pot_basis->transposeMult(fom_ones, rom_ones); /* Save ROM operator */ // write the file from PE0 only @@ -179,10 +230,20 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) h5_helper.putDoubleArray("potential_rom_inverse", pot_rom.getData(), num_pot_basis * num_pot_basis, false); + /* save right-hand side hyper-reduction operator */ + h5_helper.putDoubleArray("potential_rhs_rom_inverse", pot_rhs_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save right-hand side rescaling operator */ + h5_helper.putDoubleArray("potential_rhs_rescaler", rom_ones.getData(), + num_pot_basis, false); + h5_helper.close(); } } +/* test routines */ + template void testROMPoissonOperator(MGmolInterface *mgmol_) { @@ -429,6 +490,237 @@ void testROMPoissonOperator(MGmolInterface *mgmol_) } } +template +void testROMRhoOperator(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + // if (ct.isLocMode()) + // printf("LocMode is On!\n"); + // else + // printf("LocMode is Off!\n"); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + std::shared_ptr> rho = mgmol->getRho(); + const OrthoType ortho_type = rho->getOrthoType(); + assert(ortho_type == OrthoType::Nonorthogonal); + + /* potential should have the same size as rho */ + const int dim = pot.size(); + + /* number of restart files, start/end indices */ + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + /* Initialize libROM classes */ + /* + In practice, we do not produce rho POD basis. + This rho POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_restart, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rho"); + + /* Collect the restart files */ + std::string filename; + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + mgmol->loadRestartFile(filename); + basis_generator.takeSample(&rho->rho_[0][0]); + } + // basis_generator.writeSnapshot(); + const CAROM::Matrix rho_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix *rho_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix *proj_rho = rho_basis->transposeMult(rho_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rho_basis_inv(num_restart, num_restart, false); + std::vector global_sampled_row(num_restart), sampled_rows_per_proc(nprocs); + DEIM(rho_basis, num_restart, global_sampled_row, sampled_rows_per_proc, + rho_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank+1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* load only the first restart file for now */ + const int test_idx = 2; + + filename = string_format(rom_options.restart_file_fmt, test_idx + minidx); + /* + currently, this does not update rho. + computeRhoOnSamplePts computes with the new density matrix, + while mgmol rho remains the same as the initial condition. + Commenting line 516 gives a consistent result, both for the initial condition. + */ + mgmol->loadRestartFile(filename); + + const int nrows = mymesh->locNumpt(); + // printf("mesh::locNumpt: %d\n", nrows); + + OrbitalsType *orbitals = mgmol->getOrbitals(); + // printf("orbitals::locNumpt: %d\n", orbitals->getLocNumpt()); + + /* NOTE(kevin): we assume we only use ProjectedMatrices class */ + // ProjectedMatrices> *proj_matrices = + // static_cast> *>(orbitals->getProjMatrices()); + ProjectedMatricesInterface *proj_matrices = orbitals->getProjMatrices(); + proj_matrices->updateSubMatX(); + SquareLocalMatrices& localX(proj_matrices->getLocalX()); + + // printf("localX nmat: %d\n", localX.nmat()); + // printf("localX n: %d\n", localX.n()); + // printf("localX m: %d\n", localX.m()); + + bool dm_distributed = (localX.nmat() > 1); + assert(!dm_distributed); + + /* copy density matrix */ + CAROM::Matrix dm(localX.getRawPtr(), localX.m(), localX.n(), dm_distributed, true); + + // /* random density matrix */ + // /* NOTE(kevin): Due to rescaleTotalCharge, the result is slightly inconsistent. */ + // CAROM::Matrix dm(localX.m(), localX.n(), dm_distributed, true); + // dm = 0.0; + // dist_matrix::DistMatrix dm_mgmol(proj_matrices->dm()); + // for (int i = 0; i < dm_mgmol.m(); i++) + // { + // for (int j = 0; j < dm_mgmol.m(); j++) + // { + // if (dm_mgmol.getVal(i, j) == 0.0) + // continue; + + // dm(i, j) = dm_mgmol.getVal(i, j) * (0.975 + 0.05 * ran0()); + // dm_mgmol.setVal(i, j, dm(i, j)); + // } + // } + + /* update rho first */ + // rho->computeRho(*orbitals, dm_mgmol); + // rho->update(*orbitals); + + const int chrom_num = orbitals->chromatic_number(); + CAROM::Matrix psi(dim, chrom_num, true); + for (int c = 0; c < chrom_num; c++) + { + ORBDTYPE *d_psi = orbitals->getPsi(c); + for (int d = 0; d < dim ; d++) + psi.item(d, c) = *(d_psi + d); + } + + CAROM::Matrix rom_psi(chrom_num, chrom_num, false); + for (int i = 0; i < chrom_num; i++) + for (int j = 0; j < chrom_num; j++) + rom_psi(i, j) = (i == j) ? 1 : 0; + + /* this will be resized in computeRhoOnSamplePts */ + CAROM::Vector sample_rho(1, true); + + computeRhoOnSamplePts(dm, psi, rom_psi, sampled_row, sample_rho); + + for (int s = 0; s < sampled_row.size(); s++) + { + const double error = abs(rho->rho_[0][sampled_row[s]] - sample_rho(s)); + if (error > 1.0e-4) + printf("rank %d, rho[%d]: %.5e, sample_rho: %.5e, librom_snapshot: %.5e\n", + rank, sampled_row[s], rho->rho_[0][sampled_row[s]], sample_rho(s), rho_snapshots(sampled_row[s], test_idx)); + CAROM_VERIFY(error < 1.0e-4); + } + + sample_rho.gather(); + + CAROM::Vector *rom_rho = rho_basis_inv.mult(sample_rho); + for (int d = 0; d < rom_rho->dim(); d++) + { + if ((rank == 0) && (abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) > 1.0e-3)) + printf("rom_rho error: %.3e\n", abs(proj_rho->item(d, test_idx) - rom_rho->item(d))); + CAROM_VERIFY(abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) < 1.0e-3); + } + + CAROM::Vector *fom_rho = rho_basis->mult(*rom_rho); + + CAROM_VERIFY(fom_rho->dim() == rho->rho_[0].size()); + for (int d = 0; d < fom_rho->dim(); d++) + CAROM_VERIFY(abs(fom_rho->item(d) - rho->rho_[0][d]) < 1.0e-4); + + delete rom_rho; + delete fom_rho; +} + +/* + dm: density matrix converted to CAROM::Matrix + phi_basis: POD basis matrix of orbitals, or orbitals themselves + rom_phi: ROM coefficients of POD Basis. If phi_basis is orbitals themselves, then simply an identity. + local_idx: sampled local grid indices on this processor. + sampled_rho: FOM density on sampled grid points. +*/ +void computeRhoOnSamplePts(const CAROM::Matrix &dm, + const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, + const std::vector &local_idx, CAROM::Vector &sampled_rho) +{ + assert(!dm.distributed()); + assert(sampled_rho.distributed() == phi_basis.distributed()); + + // this will be resized. + CAROM::Matrix sampled_phi(1, 1, phi_basis.distributed()); + + CAROM::mult(phi_basis, local_idx, rom_phi, sampled_phi); + + /* same product as in computeRhoSubdomainUsingBlas3 */ + CAROM::Matrix product(1, 1, sampled_phi.distributed()); + sampled_phi.mult(dm, product); + + sampled_rho.setSize(sampled_phi.numRows()); + double *d_product = product.getData(); + double *d_phi = sampled_phi.getData(); + for (int d = 0; d < sampled_rho.dim(); d++) + { + double val = 0.0; + /* CAROM Matrices are row-major */ + for (int c = 0; c < sampled_phi.numColumns(); c++, d_product++, d_phi++) + val += (*d_product) * (*d_phi); + + sampled_rho(d) = val; + } + + /* + TODO(kevin): need to figure out what these functions do. + and probably need to make ROM-equivalent functions with another hyper-reduction? + */ + // gatherSpin(); + + /* + rescaleTotalCharge is handled after hyperreduction. + */ + // rescaleTotalCharge(); +} + template void readRestartFiles(MGmolInterface *mgmol_); template void readRestartFiles(MGmolInterface *mgmol_); @@ -436,4 +728,7 @@ template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); -template void testROMPoissonOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMPoissonOperator(MGmolInterface *mgmol_); + +template void testROMRhoOperator(MGmolInterface *mgmol_); +template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h index 2399d065..fd00c687 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -12,6 +12,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" +#include "ProjectedMatrices.h" #include "LocGridOrbitals.h" #include "Potentials.h" #include "MGmol.h" @@ -36,6 +37,7 @@ namespace po = boost::program_options; #include "librom.h" #include "utils/HDFDatabase.h" +#include "utils/mpi_utils.h" template void readRestartFiles(MGmolInterface *mgmol_); @@ -46,4 +48,11 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); +template +void testROMRhoOperator(MGmolInterface *mgmol_); + +void computeRhoOnSamplePts(const CAROM::Matrix &dm, + const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, + const std::vector &local_idx, CAROM::Vector &sampled_rho); + #endif // ROM_WORKFLOWS_H From da33f942676e7a536743fff42573974f0690abb2 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sat, 21 Sep 2024 22:04:48 -0700 Subject: [PATCH 17/50] Fix CMakeList --- src/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 05cc45a1..e602101f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -208,7 +208,6 @@ if(${MGMOL_WITH_LIBXC}) endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) -<<<<<<< HEAD # build ROM executable if(USE_LIBROM) @@ -231,5 +230,3 @@ if(USE_LIBROM) endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-rom DESTINATION bin) endif(USE_LIBROM) -======= ->>>>>>> release From f1d42a312a1cde1f69bd766c1fe4c11c79d1d511 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 14:51:42 -0700 Subject: [PATCH 18/50] Fix test --- src/MGmol.cc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 085d952b..8da0d331 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1135,21 +1135,6 @@ void MGmol::dumpRestart() #endif } } -template -void MGmol::cleanup() -{ - closing_tm_.start(); - - Control& ct = *(Control::instance()); - - printTimers(); - - // Save data to restart file - if (!ct.AtomsMove()) - { - dumpRestart(); - } -} template void MGmol::cleanup() From f0e2d22399133c07ffba1895edd9dccbd34c05f2 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 19:49:43 -0700 Subject: [PATCH 19/50] Bypass build error --- src/rom_workflows.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 68349e90..5dc3718e 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -512,7 +512,7 @@ void testROMRhoOperator(MGmolInterface *mgmol_) MGmol *mgmol = static_cast *>(mgmol_); Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); Potentials& pot = mgmol->getHamiltonian()->potential(); - std::shared_ptr> rho = mgmol->getRho(); + std::shared_ptr> rho = NULL; // mgmol->getRho(); const OrthoType ortho_type = rho->getOrthoType(); assert(ortho_type == OrthoType::Nonorthogonal); @@ -731,4 +731,4 @@ template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); -template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMRhoOperator(MGmolInterface *mgmol_); From a03c891b19f8f888d89959e45230a4dfcbc61bed Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sun, 22 Sep 2024 20:03:06 -0700 Subject: [PATCH 20/50] Fix merge mistake --- src/MGmol.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/MGmol.h b/src/MGmol.h index 14e661fc..6a4cc830 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,10 +182,8 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() - { - return hamiltonian_; - } + std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; From a7478331b9fc73b6d99bb5dc7a0facb3b9a566d3 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Sat, 12 Oct 2024 14:25:05 -0700 Subject: [PATCH 21/50] Add verification of orbital projection --- examples/PinnedH2O/job.basis | 8 +++--- examples/PinnedH2O/job.rom | 2 +- examples/PinnedH2O/mgmol_rom.cfg | 2 +- src/MGmol.h | 1 + src/md.cc | 42 ++++++++++++++++++++++++++++++-- src/rom.cc | 34 ++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 8 deletions(-) diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis index 8f8c79f6..bf02a621 100644 --- a/examples/PinnedH2O/job.basis +++ b/examples/PinnedH2O/job.basis @@ -1,7 +1,7 @@ #!/bin/tcsh #SBATCH -N 2 #SBATCH -t 0:10:00 -#SBATCH -p pdebug +#SBATCH -p pbatch date @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240815 +set maindir = /p/lustre2/cheung26/mgmol-20240919 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -18,14 +18,14 @@ set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples set snapshot_files = "" set increment_md_steps = 1 -set num_md_steps = 500 +set num_md_steps = 50 foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`) set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot" end echo "Snapshot files: $snapshot_files" -set basis_file = "PinnedH2O_orbitals_basis" +set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}" srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom index 854c56d8..8de3d6b7 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.rom @@ -21,7 +21,7 @@ cp $maindir/install_quartz/bin/$exe . set datadir = $maindir/examples/PinnedH2O set cfg_rom = mgmol_rom.cfg -#cp $datadir/$cfg_rom . +cp $datadir/$cfg_rom . cp $datadir/coords.in . diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_rom.cfg index 332dd77d..1f724546 100644 --- a/examples/PinnedH2O/mgmol_rom.cfg +++ b/examples/PinnedH2O/mgmol_rom.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=500 +num_steps=50 dt=40. thermostat=ON [Thermostat] diff --git a/src/MGmol.h b/src/MGmol.h index 6a4cc830..78ef92d3 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -356,6 +356,7 @@ class MGmol : public MGmolInterface #ifdef MGMOL_HAS_LIBROM int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); + void project_orbital(std::string snapshot_dir, int rdim, OrbitalsType& orbitals); #endif }; // Instantiate static variables here to avoid clang warnings diff --git a/src/md.cc b/src/md.cc index ed1e5969..920ac1f3 100644 --- a/src/md.cc +++ b/src/md.cc @@ -492,6 +492,44 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) // Compute forces force(**orbitals, ions); +#ifdef MGMOL_HAS_LIBROM + // TODO: cleanup + int rdim = 39; + std::string basis_filename = "PinnedH2O_orbitals_basis_1_50"; + bool ROM_md = false; + + if (rdim > 0) + { + if (onpe0) + { + os_ << "Projecting orbitals onto ROM subspaces" << std::endl; + os_ << "Loading ROM basis " << basis_filename << std::endl; + os_ << "ROM basis dimension = " << rdim << std::endl; + } + project_orbital(basis_filename, rdim, **orbitals); + if (ROM_md) + { + force(**orbitals, ions); + } + else + { + double shift[3]; + for (short i = 0; i < 3; i++) shift[i] = 0.; + Ions ROM_ions(ions, shift); + force(**orbitals, ROM_ions); + std::string zero = "0"; + if (ions_->getNumIons() < 256 || ct.verbose > 2) + { + if (ct.verbose > 0) ROM_ions.printForcesGlobal(os_); + } + else if (zero.compare(ct.md_print_filename) == 0) + { + ROM_ions.printForcesLocal(os_); + } + } + } +#endif + // set fion ions.getLocalForces(fion); @@ -633,8 +671,8 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) #ifdef MGMOL_HAS_LIBROM // Save orbital snapshots - if (md_iteration_ % librom_snapshot_freq == 0 - && ct.getROMOptions().save_librom_snapshot > 0) + if (ct.getROMOptions().save_librom_snapshot > 0 && + md_iteration_ % librom_snapshot_freq == 0) { int ierr = save_orbital_snapshot( ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); diff --git a/src/rom.cc b/src/rom.cc index a554d2cf..19ff66a3 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -57,6 +57,40 @@ int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsTy return 0; } +template +void MGmol::project_orbital(std::string file_path, int rdim, OrbitalsType& orbitals) +{ + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, "foo"); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + const CAROM::Matrix* orbital_snapshots = basis_generator.getSnapshotMatrix(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + CAROM::Matrix* proj_orbital_coeff = orbital_basis->transposeMult(orbital_snapshots); + CAROM::Matrix* proj_orbital_snapshots = orbital_basis->mult(proj_orbital_coeff); + + Control& ct = *(Control::instance()); + Mesh* mesh = Mesh::instance(); + pb::GridFunc gf_psi(mesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector snapshot, proj_snapshot; + for (int i = 0; i < totalSamples; ++i) + { + orbital_snapshots->getColumn(i, snapshot); + proj_orbital_snapshots->getColumn(i, proj_snapshot); + gf_psi.assign(proj_snapshot.getData()); + orbitals.setPsi(gf_psi, i); + snapshot -= proj_snapshot; + std::cout << "Error for orbital " << i << " = " << snapshot.norm() << std::endl; + } +} + template class MGmol; template class MGmol; From e9c3807c4294486f1b7211b7eb88e3f9a9b54540 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 15 Oct 2024 12:11:15 -0700 Subject: [PATCH 22/50] Merge release (#276) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions * Add functionality to compute energy and forces (#270) * use specified wavefunctions as solution, with unknown DM * Add ONCV for Sulfur + example (#275) * Fix CMakeList * Fix test * Bypass build error * Fix merge mistake --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Kevin" Seung Whan Chung --- examples/S2/coords.in | 2 + examples/S2/mgmol.cfg | 23 + potentials/pseudo.S_ONCV_PBE_SG15 | 1921 +++++++++++++++++ src/DensityMatrix.cc | 9 +- src/MGmol.cc | 45 +- src/MGmol.h | 25 +- src/MGmolInterface.h | 10 +- src/MVPSolver.cc | 12 + src/ProjectedMatrices.cc | 5 +- src/Rho.cc | 6 +- src/rom_workflows.cc | 4 +- tests/CMakeLists.txt | 11 + tests/DMandEnergyAndForces/coords.in | 2 + tests/DMandEnergyAndForces/lrs.in | 6 + tests/DMandEnergyAndForces/mgmol.cfg | 35 + tests/DMandEnergyAndForces/test.py | 73 + .../testDMandEnergyAndForces.cc | 211 ++ 17 files changed, 2380 insertions(+), 20 deletions(-) create mode 100644 examples/S2/coords.in create mode 100644 examples/S2/mgmol.cfg create mode 100644 potentials/pseudo.S_ONCV_PBE_SG15 create mode 100644 tests/DMandEnergyAndForces/coords.in create mode 100644 tests/DMandEnergyAndForces/lrs.in create mode 100644 tests/DMandEnergyAndForces/mgmol.cfg create mode 100755 tests/DMandEnergyAndForces/test.py create mode 100644 tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc diff --git a/examples/S2/coords.in b/examples/S2/coords.in new file mode 100644 index 00000000..f2db9834 --- /dev/null +++ b/examples/S2/coords.in @@ -0,0 +1,2 @@ +S1 1 0. 0. 0. +S2 1 0. 0. 3.61 diff --git a/examples/S2/mgmol.cfg b/examples/S2/mgmol.cfg new file mode 100644 index 00000000..3e31f57f --- /dev/null +++ b/examples/S2/mgmol.cfg @@ -0,0 +1,23 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-8. +oy=-8. +oz=-8. +lx=16. +ly=16. +lz=16. +[Potentials] +pseudopotential=pseudo.S_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=200 +atol=1.e-8 +[Orbitals] +initial_width=2. diff --git a/potentials/pseudo.S_ONCV_PBE_SG15 b/potentials/pseudo.S_ONCV_PBE_SG15 new file mode 100644 index 00000000..93e0e237 --- /dev/null +++ b/potentials/pseudo.S_ONCV_PBE_SG15 @@ -0,0 +1,1921 @@ +# +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# S 16.00 3 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 6.00 +# 3 0 2.00 +# 3 1 4.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 1 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 2.12945 -0.63416 5 8 6.75573 +# 1 1.52480 -0.25757 5 8 8.63513 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 0.95758 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.22534 +# 1 2 1.65602 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +S_ONCV_PBE-1 +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +16 +# Atomic mass +32.06000137 +# Number of valence electrons +6 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +3 +# l-value for state which is local, then type of potential format +2 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 0.1035057429E+02 0.6748828187E+00 +1 2 0.9516274860E+01 0.2804708546E+01 +# l= 0 +0.0 3.7717412366 2.9189109498 +0.01 3.7719018776 2.9183509866 +0.02 3.7724026253 2.9166916915 +0.03 3.7732245023 2.9139122053 +0.04 3.774348531 2.9099916687 +0.05 3.7757482954 2.9049011428 +0.06 3.7773900923 2.898603871 +0.07 3.7792331245 2.8910556129 +0.08 3.7812297354 2.8822050481 +0.09 3.7833256829 2.8719942473 +0.1 3.7854604514 2.8603592078 +0.11 3.7875676008 2.8472304493 +0.12 3.7895751502 2.8325336672 +0.13 3.7914059948 2.8161904385 +0.14 3.7929783525 2.7981189755 +0.15 3.7942062396 2.7782349235 +0.16 3.7949999726 2.7564521953 +0.17 3.7952666921 2.7326838392 +0.18 3.7949109084 2.706842933 +0.19 3.7938350635 2.6788434987 +0.2 3.7919401088 2.6486014322 +0.21 3.7891260935 2.6160354399 +0.22 3.7852927623 2.5810679779 +0.23 3.7803401572 2.5436261857 +0.24 3.7741692231 2.5036428099 +0.25 3.766682409 2.4610571015 +0.26 3.7577842707 2.4158157069 +0.27 3.7473820587 2.3678735041 +0.28 3.7353862998 2.3171944159 +0.29 3.7217113638 2.2637521758 +0.3 3.7062760135 2.2075310435 +0.31 3.6890039342 2.1485264641 +0.32 3.6698242411 2.0867456663 +0.33 3.6486719506 2.0222081731 +0.34 3.6254884511 1.9549462973 +0.35 3.6002219022 1.8850054775 +0.36 3.5728276471 1.8124446249 +0.37 3.5432685235 1.7373362626 +0.38 3.5115152339 1.6597667516 +0.39 3.4775465763 1.5798362469 +0.4 3.4413496887 1.4976586593 +0.41 3.4029202406 1.4133615024 +0.42 3.3622625791 1.327085641 +0.43 3.319389829 1.2389849408 +0.44 3.2743239481 1.1492258211 +0.45 3.2270957347 1.0579867109 +0.46 3.1777447898 0.96545740865 +0.47 3.1263194312 0.87183834757 +0.48 3.0728765592 0.77733976783 +0.49 3.0174814494 0.68218077698 +0.5 2.9602075867 0.58658841233 +0.51 2.9011363835 0.49079653457 +0.52 2.8403567906 0.39504462368 +0.53 2.7779650276 0.29957662678 +0.54 2.7140641957 0.2046396408 +0.55 2.6487636899 0.11048255501 +0.56 2.5821789998 0.017354672646 +0.57 2.5144308728 -0.07449568823 +0.58 2.4456451103 -0.1648227841 +0.59 2.3759516826 -0.25338470387 +0.6 2.3054844173 -0.33994520629 +0.61 2.2343801679 -0.42427487726 +0.62 2.1627782898 -0.50615285932 +0.63 2.0908199666 -0.58536839417 +0.64 2.0186475097 -0.66172187533 +0.65 1.9464037206 -0.73502682527 +0.66 1.8742311772 -0.80511101968 +0.67 1.8022715715 -0.87181759713 +0.68 1.7306650412 -0.93500637095 +0.69 1.6595494083 -0.99455553892 +0.7 1.5890595921 -1.0503620586 +0.71 1.5193269353 -1.1023428144 +0.72 1.4504786116 -1.1504352703 +0.73 1.3826369115 -1.1945986146 +0.74 1.31591866 -1.2348142122 +0.75 1.2504347283 -1.2710857281 +0.76 1.1862894709 -1.3034395225 +0.77 1.123580233 -1.3319247288 +0.78 1.0623968961 -1.3566131489 +0.79 1.0028214682 -1.3775989572 +0.8 0.94492772055 -1.3949982017 +0.81 0.88878087471 -1.4089480982 +0.82 0.83443734235 -1.4196061062 +0.83 0.78194454126 -1.4271487482 +0.84 0.73134067628 -1.4317703672 +0.85 0.6826547342 -1.4336813557 +0.86 0.63590645531 -1.4331063392 +0.87 0.5911063603 -1.4302821209 +0.88 0.54825581874 -1.4254553729 +0.89 0.50734727773 -1.4188799828 +0.9 0.46836449306 -1.4108142305 +0.91 0.43128284695 -1.4015176891 +0.92 0.39606973745 -1.3912478723 +0.93 0.36268504025 -1.3802566269 +0.94 0.3310816433 -1.3687862735 +0.95 0.30120608287 -1.3570657607 +0.96 0.27299926125 -1.3453069539 +0.97 0.24639578856 -1.3336862799 +0.98 0.22132168405 -1.3223057818 +0.99 0.19769389367 -1.3111537553 +1.0 0.17542633582 -1.3001149815 +1.01 0.1544351692 -1.2889999625 +1.02 0.13464260071 -1.2775700083 +1.03 0.11597959258 -1.2655663263 +1.04 0.098386926129 -1.2527354371 +1.05 0.081815967788 -1.2388526331 +1.06 0.066227886294 -1.2237437341 +1.07 0.051590732077 -1.2072859225 +1.08 0.037877974845 -1.1894097625 +1.09 0.025066652728 -1.1700936969 +1.1 0.013135956759 -1.1493653828 +1.11 0.0020652968231 -1.1272870244 +1.12 -0.0081664103401 -1.1039449016 +1.13 -0.017581129962 -1.0794433933 +1.14 -0.026202272846 -1.0539001366 +1.15 -0.034054897507 -1.0274377592 +1.16 -0.041165298371 -1.0001795668 +1.17 -0.047560882987 -0.9722482902 +1.18 -0.053270364115 -0.94376212735 +1.19 -0.058323151395 -0.91483359715 +1.2 -0.062749148379 -0.88557010392 +1.21 -0.066578693596 -0.85607305284 +1.22 -0.069842280335 -0.8264354089 +1.23 -0.072570319662 -0.79674499202 +1.24 -0.074793187344 -0.76708482733 +1.25 -0.076540789321 -0.73752894997 +1.26 -0.077842647818 -0.70814740364 +1.27 -0.078728007273 -0.67900641864 +1.28 -0.079225164905 -0.65016418765 +1.29 -0.079361907267 -0.62167582309 +1.3 -0.079165594132 -0.59359326795 +1.31 -0.078662315233 -0.56596149123 +1.32 -0.077877693273 -0.5388234109 +1.33 -0.076836664008 -0.51221851122 +1.34 -0.075562844569 -0.48618085279 +1.35 -0.074079490886 -0.46074313072 +1.36 -0.072408738606 -0.43593375667 +1.37 -0.070571646522 -0.41177753661 +1.38 -0.068589032885 -0.38829837557 +1.39 -0.066479963161 -0.36551457197 +1.4 -0.064263165708 -0.34344348953 +1.41 -0.061956426491 -0.32209959305 +1.42 -0.059576161543 -0.30149349744 +1.43 -0.057138757575 -0.28163569015 +1.44 -0.054658681977 -0.26253144748 +1.45 -0.052150267116 -0.24418575291 +1.46 -0.049626683011 -0.22660053136 +1.47 -0.047100026817 -0.20977509975 +1.48 -0.044582198816 -0.19370822964 +1.49 -0.042083238373 -0.1783943207 +1.5 -0.039613494312 -0.16382848229 +1.51 -0.037181427181 -0.15000157131 +1.52 -0.034795503397 -0.13690452119 +1.53 -0.032462954533 -0.12452560427 +1.54 -0.030190365866 -0.11285187679 +1.55 -0.027983786923 -0.10186926883 +1.56 -0.025848056637 -0.09156155708 +1.57 -0.023788076285 -0.08191254844 +1.58 -0.02180709884 -0.072903372245 +1.59 -0.019908910212 -0.064515938844 +1.6 -0.018095394178 -0.056729413683 +1.61 -0.016369129845 -0.049523900811 +1.62 -0.01473094107 -0.042877342745 +1.63 -0.013182202875 -0.036768329307 +1.64 -0.011722836062 -0.031174028374 +1.65 -0.010353138167 -0.026071977824 +1.66 -0.0090722113073 -0.021438913179 +1.67 -0.0078794281674 -0.017251778077 +1.68 -0.0067731608474 -0.013487222834 +1.69 -0.0057520052889 -0.01012199958 +1.7 -0.0048136986886 -0.0071330256482 +1.71 -0.0039561989961 -0.0044972001989 +1.72 -0.0031767112554 -0.0021920616629 +1.73 -0.0024726810814 -0.0001949267295 +1.74 -0.0018408921881 0.0015156954421 +1.75 -0.0012783877013 0.0029618665928 +1.76 -0.0007816485767 0.0041637712608 +1.77 -0.00034740795298 0.0051427045896 +1.78 2.8026401829e-05 0.0059173024502 +1.79 0.00034816989119 0.0065077964019 +1.8 0.00061675389244 0.0069313251376 +1.81 0.00083746198141 0.0072065643935 +1.82 0.0010139835081 0.0073493701414 +1.83 0.0011500484087 0.0073764932044 +1.84 0.0012492723645 0.0073026182129 +1.85 0.0013152933449 0.0071422877869 +1.86 0.0013516442618 0.0069091520426 +1.87 0.0013617291682 0.0066153224449 +1.88 0.0013490054888 0.0062735408013 +1.89 0.0013165068112 0.0058933579291 +1.9 0.0012676271725 0.0054866792447 +1.91 0.001204999954 0.0050609209086 +1.92 0.0011316946665 0.0046258512343 +1.93 0.0010501549063 0.0041881793203 +1.94 0.00096292345948 0.0037549844277 +1.95 0.00087232492188 0.0033324956288 +1.96 0.0007802401781 0.0029250611317 +1.97 0.00068893170795 0.0025385304964 +1.98 0.00059963649799 0.0021750068783 +1.99 0.00051423875926 0.0018388258619 +2.0 0.00043370922442 0.0015313603799 +2.01 0.0003591837815 0.0012544582322 +2.02 0.0002916241788 0.0010094946409 +2.03 0.00023127892332 0.00079576300695 +2.04 0.0001791120696 0.00061453546295 +2.05 0.00013472582495 0.00046345775503 +2.06 9.8384817291e-05 0.00034188555777 +2.07 6.9768245347e-05 0.00024773278945 +2.08 4.8147423849e-05 0.00017790159289 +2.09 3.3341603917e-05 0.00013060236927 +2.1 2.3868358715e-05 0.00010101589887 +2.11 1.9371649837e-05 8.8269925068e-05 +2.12 1.7922694389e-05 8.4856271419e-05 +2.13 1.6497714121e-05 7.859897815e-05 +2.14 9.7420279224e-06 4.6912008392e-05 +2.15 2.9056005592e-06 1.4094314355e-05 +2.16 -8.0918591293e-07 -3.8523450141e-06 +2.17 -9.9639126312e-07 -4.743585934e-06 +2.18 -9.6753942982e-08 -4.6062290987e-07 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 -0.18215119847 -0.026814620552 +0.02 -0.36272317524 -0.052415548675 +0.03 -0.54015344747 -0.07560362298 +0.04 -0.71291283526 -0.095208589256 +0.05 -0.87952167778 -0.11010316826 +0.06 -1.0385655317 -0.11921666472 +0.07 -1.188710187 -0.12154797139 +0.08 -1.3287158419 -0.11617782803 +0.09 -1.4574502862 -0.10228020199 +0.1 -1.5739009532 -0.079132666269 +0.11 -1.67718571 -0.046125659777 +0.12 -1.7665622681 -0.0027705264466 +0.13 -1.8414361117 0.051293759259 +0.14 -1.9013668551 0.11629525692 +0.15 -1.9460729525 0.19232518352 +0.16 -1.9754347089 0.27933553968 +0.17 -1.989495546 0.37713828001 +0.18 -1.9884615105 0.48540603768 +0.19 -1.9726990125 0.6036744131 +0.2 -1.9427308156 0.73134580733 +0.21 -1.8992303133 0.86769476907 +0.22 -1.8430141422 1.0118748079 +0.23 -1.7750332038 1.1629266125 +0.24 -1.6963621806 1.3197875925 +0.25 -1.6081876946 1.4813026155 +0.26 -1.5117951026 1.6462359424 +0.27 -1.4085542777 1.8132840472 +0.28 -1.299904332 1.981089365 +0.29 -1.1873375371 2.1482547351 +0.3 -1.0723826049 2.3133583982 +0.31 -0.95658750373 2.4749693875 +0.32 -0.84150200799 2.6316631397 +0.33 -0.72866032208 2.7820370206 +0.34 -0.6195634148 2.9247260924 +0.35 -0.51566229021 3.0584180243 +0.36 -0.41834109033 3.1818681389 +0.37 -0.32890186775 3.2939129473 +0.38 -0.24854887804 3.3934841005 +0.39 -0.17837537049 3.4796200912 +0.4 -0.11935084639 3.5514775245 +0.41 -0.072309863841 3.6083409945 +0.42 -0.037942402868 3.6496315528 +0.43 -0.016785896057 3.6749136782 +0.44 -0.0092190212914 3.6839006618 +0.45 -0.015457336 3.6764583357 +0.46 -0.035550815344 3.6526070933 +0.47 -0.069383337328 3.6125221611 +0.48 -0.1166741332 3.5565321086 +0.49 -0.1769813265 3.4851154898 +0.5 -0.24970675592 3.3988963203 +0.51 -0.33410335936 3.2986372681 +0.52 -0.42928430785 3.1852312936 +0.53 -0.53423308634 3.0596924101 +0.54 -0.64781595293 2.9231443229 +0.55 -0.76879490854 2.7768086642 +0.56 -0.89584335116 2.6219906967 +0.57 -1.0275600348 2.4600667262 +0.58 -1.1624883636 2.2924665043 +0.59 -1.2991297631 2.1206611932 +0.6 -1.4359660279 1.946143015 +0.61 -1.5714725575 1.7704133204 +0.62 -1.7041389887 1.5949638382 +0.63 -1.83248722 1.4212603213 +0.64 -1.9550843518 1.2507308243 +0.65 -2.0705647191 1.0847456461 +0.66 -2.1776428361 0.9246055585 +0.67 -2.2751261301 0.77153019367 +0.68 -2.3619290324 0.62664516025 +0.69 -2.4370892058 0.4909671653 +0.7 -2.4997720928 0.36539976592 +0.71 -2.5492813476 0.25072369837 +0.72 -2.5850645145 0.1475915076 +0.73 -2.6067209504 0.056520054476 +0.74 -2.6140038528 -0.022111641313 +0.75 -2.6068196146 -0.088062154853 +0.76 -2.5852280088 -0.14122824405 +0.77 -2.5494395231 -0.18164288689 +0.78 -2.4998108504 -0.20947169362 +0.79 -2.4368386188 -0.22500776758 +0.8 -2.361151466 -0.22866511 +0.81 -2.2735005867 -0.22097068355 +0.82 -2.1747489012 -0.20255526698 +0.83 -2.0658589876 -0.17414321461 +0.84 -1.9478801281 -0.13654150961 +0.85 -1.8219341314 -0.090627645356 +0.86 -1.6892009619 -0.03733740677 +0.87 -1.5509038255 0.022347806514 +0.88 -1.4082937146 0.087414096918 +0.89 -1.2626343255 0.15682791127 +0.9 -1.1151869952 0.22954880156 +0.91 -0.96719607558 0.30454179611 +0.92 -0.81987491325 0.38078922285 +0.93 -0.67439262381 0.45730181877 +0.94 -0.53186183868 0.53312897122 +0.95 -0.39332757469 0.60736810359 +0.96 -0.25975734689 0.67917309919 +0.97 -0.13203329184 0.74775321949 +0.98 -0.01094906337 0.81235698313 +0.99 0.10279630832 0.87226236508 +1.0 0.20860467089 0.92679716672 +1.01 0.30597435445 0.97535396608 +1.02 0.39450484154 1.0174098802 +1.03 0.47389811631 1.0525426144 +1.04 0.54395642972 1.080438937 +1.05 0.60458713018 1.1009088448 +1.06 0.65580515498 1.11389353 +1.07 0.69771928596 1.119447206 +1.08 0.7305315527 1.1177346277 +1.09 0.75452833442 1.10901769 +1.1 0.77008388967 1.0936559167 +1.11 0.77764462483 1.0720831454 +1.12 0.77771729738 1.04479204 +1.13 0.77086206847 1.0123250096 +1.14 0.75768669294 0.97526556969 +1.15 0.73883116761 0.93422091892 +1.16 0.71495745184 0.88981251387 +1.17 0.68674624699 0.84267298854 +1.18 0.6548790065 0.79342705498 +1.19 0.62002916582 0.74268516285 +1.2 0.58286154107 0.6910437846 +1.21 0.54402441605 0.63907779944 +1.22 0.50412083068 0.58731389232 +1.23 0.46372675315 0.53624961293 +1.24 0.42339395221 0.48635589533 +1.25 0.38359561591 0.43802682011 +1.26 0.34477020037 0.39162193845 +1.27 0.30732463519 0.34746880189 +1.28 0.27157134852 0.30580606771 +1.29 0.23778707603 0.26683810864 +1.3 0.20621131665 0.23073313381 +1.31 0.17698834696 0.19757219659 +1.32 0.15023637673 0.16741147173 +1.33 0.12602499295 0.14026193142 +1.34 0.10435202907 0.1160705183 +1.35 0.085197164124 0.094766660847 +1.36 0.068484932802 0.076230851442 +1.37 0.054104389019 0.060313067757 +1.38 0.041928184436 0.046847269937 +1.39 0.031785627449 0.035632582704 +1.4 0.023500713689 0.026463074369 +1.41 0.016876824869 0.019116104017 +1.42 0.011714252485 0.013370408212 +1.43 0.0078034446545 0.0089945394479 +1.44 0.0049469134056 0.0057748945206 +1.45 0.0029474249359 0.0034976284065 +1.46 0.0016205346613 0.0019645451715 +1.47 0.00080702693374 0.0010042103105 +1.48 0.00034539687203 0.00044320077201 +1.49 0.00012966994735 0.00016608419693 +1.5 4.0352556958e-05 4.3207922912e-05 +1.51 2.2926656376e-05 1.1362596335e-05 +1.52 1.9476388882e-05 4.6382966744e-06 +1.53 1.5321466077e-05 5.2321743433e-06 +1.54 3.138718166e-06 9.0699570749e-07 +1.55 -1.4215409932e-06 -5.417831369e-07 +1.56 -3.2732566399e-07 -1.247516082e-07 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -2.8265571261E+01 +0.01 -2.2359855375E+01 +0.02 -2.0368958441E+01 +0.03 -2.0339038676E+01 +0.04 -2.0316254296E+01 +0.05 -2.0282114131E+01 +0.06 -2.0240247209E+01 +0.07 -2.0191868895E+01 +0.08 -2.0137404927E+01 +0.09 -2.0077087977E+01 +0.1 -2.0011108927E+01 +0.11 -1.9939651235E+01 +0.12 -1.9862895945E+01 +0.13 -1.9781019764E+01 +0.14 -1.9694191855E+01 +0.15 -1.9602570651E+01 +0.16 -1.9506300948E+01 +0.17 -1.9405511319E+01 +0.18 -1.9300311898E+01 +0.19 -1.9190792636E+01 +0.2 -1.9077022111E+01 +0.21 -1.8959047040E+01 +0.22 -1.8836892547E+01 +0.23 -1.8710563271E+01 +0.24 -1.8580045317E+01 +0.25 -1.8445309034E+01 +0.26 -1.8306312492E+01 +0.27 -1.8163005607E+01 +0.28 -1.8015334685E+01 +0.29 -1.7863247225E+01 +0.3 -1.7706696809E+01 +0.31 -1.7545647845E+01 +0.32 -1.7380080014E+01 +0.33 -1.7209992216E+01 +0.34 -1.7035406021E+01 +0.35 -1.6856368304E+01 +0.36 -1.6672953309E+01 +0.37 -1.6485263714E+01 +0.38 -1.6293431310E+01 +0.39 -1.6097616584E+01 +0.4 -1.5898007851E+01 +0.41 -1.5694819743E+01 +0.42 -1.5488291167E+01 +0.43 -1.5278682835E+01 +0.44 -1.5066274431E+01 +0.45 -1.4851361530E+01 +0.46 -1.4634252359E+01 +0.47 -1.4415264476E+01 +0.48 -1.4194721483E+01 +0.49 -1.3972949858E+01 +0.5 -1.3750275981E+01 +0.51 -1.3527023349E+01 +0.52 -1.3303510245E+01 +0.53 -1.3080047626E+01 +0.54 -1.2856937358E+01 +0.55 -1.2634471213E+01 +0.56 -1.2412929405E+01 +0.57 -1.2192580697E+01 +0.58 -1.1973681335E+01 +0.59 -1.1756475863E+01 +0.6 -1.1541196471E+01 +0.61 -1.1328064042E+01 +0.62 -1.1117288193E+01 +0.63 -1.0909067786E+01 +0.64 -1.0703591786E+01 +0.65 -1.0501039311E+01 +0.66 -1.0301580308E+01 +0.67 -1.0105376044E+01 +0.68 -9.9125793483E+00 +0.69 -9.7233346213E+00 +0.7 -9.5377783000E+00 +0.71 -9.3560388251E+00 +0.72 -9.1782367054E+00 +0.73 -9.0044842839E+00 +0.74 -8.8348856514E+00 +0.75 -8.6695365678E+00 +0.76 -8.5085241813E+00 +0.77 -8.3519267656E+00 +0.78 -8.1998134301E+00 +0.79 -8.0522438094E+00 +0.8 -7.9092677372E+00 +0.81 -7.7709249112E+00 +0.82 -7.6372445518E+00 +0.83 -7.5082450841E+00 +0.84 -7.3839337053E+00 +0.85 -7.2643061416E+00 +0.86 -7.1493463077E+00 +0.87 -7.0390259598E+00 +0.88 -6.9333043128E+00 +0.89 -6.8321277895E+00 +0.9 -6.7354296853E+00 +0.91 -6.6431298585E+00 +0.92 -6.5551344233E+00 +0.93 -6.4713354453E+00 +0.94 -6.3916106378E+00 +0.95 -6.3158232811E+00 +0.96 -6.2438223651E+00 +0.97 -6.1754307543E+00 +0.98 -6.1104172671E+00 +0.99 -6.0484745559E+00 +1.0 -5.9892424903E+00 +1.01 -5.9323378585E+00 +1.02 -5.8773782328E+00 +1.03 -5.8240043388E+00 +1.04 -5.7718954850E+00 +1.05 -5.7207836611E+00 +1.06 -5.6704621661E+00 +1.07 -5.6207763507E+00 +1.08 -5.5716202763E+00 +1.09 -5.5229278277E+00 +1.1 -5.4746697590E+00 +1.11 -5.4268397857E+00 +1.12 -5.3794464898E+00 +1.13 -5.3325087696E+00 +1.14 -5.2860519007E+00 +1.15 -5.2401023226E+00 +1.16 -5.1946856843E+00 +1.17 -5.1498262685E+00 +1.18 -5.1055447926E+00 +1.19 -5.0618585531E+00 +1.2 -5.0187819454E+00 +1.21 -4.9763261333E+00 +1.22 -4.9344983286E+00 +1.23 -4.8933035585E+00 +1.24 -4.8527447656E+00 +1.25 -4.8128214763E+00 +1.26 -4.7735319010E+00 +1.27 -4.7348728924E+00 +1.28 -4.6968388415E+00 +1.29 -4.6594233562E+00 +1.3 -4.6226191303E+00 +1.31 -4.5864172119E+00 +1.32 -4.5508082761E+00 +1.33 -4.5157822571E+00 +1.34 -4.4813282169E+00 +1.35 -4.4474350405E+00 +1.36 -4.4140910352E+00 +1.37 -4.3812842360E+00 +1.38 -4.3490025608E+00 +1.39 -4.3172336495E+00 +1.4 -4.2859651845E+00 +1.41 -4.2551847731E+00 +1.42 -4.2248801719E+00 +1.43 -4.1950391076E+00 +1.44 -4.1656496515E+00 +1.45 -4.1366999524E+00 +1.46 -4.1081784403E+00 +1.47 -4.0800739508E+00 +1.48 -4.0523753529E+00 +1.49 -4.0250723040E+00 +1.5 -3.9981543198E+00 +1.51 -3.9716118268E+00 +1.52 -3.9454352845E+00 +1.53 -3.9196158557E+00 +1.54 -3.8941450333E+00 +1.55 -3.8690146230E+00 +1.56 -3.8442169737E+00 +1.57 -3.8197444086E+00 +1.58 -3.7955900088E+00 +1.59 -3.7717466974E+00 +1.6 -3.7482079063E+00 +1.61 -3.7249668234E+00 +1.62 -3.7020174180E+00 +1.63 -3.6793534228E+00 +1.64 -3.6569691197E+00 +1.65 -3.6348586486E+00 +1.66 -3.6130166420E+00 +1.67 -3.5914376227E+00 +1.68 -3.5701165469E+00 +1.69 -3.5490482769E+00 +1.7 -3.5282280799E+00 +1.71 -3.5076511215E+00 +1.72 -3.4873129676E+00 +1.73 -3.4672090542E+00 +1.74 -3.4473352335E+00 +1.75 -3.4276871831E+00 +1.76 -3.4082610295E+00 +1.77 -3.3890526663E+00 +1.78 -3.3700584716E+00 +1.79 -3.3512745626E+00 +1.8 -3.3326975135E+00 +1.81 -3.3143236922E+00 +1.82 -3.2961498163E+00 +1.83 -3.2781725056E+00 +1.84 -3.2603885991E+00 +1.85 -3.2427949590E+00 +1.86 -3.2253885270E+00 +1.87 -3.2081663955E+00 +1.88 -3.1911255950E+00 +1.89 -3.1742634310E+00 +1.9 -3.1575770162E+00 +1.91 -3.1410638156E+00 +1.92 -3.1247211160E+00 +1.93 -3.1085464256E+00 +1.94 -3.0925372294E+00 +1.95 -3.0766910669E+00 +1.96 -3.0610056048E+00 +1.97 -3.0454784118E+00 +1.98 -3.0301072913E+00 +1.99 -3.0148899109E+00 +2.0 -2.9998241122E+00 +2.01 -2.9849077106E+00 +2.02 -2.9701385466E+00 +2.03 -2.9555145757E+00 +2.04 -2.9410336472E+00 +2.05 -2.9266938006E+00 +2.06 -2.9124929871E+00 +2.07 -2.8984292304E+00 +2.08 -2.8845006052E+00 +2.09 -2.8707051247E+00 +2.1 -2.8570409378E+00 +2.11 -2.8435061046E+00 +2.12 -2.8300987849E+00 +2.13 -2.8168171860E+00 +2.14 -2.8036593975E+00 +2.15 -2.7906238243E+00 +2.16 -2.7777087263E+00 +2.17 -2.7649124420E+00 +2.18 -2.7522333894E+00 +2.19 -2.7396698723E+00 +2.2 -2.7272204068E+00 +2.21 -2.7148834353E+00 +2.22 -2.7026574071E+00 +2.23 -2.6905409040E+00 +2.24 -2.6785324058E+00 +2.25 -2.6666305058E+00 +2.26 -2.6548338281E+00 +2.27 -2.6431409030E+00 +2.28 -2.6315504527E+00 +2.29 -2.6200611214E+00 +2.3 -2.6086715522E+00 +2.31 -2.5973805206E+00 +2.32 -2.5861867125E+00 +2.33 -2.5750888816E+00 +2.34 -2.5640858453E+00 +2.35 -2.5531763351E+00 +2.36 -2.5423592043E+00 +2.37 -2.5316333101E+00 +2.38 -2.5209974313E+00 +2.39 -2.5104505085E+00 +2.4 -2.4999914335E+00 +2.41 -2.4896190490E+00 +2.42 -2.4793323557E+00 +2.43 -2.4691302829E+00 +2.44 -2.4590117456E+00 +2.45 -2.4489757861E+00 +2.46 -2.4390213773E+00 +2.47 -2.4291474977E+00 +2.48 -2.4193532305E+00 +2.49 -2.4096375927E+00 +2.5 -2.3999996175E+00 +2.51 -2.3904384291E+00 +2.52 -2.3809530882E+00 +2.53 -2.3715426750E+00 +2.54 -2.3622063543E+00 +2.55 -2.3529432311E+00 +2.56 -2.3437524248E+00 +2.57 -2.3346331407E+00 +2.58 -2.3255845284E+00 +2.59 -2.3166057390E+00 +2.6 -2.3076960187E+00 +2.61 -2.2988545615E+00 +2.62 -2.2900805438E+00 +2.63 -2.2813732525E+00 +2.64 -2.2727319259E+00 +2.65 -2.2641557604E+00 +2.66 -2.2556440824E+00 +2.67 -2.2471961689E+00 +2.68 -2.2388112499E+00 +2.69 -2.2304886716E+00 +2.7 -2.2222277509E+00 +2.71 -2.2140277620E+00 +2.72 -2.2058880551E+00 +2.73 -2.1978079902E+00 +2.74 -2.1897868865E+00 +2.75 -2.1818240928E+00 +2.76 -2.1739190118E+00 +2.77 -2.1660710082E+00 +2.78 -2.1582794254E+00 +2.79 -2.1505437072E+00 +2.8 -2.1428632558E+00 +2.81 -2.1352374327E+00 +2.82 -2.1276656945E+00 +2.83 -2.1201474798E+00 +2.84 -2.1126821952E+00 +2.85 -2.1052692798E+00 +2.86 -2.0979082160E+00 +2.87 -2.0905984571E+00 +2.88 -2.0833394215E+00 +2.89 -2.0761306334E+00 +2.9 -2.0689715744E+00 +2.91 -2.0618616971E+00 +2.92 -2.0548005091E+00 +2.93 -2.0477875341E+00 +2.94 -2.0408222714E+00 +2.95 -2.0339041981E+00 +2.96 -2.0270328810E+00 +2.97 -2.0202078488E+00 +2.98 -2.0134286031E+00 +2.99 -2.0066946955E+00 +3.0 -2.0000056943E+00 +3.01 -1.9933611482E+00 +3.02 -1.9867605708E+00 +3.03 -1.9802035734E+00 +3.04 -1.9736897277E+00 +3.05 -1.9672185866E+00 +3.06 -1.9607897246E+00 +3.07 -1.9544027573E+00 +3.08 -1.9480572765E+00 +3.09 -1.9417528417E+00 +3.1 -1.9354890815E+00 +3.11 -1.9292656159E+00 +3.12 -1.9230820522E+00 +3.13 -1.9169379695E+00 +3.14 -1.9108330323E+00 +3.15 -1.9047668696E+00 +3.16 -1.8987390964E+00 +3.17 -1.8927493343E+00 +3.18 -1.8867972555E+00 +3.19 -1.8808825061E+00 +3.2 -1.8750047107E+00 +3.21 -1.8691635277E+00 +3.22 -1.8633586373E+00 +3.23 -1.8575897021E+00 +3.24 -1.8518563576E+00 +3.25 -1.8461582932E+00 +3.26 -1.8404951981E+00 +3.27 -1.8348667506E+00 +3.28 -1.8292725984E+00 +3.29 -1.8237124563E+00 +3.3 -1.8181860236E+00 +3.31 -1.8126929936E+00 +3.32 -1.8072330273E+00 +3.33 -1.8018058595E+00 +3.34 -1.7964112011E+00 +3.35 -1.7910487594E+00 +3.36 -1.7857182099E+00 +3.37 -1.7804193024E+00 +3.38 -1.7751517604E+00 +3.39 -1.7699153049E+00 +3.4 -1.7647096262E+00 +3.41 -1.7595344851E+00 +3.42 -1.7543896185E+00 +3.43 -1.7492747607E+00 +3.44 -1.7441896176E+00 +3.45 -1.7391339564E+00 +3.46 -1.7341075289E+00 +3.47 -1.7291100816E+00 +3.48 -1.7241413371E+00 +3.49 -1.7192010651E+00 +3.5 -1.7142890331E+00 +3.51 -1.7094049992E+00 +3.52 -1.7045487037E+00 +3.53 -1.6997199148E+00 +3.54 -1.6949184165E+00 +3.55 -1.6901439783E+00 +3.56 -1.6853963587E+00 +3.57 -1.6806753213E+00 +3.58 -1.6759806670E+00 +3.59 -1.6713121760E+00 +3.6 -1.6666696263E+00 +3.61 -1.6620527736E+00 +3.62 -1.6574614359E+00 +3.63 -1.6528954039E+00 +3.64 -1.6483544681E+00 +3.65 -1.6438383964E+00 +3.66 -1.6393470018E+00 +3.67 -1.6348800920E+00 +3.68 -1.6304374672E+00 +3.69 -1.6260189163E+00 +3.7 -1.6216242366E+00 +3.71 -1.6172532555E+00 +3.72 -1.6129057828E+00 +3.73 -1.6085816279E+00 +3.74 -1.6042805737E+00 +3.75 -1.6000024639E+00 +3.76 -1.5957471180E+00 +3.77 -1.5915143546E+00 +3.78 -1.5873039784E+00 +3.79 -1.5831158111E+00 +3.8 -1.5789496941E+00 +3.81 -1.5748054543E+00 +3.82 -1.5706829186E+00 +3.83 -1.5665818885E+00 +3.84 -1.5625022227E+00 +3.85 -1.5584437575E+00 +3.86 -1.5544063279E+00 +3.87 -1.5503897588E+00 +3.88 -1.5463938812E+00 +3.89 -1.5424185545E+00 +3.9 -1.5384636215E+00 +3.91 -1.5345289249E+00 +3.92 -1.5306142882E+00 +3.93 -1.5267195713E+00 +3.94 -1.5228446314E+00 +3.95 -1.5189893184E+00 +3.96 -1.5151534815E+00 +3.97 -1.5113369477E+00 +3.98 -1.5075395980E+00 +3.99 -1.5037612894E+00 +4.0 -1.5000018789E+00 +4.01 -1.4962612164E+00 +4.02 -1.4925391500E+00 +4.03 -1.4888355611E+00 +4.04 -1.4851503135E+00 +4.05 -1.4814832707E+00 +4.06 -1.4778342848E+00 +4.07 -1.4742032212E+00 +4.08 -1.4705899626E+00 +4.09 -1.4669943793E+00 +4.1 -1.4634163411E+00 +4.11 -1.4598557035E+00 +4.12 -1.4563123453E+00 +4.13 -1.4527861519E+00 +4.14 -1.4492769995E+00 +4.15 -1.4457847640E+00 +4.16 -1.4423093057E+00 +4.17 -1.4388505130E+00 +4.18 -1.4354082750E+00 +4.19 -1.4319824738E+00 +4.2 -1.4285729910E+00 +4.21 -1.4251796927E+00 +4.22 -1.4218024736E+00 +4.23 -1.4184412277E+00 +4.24 -1.4150958425E+00 +4.25 -1.4117662053E+00 +4.26 -1.4084521888E+00 +4.27 -1.4051536908E+00 +4.28 -1.4018706114E+00 +4.29 -1.3986028433E+00 +4.3 -1.3953502791E+00 +4.31 -1.3921127992E+00 +4.32 -1.3888903014E+00 +4.33 -1.3856826929E+00 +4.34 -1.3824898713E+00 +4.35 -1.3793117343E+00 +4.36 -1.3761481711E+00 +4.37 -1.3729990766E+00 +4.38 -1.3698643659E+00 +4.39 -1.3667439414E+00 +4.4 -1.3636377056E+00 +4.41 -1.3605455574E+00 +4.42 -1.3574673862E+00 +4.43 -1.3544031159E+00 +4.44 -1.3513526534E+00 +4.45 -1.3483159058E+00 +4.46 -1.3452927799E+00 +4.47 -1.3422831652E+00 +4.48 -1.3392869871E+00 +4.49 -1.3363041596E+00 +4.5 -1.3333345938E+00 +4.51 -1.3303782011E+00 +4.52 -1.3274348824E+00 +4.53 -1.3245045517E+00 +4.54 -1.3215871341E+00 +4.55 -1.3186825448E+00 +4.56 -1.3157906994E+00 +4.57 -1.3129115112E+00 +4.58 -1.3100448808E+00 +4.59 -1.3071907446E+00 +4.6 -1.3043490220E+00 +4.61 -1.3015196324E+00 +4.62 -1.2987024949E+00 +4.63 -1.2958975175E+00 +4.64 -1.2931046259E+00 +4.65 -1.2903237505E+00 +4.66 -1.2875548145E+00 +4.67 -1.2847977410E+00 +4.68 -1.2820524517E+00 +4.69 -1.2793188544E+00 +4.7 -1.2765968926E+00 +4.71 -1.2738864929E+00 +4.72 -1.2711875820E+00 +4.73 -1.2685000867E+00 +4.74 -1.2658239254E+00 +4.75 -1.2631590246E+00 +4.76 -1.2605053242E+00 +4.77 -1.2578627544E+00 +4.78 -1.2552312452E+00 +4.79 -1.2526107267E+00 +4.8 -1.2500011156E+00 +4.81 -1.2474023538E+00 +4.82 -1.2448143787E+00 +4.83 -1.2422371237E+00 +4.84 -1.2396705221E+00 +4.85 -1.2371145069E+00 +4.86 -1.2345689953E+00 +4.87 -1.2320339404E+00 +4.88 -1.2295092784E+00 +4.89 -1.2269949459E+00 +4.9 -1.2244908794E+00 +4.91 -1.2219970119E+00 +4.92 -1.2195132705E+00 +4.93 -1.2170396079E+00 +4.94 -1.2145759636E+00 +4.95 -1.2121222768E+00 +4.96 -1.2096784872E+00 +4.97 -1.2072445288E+00 +4.98 -1.2048203360E+00 +4.99 -1.2024058622E+00 +5.0 -1.2000010495E+00 +5.01 -1.1976058402E+00 +5.02 -1.1952201766E+00 +5.03 -1.1928439953E+00 +5.04 -1.1904772347E+00 +5.05 -1.1881198501E+00 +5.06 -1.1857717864E+00 +5.07 -1.1834329885E+00 +5.08 -1.1811034014E+00 +5.09 -1.1787829649E+00 +5.1 -1.1764716193E+00 +5.11 -1.1741693226E+00 +5.12 -1.1718760222E+00 +5.13 -1.1695916658E+00 +5.14 -1.1673162007E+00 +5.15 -1.1650495712E+00 +5.16 -1.1627917167E+00 +5.17 -1.1605425990E+00 +5.18 -1.1583021681E+00 +5.19 -1.1560703738E+00 +5.2 -1.1538471662E+00 +5.21 -1.1516324945E+00 +5.22 -1.1494262954E+00 +5.23 -1.1472285351E+00 +5.24 -1.1450391658E+00 +5.25 -1.1428581400E+00 +5.26 -1.1406854098E+00 +5.27 -1.1385209276E+00 +5.28 -1.1363646342E+00 +5.29 -1.1342164918E+00 +5.3 -1.1320764582E+00 +5.31 -1.1299444877E+00 +5.32 -1.1278205350E+00 +5.33 -1.1257045544E+00 +5.34 -1.1235964939E+00 +5.35 -1.1214963079E+00 +5.36 -1.1194039607E+00 +5.37 -1.1173194088E+00 +5.38 -1.1152426088E+00 +5.39 -1.1131735175E+00 +5.4 -1.1111120906E+00 +5.41 -1.1090582727E+00 +5.42 -1.1070120354E+00 +5.43 -1.1049733372E+00 +5.44 -1.1029421369E+00 +5.45 -1.1009183930E+00 +5.46 -1.0989020643E+00 +5.47 -1.0968931015E+00 +5.48 -1.0948914664E+00 +5.49 -1.0928971254E+00 +5.5 -1.0909100390E+00 +5.51 -1.0889301677E+00 +5.52 -1.0869574722E+00 +5.53 -1.0849919127E+00 +5.54 -1.0830334374E+00 +5.55 -1.0810820213E+00 +5.56 -1.0791376269E+00 +5.57 -1.0772002166E+00 +5.58 -1.0752697529E+00 +5.59 -1.0733461980E+00 +5.6 -1.0714295094E+00 +5.61 -1.0695196473E+00 +5.62 -1.0676165836E+00 +5.63 -1.0657202827E+00 +5.64 -1.0638307085E+00 +5.65 -1.0619478254E+00 +5.66 -1.0600715973E+00 +5.67 -1.0582019802E+00 +5.68 -1.0563389434E+00 +5.69 -1.0544824569E+00 +5.7 -1.0526324867E+00 +5.71 -1.0507889984E+00 +5.72 -1.0489519580E+00 +5.73 -1.0471213313E+00 +5.74 -1.0452970734E+00 +5.75 -1.0434791606E+00 +5.76 -1.0416675619E+00 +5.77 -1.0398622448E+00 +5.78 -1.0380631765E+00 +5.79 -1.0362703247E+00 +5.8 -1.0344836566E+00 +5.81 -1.0327031279E+00 +5.82 -1.0309287193E+00 +5.83 -1.0291603996E+00 +5.84 -1.0273981380E+00 +5.85 -1.0256419032E+00 +5.86 -1.0238916644E+00 +5.87 -1.0221473898E+00 +5.88 -1.0204090379E+00 +5.89 -1.0186765901E+00 +5.9 -1.0169500168E+00 +5.91 -1.0152292884E+00 +5.92 -1.0135143752E+00 +5.93 -1.0118052478E+00 +5.94 -1.0101018764E+00 +5.95 -1.0084042202E+00 +5.96 -1.0067122621E+00 +5.97 -1.0050259739E+00 +5.98 -1.0033453274E+00 +5.99 -1.0016702943E+00 +6.0 -1.0000008465E+00 +6.01 -9.9833695567E-01 diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 87b30224..6e6ea652 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -149,14 +149,11 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { -#ifdef PRINT_OPERATIONS - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) - std::cout << "template " - "DensityMatrix::setUniform()" + std::cout << "DensityMatrix::setUniform(), occupation = " << occ << std::endl; -#endif - const double occ = (double)((double)nel / (double)dim_); assert(occ < 1.01); for (int i = 0; i < dim_; i++) occupation_[i] = occ; diff --git a/src/MGmol.cc b/src/MGmol.cc index 5eb37c2c..8da0d331 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1135,6 +1135,7 @@ void MGmol::dumpRestart() #endif } } + template void MGmol::cleanup() { @@ -1443,7 +1444,7 @@ void MGmol::getAtomicNumbers(std::vector& an) template double MGmol::evaluateEnergyAndForces( - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) { return evaluateEnergyAndForces(current_orbitals_, tau, atnumbers, forces); @@ -1451,7 +1452,7 @@ double MGmol::evaluateEnergyAndForces( template double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) { assert(tau.size() == 3 * atnumbers.size()); @@ -1473,6 +1474,46 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, return eks; } +template +double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces) +{ + OrbitalsType* dorbitals = dynamic_cast(orbitals); + + ions_->setPositions(tau, atnumbers); + + moveVnuc(*ions_); + + // initialize electronic density + rho_->update(*dorbitals); + + // initialize potential + update_pot(*ions_); + + // initialize projected matrices + updateHmatrix(*dorbitals, *ions_); + proj_matrices_->updateThetaAndHB(); + + // compute DM + std::shared_ptr> dm_strategy( + DMStrategyFactory>::create(comm_, os_, *ions_, + rho_.get(), energy_.get(), electrostat_.get(), this, + proj_matrices_.get(), dorbitals)); + + dm_strategy->update(*dorbitals); + + // evaluate energy and forces + double ts = 0.; + double eks + = energy_->evaluateTotal(ts, proj_matrices_.get(), *dorbitals, 2, os_); + + force(*dorbitals, *ions_); + + return eks; +} + template class MGmol; template class MGmol; template int MGmol::initial(); diff --git a/src/MGmol.h b/src/MGmol.h index ab07703d..6a4cc830 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -187,11 +187,30 @@ class MGmol : public MGmolInterface void run() override; + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input) + */ double evaluateEnergyAndForces(const std::vector& tau, - std::vector& atnumbers, std::vector& forces); + const std::vector& atnumbers, std::vector& forces); - double evaluateEnergyAndForces(Orbitals*, const std::vector& tau, - std::vector& atnumbers, std::vector& forces); + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input), using the input "orbitals" as initial + * guess for the wavefunctions + */ + double evaluateEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces); + + /* + * Evaluate the energy and forces for an atomic configuration + * specified by tau (input), using the input "orbitals" as wavefunctions + * (fixed) + */ + double evaluateDMandEnergyAndForces(Orbitals* orbitals, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces); /* * get internal atomic positions diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index af7be67b..9a9bf8a6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -25,13 +25,19 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual double evaluateEnergyAndForces(const std::vector& tau, - std::vector& atnumbers, std::vector& forces) + const std::vector& atnumbers, std::vector& forces) = 0; virtual double evaluateEnergyAndForces(Orbitals*, - const std::vector& tau, std::vector& atnumbers, + const std::vector& tau, const std::vector& atnumbers, + std::vector& forces) + = 0; + virtual double evaluateDMandEnergyAndForces(Orbitals*, + const std::vector& tau, const std::vector& atnumbers, std::vector& forces) = 0; + virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; virtual std::shared_ptr getProjectedMatrices() diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 7feb564d..6115f933 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -208,6 +208,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) kbpsi.computeHvnlMatrix(&kbpsi, ions_, h11_nl); + const double tol_de0 = 1.e-12; for (int inner_it = 0; inner_it < n_inner_steps_; inner_it++) { if (onpe0 && ct.verbose > 1) @@ -270,6 +271,14 @@ int MVPSolver::solve(OrbitalsType& orbitals) double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + if (std::abs(de0) < tol_de0 && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + // // evaluate free energy at beta=1 // @@ -306,6 +315,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) // line minimization const double beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); + assert(!std::isnan(beta)); if (onpe0 && ct.verbose > 0) { @@ -348,6 +358,8 @@ int MVPSolver::solve(OrbitalsType& orbitals) orbitals.getProjMatrices()); projmatrices->setDM(*work_, orbitals.getIterativeIndex()); projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); + projmatrices->assignH(proj_mat_work_->getH()); + projmatrices->setHB2H(); } // Generate new density diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index bdfe7786..e0e679e9 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -650,8 +650,9 @@ double ProjectedMatrices::computeEntropy() else { if (mmpi.PE0() && ct.verbose > 1) - (*MPIdata::sout) - << "occupations uptodate, skip computation..." << std::endl; + (*MPIdata::sout) << "computeEntropy: occupations uptodate, " + "skip computation..." + << std::endl; } entropy = computeEntropy(width_); } diff --git a/src/Rho.cc b/src/Rho.cc index ca0f20ea..45a41f3e 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -90,8 +90,8 @@ void Rho::update(OrbitalsType& current_orbitals) update_tm_.start(); - if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Rho::update()" << std::endl; + if (verbosity_level_ > 1 && onpe0) + (*MPIdata::sout) << "Rho::update()..." << std::endl; const int new_iterative_index = ((1 + current_orbitals.getIterativeIndex()) % 100) @@ -99,7 +99,7 @@ void Rho::update(OrbitalsType& current_orbitals) if (iterative_index_ == new_iterative_index) { - if (onpe0 && verbosity_level_ > 2) + if (onpe0 && verbosity_level_ > 1) (*MPIdata::sout) << "Rho already up to date, iterative_index_=" << iterative_index_ << std::endl; return; diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 68349e90..5dc3718e 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -512,7 +512,7 @@ void testROMRhoOperator(MGmolInterface *mgmol_) MGmol *mgmol = static_cast *>(mgmol_); Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); Potentials& pot = mgmol->getHamiltonian()->potential(); - std::shared_ptr> rho = mgmol->getRho(); + std::shared_ptr> rho = NULL; // mgmol->getRho(); const OrthoType ortho_type = rho->getOrthoType(); assert(ortho_type == OrthoType::Nonorthogonal); @@ -731,4 +731,4 @@ template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); -template void testROMRhoOperator(MGmolInterface *mgmol_); \ No newline at end of file +template void testROMRhoOperator(MGmolInterface *mgmol_); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 11125081..cc205167 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,6 +250,8 @@ add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) add_executable(testWFEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) +add_executable(testDMandEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -353,6 +355,14 @@ add_test(NAME testWFEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/WFEnergyAndForces/sih4.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testDMandEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testDMandEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -539,6 +549,7 @@ target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/DMandEnergyAndForces/coords.in b/tests/DMandEnergyAndForces/coords.in new file mode 100644 index 00000000..cda898b9 --- /dev/null +++ b/tests/DMandEnergyAndForces/coords.in @@ -0,0 +1,2 @@ +N1 1 0. 0. -1.0345 +N2 1 0. 0. 1.0345 diff --git a/tests/DMandEnergyAndForces/lrs.in b/tests/DMandEnergyAndForces/lrs.in new file mode 100644 index 00000000..9136a500 --- /dev/null +++ b/tests/DMandEnergyAndForces/lrs.in @@ -0,0 +1,6 @@ +0.00 0.7 1.4 +0.00 -0.7 -1.4 +0.00 0.7 -1.4 +0.00 -0.7 1.4 +0.46 0.0 0.0 +-0.46 0.0 0.0 diff --git a/tests/DMandEnergyAndForces/mgmol.cfg b/tests/DMandEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..5278f40b --- /dev/null +++ b/tests/DMandEnergyAndForces/mgmol.cfg @@ -0,0 +1,35 @@ +verbosity=2 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.N_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-8 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=Gaussian +initial_width=1.5 +temperature=10. +nempty=2 +[Restart] +output_level=3 +output_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=1 diff --git a/tests/DMandEnergyAndForces/test.py b/tests/DMandEnergyAndForces/test.py new file mode 100755 index 00000000..c36f0767 --- /dev/null +++ b/tests/DMandEnergyAndForces/test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test DMandEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp = sys.argv[nargs-4] +coords = sys.argv[nargs-3] +print("coordinates file: %s"%coords) +lrs = sys.argv[-2] + +#create links to potentials files +dst = 'pseudo.N_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run +command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp,coords,lrs) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +energies=[] +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + energies.append(energy) + break + +for line in lines: + if line.count(b'MVP') and line.count(b'iteration'): + print(line) + if line.count(b'Eks2'): + print(line) + words=line.split() + word=words[2] + energy = word.decode() + energies.append(energy) + break + +print("Check energies...") +print( energies ) +if len(energies)<2: + print("Expected two converged energies") + sys.exit(1) + +tol = 1.e-6 +diff=eval(energies[1])-eval(energies[0]) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(energies[0],energies[1])) + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc new file mode 100644 index 00000000..1cf1cf88 --- /dev/null +++ b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + else + mgmol = new MGmol(global_comm, *MPIdata::sout, + input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // compute energy and forces using all MPI tasks + // expect positions to be replicated on all MPI tasks + std::vector forces; + double eks + = mgmol->evaluateEnergyAndForces(positions, anumbers, forces); + mgmol->dumpRestart(); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks1 : " << eks << std::endl; + std::cout << "Forces2 :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + // compute energy and forces again using wavefunctions + // from previous call + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + const pb::PEenv& myPEenv = mymesh->peenv(); + HDFrestart h5file("WF", myPEenv, ct.out_restart_file_type); + orbitals.read_hdf5(h5file); + + // + // evaluate energy and forces again, with wavefunctions + // frozen to solution of previous problem + // + + // reset initial DM to test iterative solve for it + projmatrices->setDMuniform(ct.getNelSpin(), 0); + ct.dm_inner_steps = 50; + eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks2 : " << eks << std::endl; + std::cout << "Forces2 :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From b32d6868f87179afd825a9d2450262ddba3b7f9e Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 15 Oct 2024 12:42:50 -0700 Subject: [PATCH 23/50] Merge release 2 (#283) * Reenable testShortSighted test (#248) * Make new driver to check input (#247) * clean up/reorganize main.cc * use shared_ptr in class MGmol * Add possible periodic dimensions to xyz2in.py (#249) * Add possible periodic dimensions to xyz2in.py * Remove unused/untested option extrapolateH (#250) * Exit with failure if density off by more than 2% (#251) * Exit with failure if density off by more than 2% * adapt SiH4 test to catch that * fix bug in DFTsolver that was leading to wrong density * Example driver (#252) * add example driver, showing use of MGmol as a force/energy computational engine * clean up related functions in class Ions * loadOrbitalsFromRestartFile -> loadRestartFile (#253) * Add SG15 PBE potential for N (#258) * Update 2-pyridone example (#259) * Adjust verbosity in some functions (#260) * Add new example: pinned H2O (#261) * Print out eigenvalues out of MVP solver (#262) Previously, the wrong eigenvalues (0) were printed out because eigenvalues outside solver were not up-to-date. * Add Li2 example with local GTH potential (#263) * Fix LBFGS termination when converged (#264) * Remove unused code to extrapolate rho (#265) * Fix and test EnergyAndForces interface (#266) * Atomic potentials were not updated when atomic positions were changed * Added test to make sure energies and forces are the same after positions move by one mesh spacing * Add new functionality to compute energy and forces (#267) * use specified initial conditions for wavefunctions * Add functionality to compute energy and forces (#270) * use specified wavefunctions as solution, with unknown DM * Add ONCV for Sulfur + example (#275) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) --------- Co-authored-by: Jean-Luc Fattebert --- src/Control.cc | 1 + src/DensityMatrix.cc | 7 +- src/DensityMatrix.h | 13 +- src/Electrostatic.cc | 200 +----------------- src/Electrostatic.h | 1 + src/Hartree.cc | 9 +- src/Hartree.h | 2 - src/Hartree_CG.cc | 1 + src/Ions.cc | 324 ++++++++++++------------------ src/Ions.h | 23 ++- src/KBprojectorSparse.cc | 4 +- src/MGmol.cc | 2 +- src/MGmol.h | 5 +- src/PCGSolver.cc | 59 +++--- src/PCGSolver.h | 52 +++-- src/PCGSolver_Diel.cc | 68 +++---- src/PCGSolver_Diel.h | 30 +-- src/PoissonSolverFactory.h | 228 +++++++++++++++++++++ src/Potentials.cc | 24 +-- src/Potentials.h | 56 ++++-- src/ProjectedMatrices.cc | 1 + src/pb/Mgm.h | 3 - src/pb/Solver.h | 7 +- src/pb/SolverLap.cc | 77 +------ src/pb/SolverLap.h | 2 +- src/pb/SolverPB.cc | 5 +- src/pb/SolverPB.h | 4 +- src/tools.cc | 17 +- src/tools.h | 1 + tests/CMakeLists.txt | 8 + tests/WFEnergyAndForces/mgmol.cfg | 2 +- tests/testIons.cc | 104 ++++++++++ 32 files changed, 704 insertions(+), 636 deletions(-) create mode 100644 src/PoissonSolverFactory.h create mode 100644 tests/testIons.cc diff --git a/src/Control.cc b/src/Control.cc index 2231813e..f1dcef23 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -70,6 +70,7 @@ Control::Control() dm_approx_ndigits = 1; dm_approx_power_maxits = 100; wf_extrapolation_ = 1; + verbose = 0; // undefined values dm_algo_ = -1; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 6e6ea652..e2c7b473 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -28,7 +28,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) { - assert(ndim >= 0); + assert(ndim > 0); dim_ = ndim; @@ -45,6 +45,7 @@ DensityMatrix::DensityMatrix(const int ndim) kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); occupation_.resize(dim_); + setDummyOcc(); } template @@ -109,6 +110,7 @@ void DensityMatrix::build( const std::vector& occ, const int new_orbitals_index) { assert(dm_ != nullptr); + assert(!occ.empty()); setOccupations(occ); @@ -149,6 +151,8 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { + assert(!occupation_.empty()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) @@ -314,6 +318,7 @@ void DensityMatrix::computeOccupations(const MatrixType& ls) template void DensityMatrix::setOccupations(const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (mmpi.instancePE0()) diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 960ad18e..84804a86 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -84,17 +84,24 @@ class DensityMatrix *dm_ = mat; orbitals_index_ = orbitals_index; - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; stripped_ = false; } + // set occupations to meaningless values to catch uninitialized use + void setDummyOcc() + { + for (auto& occ : occupation_) + occ = -1.; + } + void initMatrix(const double* const val) { dm_->init(val, dim_); - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; @@ -105,8 +112,10 @@ class DensityMatrix void getOccupations(std::vector& occ) const { + assert(!occupation_.empty()); assert(occ_uptodate_); assert((int)occ.size() == dim_); + memcpy(&occ[0], &occupation_[0], dim_ * sizeof(double)); } diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 80cb754b..6ac4fef3 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -11,6 +11,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "GridFactory.h" +#include "GridFunc.h" #include "Hartree.h" #include "Hartree_CG.h" #include "Ions.h" @@ -23,15 +24,6 @@ #include "ShiftedHartree.h" #include "mputils.h" -#include "GridFunc.h" -#include "Laph2.h" -#include "Laph4.h" -#include "Laph4M.h" -#include "Laph4MP.h" -#include "Laph6.h" -#include "Laph8.h" -#include "ShiftedLaph4M.h" - Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], @@ -49,109 +41,9 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Mesh* mymesh = Mesh::instance(); const pb::Grid& myGrid = mymesh->grid(); - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } - else // use PCG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "PCG Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "PCG Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } + // create Poisson solver + poisson_solver_ = PoissonSolverFactory::create( + myGrid, lap_type, bcPoisson, screening_const); grhoc_ = nullptr; diel_flag_ = false; @@ -244,73 +136,8 @@ void Electrostatic::setupPB( ngpts, origin, cell, static_cast(laptype_), true, myPEenv); if (poisson_solver_ != nullptr) delete poisson_solver_; - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } - else // use PCG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } + poisson_solver_ = PoissonSolverFactory::createDiel( + *pbGrid_, laptype_, bc_, e0, rho0, drho0); if (grhoc_ != nullptr) { @@ -330,6 +157,7 @@ void Electrostatic::setupPB( poisson_solver_->set_vh(gf_vh); } +// This function is only useful for Hartree problem with dielectric continuum void Electrostatic::fillFuncAroundIons(const Ions& ions) { assert(grhod_ != nullptr); @@ -352,7 +180,6 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) std::vector::const_iterator ion = rc_ions.begin(); while (ion != rc_ions.end()) { - double rc = (*ion)->getRC(); // Special case: silicon if ((*ion)->isMass28()) rc = 2.0; @@ -373,43 +200,32 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) #endif for (unsigned int ix = 0; ix < pbGrid_->dim(0); ix++) { - xc[1] = pbGrid_->start(1); const int ix1 = (ix + shift) * incx; for (unsigned int iy = 0; iy < pbGrid_->dim(1); iy++) { - xc[2] = pbGrid_->start(2); const int iy1 = ix1 + (iy + shift) * incy; for (unsigned int iz = 0; iz < pbGrid_->dim(2); iz++) { - const double r = (*ion)->minimage(xc, lattice, bc_); if (r < rc) { const double alpha = 0.2 * (1. + cos(r * pi_rc)); - - const int iz1 = iy1 + iz + shift; + const int iz1 = iy1 + iz + shift; vv[iz1] += alpha; } - xc[2] += pbGrid_->hgrid(2); } - xc[1] += pbGrid_->hgrid(1); - } // end for iy - xc[0] += pbGrid_->hgrid(0); - } // end for ix } - ion++; - } // end loop on list of ions return; diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 785bc909..7ae977e1 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -13,6 +13,7 @@ #include "Control.h" #include "GridFunc.h" #include "Poisson.h" +#include "PoissonSolverFactory.h" #include "Rho.h" #include "Timer.h" diff --git a/src/Hartree.cc b/src/Hartree.cc index b3984b87..7d0ba4d8 100644 --- a/src/Hartree.cc +++ b/src/Hartree.cc @@ -7,10 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #include #include -using namespace std; #include "Control.h" #include "Hartree.h" @@ -82,6 +80,8 @@ void Hartree::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + // Hartree units + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else @@ -92,10 +92,11 @@ void Hartree::solve( double final_residual = poisson_solver_->getFinalResidual(); if (onpe0) - (*MPIdata::sout) << setprecision(2) << scientific + (*MPIdata::sout) << std::setprecision(2) << std::scientific << "Hartree: residual reduction = " << residual_reduction - << ", final residual = " << final_residual << endl; + << ", final residual = " << final_residual + << std::endl; Poisson::Int_vhrho_ = vel * Poisson::vh_->gdot(rho); Poisson::Int_vhrhoc_ = vel * Poisson::vh_->gdot(rhoc); diff --git a/src/Hartree.h b/src/Hartree.h index a7050e24..b87ff01e 100644 --- a/src/Hartree.h +++ b/src/Hartree.h @@ -7,14 +7,12 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #ifndef included_Hartree #define included_Hartree #include "Poisson.h" #include "PoissonInterface.h" -// pb #include "SolverLap.h" template diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index f9bf3860..375237f8 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -81,6 +81,7 @@ void Hartree_CG::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else diff --git a/src/Ions.cc b/src/Ions.cc index 940e12e8..852151c7 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -584,11 +584,9 @@ void Ions::printPositionsLocal(std::ostream& os, const int root) const os.setf(std::ios::right, std::ios::adjustfield); os.setf(std::ios::fixed, std::ios::floatfield); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->printPosition(os); - ion++; + ion->printPosition(os); } os << std::endl; @@ -627,14 +625,12 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - assert((*ion)->atomic_number() > 0); - assert((*ion)->atomic_number() < 200); + assert(ion->atomic_number() > 0); + assert(ion->atomic_number() < 200); - data.push_back((*ion)->atomic_number()); - ion++; + data.push_back(ion->atomic_number()); } } @@ -685,17 +681,15 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) void Ions::lockAtom(const std::string& name) { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - std::string name_ion((*ion)->name()); + std::string name_ion(ion->name()); if (name.compare(name_ion) == 0) { - (*ion)->lock(); + ion->lock(); if (onpe0) (*MPIdata::sout) << "Lock atom " << name << std::endl; break; } - ion++; } } @@ -975,12 +969,10 @@ void Ions::readRestartPositions(HDFrestart& h5_file) std::vector data; h5_file.readAtomicPositions(data); - int i = 0; - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1109,14 +1101,12 @@ void Ions::removeMassCenterMotion() #endif } - ion = local_ions_.begin(); - i = 0; - while (ion != local_ions_.end()) + i = 0; + for (auto& ion : local_ions_) { const int threei = 3 * i; - (*ion)->setVelocity(velocities[threei] - tmp[0], + ion->setVelocity(velocities[threei] - tmp[0], velocities[threei + 1] - tmp[1], velocities[threei + 2] - tmp[2]); - ion++; i++; } @@ -1124,16 +1114,15 @@ void Ions::removeMassCenterMotion() mv[0] = 0.; mv[1] = 0.; mv[2] = 0.; - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - velocities[3 * i] = (*ion)->velocity(0); - velocities[3 * i + 1] = (*ion)->velocity(1); - velocities[3 * i + 2] = (*ion)->velocity(2); + velocities[3 * i] = ion->velocity(0); + velocities[3 * i + 1] = ion->velocity(1); + velocities[3 * i + 2] = ion->velocity(2); mv[0] += mass[i] * velocities[3 * i]; mv[1] += mass[i] * velocities[3 * i + 1]; mv[2] += mass[i] * velocities[3 * i + 2]; - ion++; i++; } @@ -1155,12 +1144,10 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) std::vector data; h5_file.readAtomicVelocities(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1174,12 +1161,10 @@ void Ions::readRestartRandomStates(HDFrestart& h5f_file) std::vector data; h5f_file.readRestartRandomStates(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1202,17 +1187,14 @@ void Ions::writeForces(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get position of local ion double force[3]; - (*ion)->getForce(&force[0]); + ion->getForce(&force[0]); data.push_back(force[0]); data.push_back(force[1]); data.push_back(force[2]); - - ++ion; } } @@ -1364,64 +1346,54 @@ void Ions::printForcesLocal(std::ostream& os, const int root) const << "FX" << std::setw(10) << "FY" << std::setw(10) << "FZ" << std::endl; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { + ion->printPositionAndForce(os); - (*ion)->printPositionAndForce(os); - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } // global statistics @@ -1479,12 +1451,10 @@ int Ions::countIonsHere() const { return (int)local_ions_.size(); } int Ions::countProjectorsHere() const { - int count = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int count = 0; + for (auto& ion : local_ions_) { - count += (*ion)->nProjectors(); - ion++; + count += ion->nProjectors(); } return count; } @@ -1497,12 +1467,10 @@ int Ions::countProjectors() const Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - int nproj = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int nproj = 0; + for (auto& ion : local_ions_) { - nproj += (*ion)->nProjectors(); - ion++; + nproj += ion->nProjectors(); } int tmp = nproj; MPI_Allreduce(&tmp, &nproj, 1, MPI_INT, MPI_SUM, myPEenv.comm()); @@ -1569,13 +1537,10 @@ void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); - - ion++; + ion->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); ia++; } @@ -1780,45 +1745,8 @@ int Ions::setAtoms( if (ia < 1000) aname.append("0"); if (ia < 10000) aname.append("0"); aname.append(ss.str()); - Ion* new_ion - = new Ion(species_[isp], aname, &crds[3 * ia], velocity, locked); - new_ion->bcast(mmpi.commGlobal()); - - // Populate list_ions_ list - // std::cout<<"crds: "< 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " << crds[3 * ia + 0] - << "," << crds[3 * ia + 1] << "," << crds[3 * ia + 2] - << " added to the list... on PE" << mmpi.mypeGlobal() - << std::endl; - // populate local_ions_ list - if (inLocalIons( - crds[3 * ia + 0], crds[3 * ia + 1], crds[3 * ia + 2])) - { - (new_ion)->set_here(true); - local_ions_.push_back(new_ion); - if (onpe0 && ct.verbose > 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " - << crds[3 * ia + 0] << "," << crds[3 * ia + 1] << "," - << crds[3 * ia + 2] - << " added to the list of local ions... on PE" - << mmpi.mypeGlobal() << std::endl; - } - else - (new_ion)->set_here(false); - } - else - { - //(*MPIdata::sout)<<"Ion "<set_here(true); + local_ions_.push_back(new_ion); + if (onpe0 && ct.verbose > 2) + (*MPIdata::sout) << "Ion " << name << " at position " << crds[0] + << "," << crds[1] << "," << crds[2] + << " added to the list of local ions... on PE" + << mmpi.mypeGlobal() << std::endl; + } + else + (new_ion)->set_here(false); + } + else + { + // delete Ion if not put in list + delete new_ion; + } +} + int Ions::readNatoms(const std::string& filename, const bool cell_relative) { Control& ct(*(Control::instance())); @@ -2117,9 +2085,8 @@ void Ions::setVelocities(const std::vector& tau0, assert(tau0.size() == 3 * local_ions_.size()); assert(taup.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { double v[3]; for (short i = 0; i < 3; i++) @@ -2128,8 +2095,7 @@ void Ions::setVelocities(const std::vector& tau0, v[i] -= tau0[3 * ia + i]; v[i] /= dt; } - (*iion)->setVelocity(v[0], v[1], v[2]); - iion++; + ion->setVelocity(v[0], v[1], v[2]); ia++; } } @@ -2138,12 +2104,10 @@ void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau[3 * ia]); - iion++; + ion->getPosition(&tau[3 * ia]); ia++; } } @@ -2188,12 +2152,10 @@ void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau0_[3 * ia]); - iion++; + ion->getPosition(&tau0_[3 * ia]); ia++; } } @@ -2202,13 +2164,11 @@ void Ions::setPositionsToTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - iion++; ia++; } } @@ -2244,13 +2204,11 @@ void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { assert(3 * ia + 2 < (int)tau.size()); - (*iion)->getForce(&tau[3 * ia]); - iion++; + ion->getForce(&tau[3 * ia]); ia++; } } @@ -2351,12 +2309,10 @@ double Ions::computeMaxVlRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& ion : local_ions_) { - double r = (*iion)->computeRadiusVl(); + double r = ion->computeRadiusVl(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -2393,13 +2349,11 @@ void Ions::gatherNames(std::map& names, const int root, std::vector data; std::vector local_names; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get local name and index - local_names.push_back((*ion)->name()); - local_indexes.push_back((*ion)->index()); - ++ion; + local_names.push_back(ion->name()); + local_indexes.push_back(ion->index()); } // gather data to PE root @@ -2695,15 +2649,13 @@ bool Ions::hasLockedAtoms() const return (flag == 1); } -double Ions::getMaxNLradius() const +double Ions::getSpeciesMaxNLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double nlradius = spi->nlradius(); + const double nlradius = spi.nlradius(); radius = radius > nlradius ? radius : nlradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2711,15 +2663,13 @@ double Ions::getMaxNLradius() const return radius; } -double Ions::getMaxLradius() const +double Ions::getSpeciesMaxLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double lradius = spi->lradius(); + const double lradius = spi.lradius(); radius = radius > lradius ? radius : lradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2918,8 +2868,8 @@ void Ions::computeNumIons(void) double Ions::getMaxListRadius() const { // get radius of projectors - const double nlradius = getMaxNLradius(); - const double lradius = getMaxLradius(); + const double nlradius = getSpeciesMaxNLradius(); + const double lradius = getSpeciesMaxLradius(); double rmax = nlradius > lradius ? nlradius : lradius; @@ -3289,21 +3239,15 @@ void Ions::updateIons() Control& ct(*(Control::instance())); // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - // (*ion)->setForce(fion_[3*ia+0], - // fion_[3*ia+1], - // fion_[3*ia+2]); - - (*ion)->setRandomState(rand_states_[3 * ia + 0], - rand_states_[3 * ia + 1], rand_states_[3 * ia + 2]); + ion->setRandomState(rand_states_[3 * ia + 0], rand_states_[3 * ia + 1], + rand_states_[3 * ia + 2]); - ion++; ia++; } diff --git a/src/Ions.h b/src/Ions.h index 772c51d1..ae4f6adb 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -71,13 +71,14 @@ class Ions void setMapVL(); double computeMaxVlRadius() const; double computeMaxNLprojRadius() const; - double getMaxNLradius() const; - double getMaxLradius() const; - void updateListIons(); - // compute boundary for box containing all atoms to be known on local - // processor that is values for list_boundary_left_, list_boundary_right_ - void setupListIonsBoundaries(const double rmax); + /* + * Evaluate maximum pseudopotential radius among all species in class + */ + double getSpeciesMaxNLradius() const; + double getSpeciesMaxLradius() const; + + void updateListIons(); void augmentIonsData(const int nsteps, const int dir, const int disp, const int locSize, const int maxLocSize, std::vector& data, @@ -163,6 +164,8 @@ class Ions bool hasLockedAtoms() const; void clearLists(); + void rescaleVelocities(const double factor); + public: Ions(const double lat[3], const std::vector& sp); @@ -172,6 +175,10 @@ class Ions void setup(); + // compute boundary for box containing all atoms to be known on local + // processor that is values for list_boundary_left_, list_boundary_right_ + void setupListIonsBoundaries(const double rmax); + std::vector& getLocalNames() { return local_names_; } std::vector& getTau0() { return tau0_; } std::vector& getTaup() { return taup_; } @@ -334,13 +341,15 @@ class Ions void updateForcesInteractingIons(); void updateTaupInteractingIons(); - void rescaleVelocities(const double factor); /*! * Calculate minimum distance between local pairs */ double computeMinLocalSpacing() const; + void addIonToList(const Species& sp, const std::string& name, + const double crds[3], const double velocity[3], const bool lock); + // void checkUnicityLocalIons(); }; diff --git a/src/KBprojectorSparse.cc b/src/KBprojectorSparse.cc index afd8871c..937b95d2 100644 --- a/src/KBprojectorSparse.cc +++ b/src/KBprojectorSparse.cc @@ -103,7 +103,7 @@ void KBprojectorSparse::setNLindex( for (int i = 0; i < size_nl; i++) { - assert(i < lnumpt); + // assert(i < lnumpt); if ((pvec[i] < iloc * lnumpt) || (pvec[i] >= (iloc + 1) * lnumpt)) { (*MPIdata::sout) << " iloc=" << iloc << ", i=" << i @@ -960,7 +960,7 @@ bool KBprojectorSparse::setIndexesAndProjectors() // get "pvec" and "is_in_domain" int icount = get_index_array(pvec, iloc, index_low, index_high); assert(icount <= nl3); - assert(icount <= mymesh->npointsPatch()); + // assert(icount <= mymesh->npointsPatch()); assert(icount > 0); setNLindex(iloc, icount, pvec); diff --git a/src/MGmol.cc b/src/MGmol.cc index 8da0d331..acc2b944 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -837,7 +837,7 @@ void MGmol::initNuc(Ions& ions) pot.initialize(ions); // Check compensating charges - double comp_rho = pot.getCharge(pot.rho_comp()); + double comp_rho = getCharge(pot.rho_comp()); if (onpe0 && ct.verbose > 1) { diff --git a/src/MGmol.h b/src/MGmol.h index 6a4cc830..a9bca4ba 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -182,7 +182,10 @@ class MGmol : public MGmolInterface /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } - std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getHamiltonian() + { + return hamiltonian_; + } std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/PCGSolver.cc b/src/PCGSolver.cc index 12103aaa..b3bd7ec3 100644 --- a/src/PCGSolver.cc +++ b/src/PCGSolver.cc @@ -11,14 +11,13 @@ #include #include -using namespace std; -template -void PCGSolver::clear() +template +void PCGSolver::clear() { - for (short i = 0; i < (short)pc_oper_.size(); i++) + for (short i = 0; i < (short)precond_oper_.size(); i++) { - delete pc_oper_[i]; + delete precond_oper_[i]; } for (short i = 0; i < (short)gf_work_.size(); i++) { @@ -35,26 +34,26 @@ void PCGSolver::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { delete grid_[i]; } - pc_oper_.clear(); + precond_oper_.clear(); grid_.clear(); gf_work_.clear(); gf_rcoarse_.clear(); gf_newv_.clear(); } -template -void PCGSolver::setupPrecon() +template +void PCGSolver::setupPrecon() { // check if precon is already setup // Assumes operator does not change, hence // a single setup is sufficient - if (is_pc_setup_) return; + if (is_precond_setup_) return; // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -63,7 +62,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*grid_[0], lap_type_); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); pb::GridFunc* gf_work = new pb::GridFunc( @@ -92,7 +91,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*coarse_grid, 1); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); gf_work = new pb::GridFunc( *coarse_grid, bc_[0], bc_[1], bc_[2]); @@ -109,12 +108,13 @@ void PCGSolver::setupPrecon() mygrid = coarse_grid; } - is_pc_setup_ = true; + is_precond_setup_ = true; } // MG V-cycle with no mask -template -void PCGSolver::preconSolve(pb::GridFunc& gf_v, +template +void PCGSolver::preconSolve( + pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve(pb::GridFunc& gf_v, ncycl = 4 > (nu1_ + nu2_) ? 4 : (nu1_ + nu2_); } - pb::Lap* myoper = pc_oper_[level]; + pb::Lap* myoper = precond_oper_[level]; // SMOOTHING for (short it = 0; it < ncycl; it++) @@ -161,25 +161,23 @@ void PCGSolver::preconSolve(pb::GridFunc& gf_v, } // Left Preconditioned CG -template -bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +template +bool PCGSolver::solve( + pb::GridFunc& gf_phi, const pb::GridFunc& gf_rhs) { bool converged = false; const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual: r := b - Ax /* compute Ax */ oper_.apply(gf_phi, lhs); /* set r = b */ - pb::GridFunc res(gf_rhs); + pb::GridFunc res(gf_rhs); oper_.transform(res); - // Hartree units - const double hu = 4. * M_PI; - res *= hu; /* compute r = r - Ax */ res -= lhs; @@ -199,11 +197,11 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) /* preconditioning step */ prec_z.setValues(0.); preconSolve(prec_z, prec_res, 0); - pb::GridFunc z(prec_z); + pb::GridFunc z(prec_z); // conjugate vectors - pb::GridFunc p(prec_z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(prec_z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -251,13 +249,14 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) } // Left Preconditioned CG -template -bool PCGSolver::solve(T2* phi, T2* rhs, const char dis) +template +bool PCGSolver::solve( + ScalarType* phi, ScalarType* rhs, const char dis) { - pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_phi.assign(phi, dis); - pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_work.assign(rhs, dis); bool converged = solve(gf_phi, gf_work); diff --git a/src/PCGSolver.h b/src/PCGSolver.h index d38787f1..f6db7cc6 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_H_ -#define _PCG_SOLVER_H_ +#ifndef MGMOL_PCG_SOLVER_H +#define MGMOL_PCG_SOLVER_H #include "Control.h" #include "Lap.h" @@ -17,32 +17,41 @@ #include -template +template class PCGSolver { - private: std::vector grid_; short lap_type_; short bc_[3]; bool fully_periodic_; - // operators + + // operator to solve for T oper_; - std::vector*> pc_oper_; + + // preconditioner operator for each MG level + std::vector*> precond_oper_; std::vector*> gf_work_; std::vector*> gf_rcoarse_; std::vector*> gf_newv_; - // solver params + + // solver parameters int maxiters_; double tol_; double final_residual_; double residual_reduction_; - // precon params + + // preconditioner parameters short nu1_; short nu2_; short max_nlevels_; short nlevels_; - bool is_pc_setup_; + bool is_precond_setup_; + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + void setupPrecon(); + void clear(); public: PCGSolver(T& oper, const short px, const short py, const short pz) @@ -61,15 +70,10 @@ class PCGSolver bc_[1] = py; bc_[2] = pz; fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); - // fine grid info - // pb::Grid* mygrid=new pb::Grid(oper.grid()); - // grid_.push_back(mygrid); - // fine grid operator - Control& ct = *(Control::instance()); - lap_type_ = ct.lap_type; - is_pc_setup_ = false; - // pb::Lap* myoper = LapFactory::createLap(*grid_[0],lap_type_); - // pc_oper_.push_back(myoper); + + Control& ct = *(Control::instance()); + lap_type_ = ct.lap_type; + is_precond_setup_ = false; }; void setup(const short nu1, const short nu2, const short max_sweeps, @@ -83,16 +87,10 @@ class PCGSolver setupPrecon(); } - void clear(); - - void setupPrecon(); - - void preconSolve(pb::GridFunc& gf_v, - const pb::GridFunc& gf_f, const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve(pb::GridFunc& gf_phi, + const pb::GridFunc& gf_rhs); - bool solve(T2* phi, T2* rhs, const char dis); + bool solve(ScalarType* phi, ScalarType* rhs, const char dis); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PCGSolver_Diel.cc b/src/PCGSolver_Diel.cc index 0cdc5aed..4e7c88a9 100644 --- a/src/PCGSolver_Diel.cc +++ b/src/PCGSolver_Diel.cc @@ -9,10 +9,8 @@ #include "PCGSolver_Diel.h" -using namespace std; - -template -void PCGSolver_Diel::clear() +template +void PCGSolver_Diel::clear() { for (short i = 0; i < (short)pc_oper_.size(); i++) { @@ -33,7 +31,7 @@ void PCGSolver_Diel::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { @@ -46,8 +44,8 @@ void PCGSolver_Diel::clear() gf_newv_.clear(); } -template -void PCGSolver_Diel::setupPrecon() +template +void PCGSolver_Diel::setupPrecon() { // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -57,8 +55,8 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(oper_); pc_oper_.push_back(myoper); - pb::GridFunc* gf_work - = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_work + = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); // coarse levels @@ -86,24 +84,25 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(pc_oper_[ln - 1]->coarseOp(*mygrid)); pc_oper_.push_back(myoper); - gf_work = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + gf_work = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); - pb::GridFunc* gf_rcoarse - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_rcoarse = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_rcoarse_.push_back(gf_rcoarse); - pb::GridFunc* gf_newv - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_newv = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_newv_.push_back(gf_newv); mygrid = coarse_grid; } } -template +template // MG V-cycle with no mask -void PCGSolver_Diel::preconSolve( - pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) +void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve( // COARSE GRID CORRECTION // restrictions - pb::GridFunc* rcoarse = gf_rcoarse_[level]; + pb::GridFunc* rcoarse = gf_rcoarse_[level]; gf_work_[level]->restrict3D(*rcoarse); // storage functions for coarse grid - pb::GridFunc* newv = gf_newv_[level]; + pb::GridFunc* newv = gf_newv_[level]; // call mgrid solver on a coarser level newv->resetData(); @@ -149,15 +148,16 @@ void PCGSolver_Diel::preconSolve( if (bc_[0] != 1 || bc_[2] != 1 || bc_[2] != 1) gf_v.trade_boundaries(); } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve( - pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +bool PCGSolver_Diel::solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) { if (!oper_.initialized()) { - cout << "Error in PCGSolver_Diel::solve: operator not initialized" - << endl; + std::cout + << "Error in PCGSolver_Diel::solve: operator not initialized" + << std::endl; return 0.; } @@ -165,13 +165,13 @@ bool PCGSolver_Diel::solve( const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); - pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual oper_.apply(gf_phi, lhs); - pb::GridFunc rhs(gf_rhs); + pb::GridFunc rhs(gf_rhs); oper_.transform(rhs); // Hartree units rhs *= (4. * M_PI); @@ -180,13 +180,13 @@ bool PCGSolver_Diel::solve( double rnorm = init_rnorm; // preconditioned residual - pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); // preconditioning step z = 0.; preconSolve(z, res, 0); // conjugate vectors - pb::GridFunc p(z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -225,11 +225,11 @@ bool PCGSolver_Diel::solve( return converged; } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, - pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, - pb::GridFunc& gf_vks) +bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks) { // initialize the linear system operator and the preconditioner oper_.init(gf_rhod); diff --git a/src/PCGSolver_Diel.h b/src/PCGSolver_Diel.h index 9fd97914..f2001a4c 100644 --- a/src/PCGSolver_Diel.h +++ b/src/PCGSolver_Diel.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_DIEL_H_ -#define _PCG_SOLVER_DIEL_H_ +#ifndef MGMOL_PCG_SOLVER_DIEL_H_ +#define MGMOL_PCG_SOLVER_DIEL_H_ #include "Control.h" #include "PB.h" @@ -21,7 +21,7 @@ #include -template +template class PCGSolver_Diel { @@ -32,9 +32,9 @@ class PCGSolver_Diel // operators T oper_; std::vector pc_oper_; - std::vector*> gf_work_; - std::vector*> gf_rcoarse_; - std::vector*> gf_newv_; + std::vector*> gf_work_; + std::vector*> gf_rcoarse_; + std::vector*> gf_newv_; // solver params int maxiters_; double tol_; @@ -47,6 +47,11 @@ class PCGSolver_Diel short nlevels_; void setupPrecon(); + void clear(); + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + public: PCGSolver_Diel(T& oper, const short px, const short py, const short pz) : oper_(oper) @@ -78,15 +83,12 @@ class PCGSolver_Diel max_nlevels_ = max_nlevels; } - void clear(); - - void preconSolve(pb::GridFunc& gf_v, const pb::GridFunc& gf_f, - const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs, - pb::GridFunc& gf_rhod, pb::GridFunc& gf_vks); + bool solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PoissonSolverFactory.h b/src/PoissonSolverFactory.h new file mode 100644 index 00000000..91b99bf2 --- /dev/null +++ b/src/PoissonSolverFactory.h @@ -0,0 +1,228 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE +#ifndef MGMOL_PoissonSolverFactory +#define MGMOL_PoissonSolverFactory + +#include "Control.h" +#include "Hartree.h" +#include "Hartree_CG.h" +#include "Mesh.h" +#include "PBdiel.h" +#include "PBdiel_CG.h" +#include "ShiftedHartree.h" +#include "mputils.h" + +#include "GridFunc.h" +#include "Laph2.h" +#include "Laph4.h" +#include "Laph4M.h" +#include "Laph4MP.h" +#include "Laph6.h" +#include "Laph8.h" +#include "ShiftedLaph4M.h" + +class PoissonSolverFactory +{ + +public: + /*! + * return specific Poisson solver needed to solve Hartree problem + */ + static Poisson* create(const pb::Grid& myGrid, PoissonFDtype lap_type, + const short bc[3], const double screening_const) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + else // use PCG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + + return poisson_solver; + } + + static Poisson* createDiel(pb::Grid& pbGrid, PoissonFDtype lap_type, + const short bc[3], const double e0, const double rho0, + const double drho0) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + else // use PCG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + return poisson_solver; + } +}; + +#endif diff --git a/src/Potentials.cc b/src/Potentials.cc index ee874f10..380f05a0 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -27,6 +27,7 @@ #include using namespace std; +// unit conversion factor Ha -> Ry const double ha2ry = 2.; Potentials::~Potentials() @@ -57,13 +58,9 @@ Potentials::Potentials(const bool vh_frozen) size_ = dim_[0] * dim_[1] * dim_[2]; - mix_ = 1.; - scf_dvrho_ = 1000.; scf_dv_ = 1000.; - vh_frozen_ = vh_frozen; - vtot_.resize(size_); vtot_old_.resize(size_); @@ -101,7 +98,7 @@ void Potentials::initWithVnuc() double one = 1.; LinearAlgebraUtils::MPaxpy( size_, one, &v_ext_[0], &vtot_[0]); - // factor 2 to get total potential in [Ry] for calculations + // factor ha2ry to get total potential in [Ry] for calculations LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); } @@ -568,9 +565,8 @@ template void Potentials::setVxc(const T* const vxc, const int iterativeIndex) { assert(iterativeIndex >= 0); - // int ione=1; + itindex_vxc_ = iterativeIndex; - // Tcopy(&size_, vxc, &ione, &vxc_rho_[0], &ione); MPcpy(&vxc_rho_[0], vxc, size_); } void Potentials::setVh(const POTDTYPE* const vh, const int iterativeIndex) @@ -849,20 +845,6 @@ void Potentials::rescaleRhoComp() if (comp_rho < 0.) mmpi.abort(); } -double Potentials::getCharge(RHODTYPE* rho) -{ - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - double charge = mygrid.integralOverMesh(rho); - - if (onpe0 && ct.verbose > 0) - cout << setprecision(8) << fixed << "Charge: " << charge << endl; - - return charge; -} - void Potentials::addBackgroundToRhoComp() { if (fabs(background_charge_) > 0.) diff --git a/src/Potentials.h b/src/Potentials.h index 0cd7a8a6..205fb538 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -30,7 +30,6 @@ class Potentials int gdim_[3]; int dim_[3]; bool diel_; - double mix_; double scf_dvrho_; double scf_dv_; @@ -39,8 +38,9 @@ class Potentials double charge_in_cell_; double ionic_charge_; - bool vh_frozen_; - + /*! + * Total KS potential seen by electrons + */ std::vector vtot_; std::vector vtot_old_; @@ -48,16 +48,30 @@ class Potentials std::vector vh_rho_; std::vector vxc_rho_; - // nuclei local potential + /* + * Potential contribution from atomic cores (local pseudopotential) + */ std::vector v_nuc_; - // external potential (read from input) + /*! + * Optional external potential (read from input) + * Used only in special cases. + */ std::vector v_ext_; #ifdef HAVE_TRICUBIC pb::TriCubic* vext_tricubic_; #endif + /*! + * Potential associated with the sum of Gaussian charge distributions + * compensating the Coulomb potential of each atom + */ std::vector v_comp_; + + /*! + * Sum of Gaussian charge distributions compensating the Coulomb potential + * of each atom + */ std::vector rho_comp_; std::vector dv_; @@ -121,14 +135,11 @@ class Potentials void turnOnDiel() { diel_ = true; } int size() const { return size_; } - bool vh_frozen() const { return vh_frozen_; } - void freeze_vh() { vh_frozen_ = true; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return &vtot_[0]; } POTDTYPE* vh_rho() { return &vh_rho_[0]; } - POTDTYPE* vxc_rho() { return &vxc_rho_[0]; } RHODTYPE* rho_comp() { return &rho_comp_[0]; } const std::vector& vnuc() const { return v_nuc_; } @@ -136,12 +147,6 @@ class Potentials POTDTYPE* vext() { return &v_ext_[0]; } POTDTYPE* vepsilon() { return &vepsilon_[0]; } - void set_vcomp(const POTDTYPE val) - { - const int n = (int)v_comp_.size(); - for (int i = 0; i < n; i++) - v_comp_[i] = val; - } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -154,13 +159,29 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + /*! + * initialize total potential as local pseudopotential + */ void initWithVnuc(); void getVofRho(std::vector& vrho) const; - double delta_v(const std::vector>&); - double update(const std::vector>&); - void update(const double); + /*! + * evaluate potential correction associated with a new rho + */ + double delta_v(const std::vector>& rho); + + /*! + * update potentials based on argument rho + */ + double update(const std::vector>& rho); + + /*! + * update potentials based on potential correction delta v and mixing + * parameter + */ + void update(const double mix); + double max() const; double min() const; void readAll(std::vector& sp); @@ -172,7 +193,6 @@ class Potentials void initialize(Ions& ions); void rescaleRhoComp(); - double getCharge(RHODTYPE* rho); void initBackground(Ions& ions); void addBackgroundToRhoComp(); diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e0e679e9..3606aa40 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -509,6 +509,7 @@ template void ProjectedMatrices::setOccupations( const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS if (mmpi.instancePE0()) (*MPIdata::sout) << "ProjectedMatrices::setOccupations()" diff --git a/src/pb/Mgm.h b/src/pb/Mgm.h index d16f5610..4b5ad68d 100644 --- a/src/pb/Mgm.h +++ b/src/pb/Mgm.h @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Mgm.h,v 1.13 2010/01/28 22:56:47 jeanluc Exp $ #ifndef PB_MGM_H #define PB_MGM_H @@ -41,8 +40,6 @@ bool Mgm(T1& A, T2& vh, const GridFunc& rho, const short cogr, A.rhs(res, rhs); // Hartree units - rhs *= (4. * M_PI); - // work GridFunc GridFunc lhs(finegrid, bcx, bcy, bcz); diff --git a/src/pb/Solver.h b/src/pb/Solver.h index 062fbaa2..f82b3b6a 100644 --- a/src/pb/Solver.h +++ b/src/pb/Solver.h @@ -7,9 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Solver.h,v 1.14 2010/01/28 22:56:47 jeanluc Exp $ -#ifndef SOLVE_H -#define SOLVE_H +#ifndef PB_SOLVER_H +#define PB_SOLVER_H #include "GridFunc.h" @@ -33,7 +32,7 @@ class Solver fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); } - virtual bool solve(GridFunc&, GridFunc&) = 0; + virtual bool solve(GridFunc&, const GridFunc&) = 0; virtual void setup(const short nu1, const short nu2, const short max_sweeps, const double tol, const short max_nlevels, diff --git a/src/pb/SolverLap.cc b/src/pb/SolverLap.cc index 5a85fefe..062755a8 100644 --- a/src/pb/SolverLap.cc +++ b/src/pb/SolverLap.cc @@ -7,8 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverLap.cc,v 1.15 2010/01/28 22:56:31 jeanluc Exp $ - #include "SolverLap.h" #include "Laph2.h" #include "Laph4.h" @@ -26,9 +24,8 @@ Timer vcycle_repvcycle_tm("vcycle_repvcycle"); namespace pb { - // explicit instantiation declaration -#if 1 + // double template class SolverLap, double>; template class SolverLap, double>; @@ -45,77 +42,7 @@ template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; -#else -// double -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); - -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -// float -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -#endif -/* -template bool Mgm(Laph4MP&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4M&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph2&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph6&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(ShiftedLaph4M&, GridFunc&, const GridFunc&, -const short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -*/ -/* -template int Vcycle(ShiftedLaph4M&, GridFunc&, const GridFunc -&, const short, const short, const short, const bool); template int -Vcycle(Laph4MP&, GridFunc&, const GridFunc &, const short, -const short, const short, const bool); template int Vcycle(Laph4M&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); template int Vcycle(Laph2&, GridFunc&, const -GridFunc &, const short, const short, const short, const bool); -template int Vcycle(Laph4&, GridFunc&, const GridFunc &, const -short, const short, const short, const bool); template int Vcycle(Laph6&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); -*/ template bool SolverLap::solve(T2* phi, T2* rhs, const char dis) { @@ -139,7 +66,7 @@ bool SolverLap::solve(T2* phi, T2* rhs, const char dis) } template -bool SolverLap::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverLap::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { bool conv = Mgm(oper_, gf_phi, gf_rhs, max_nlevels_, max_sweeps_, tol_, nu1_, nu2_, gather_coarse_level_, final_residual_, diff --git a/src/pb/SolverLap.h b/src/pb/SolverLap.h index 939e15bc..567bed05 100644 --- a/src/pb/SolverLap.h +++ b/src/pb/SolverLap.h @@ -64,7 +64,7 @@ class SolverLap : public Solver bool solve(T2* phi, T2* rhs, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; short getNbSweeps() const override { return nb_sweeps_; } double getFinalResidual() const override { return final_residual_; } diff --git a/src/pb/SolverPB.cc b/src/pb/SolverPB.cc index 53252c26..756b49d8 100644 --- a/src/pb/SolverPB.cc +++ b/src/pb/SolverPB.cc @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverPB.cc,v 1.13 2010/01/28 22:56:32 jeanluc Exp $ #include "SolverPB.h" #include "Mgm.h" #include "PBh2.h" @@ -85,7 +84,7 @@ bool SolverPB::solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis) } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks) { oper_.init(gf_rhod); @@ -101,7 +100,7 @@ bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { if (!oper_.initialized()) { diff --git a/src/pb/SolverPB.h b/src/pb/SolverPB.h index a05e1d49..99eae0e3 100644 --- a/src/pb/SolverPB.h +++ b/src/pb/SolverPB.h @@ -64,9 +64,9 @@ class SolverPB : public Solver } bool solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs, + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; ~SolverPB() override{}; diff --git a/src/tools.cc b/src/tools.cc index 4e1bd314..95790c59 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -228,7 +228,7 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) int mpierr=MPI_Reduce(&s, &r, 1, MPI_INT, MPI_SUM, 0, mmpi.commGlobal()); if( mpierr!=MPI_SUCCESS ) { - cerr << " Error in MPI!!!" << endl; + cerr << " Error in MPI!!!" << std::endl; MPI_Abort(mmpi.commGlobal(),1); } if( r!=mmpi.size()*s && onpe0 ) @@ -388,3 +388,18 @@ void getkvector(const int index, const int kmax, int kvector[3]) // std::cout << " k=(" << kvector[0] << "," << kvector[1] << "," // << kvector[2] << ")" << std::endl; } + +double getCharge(double* rho) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double charge = mygrid.integralOverMesh(rho); + + if (onpe0 && ct.verbose > 0) + std::cout << std::setprecision(8) << std::fixed << "Charge: " << charge + << std::endl; + + return charge; +} diff --git a/src/tools.h b/src/tools.h index 4e8374da..be3d13e0 100644 --- a/src/tools.h +++ b/src/tools.h @@ -37,5 +37,6 @@ double minQuadPolynomial(const double e0, const double e1, const double de0, double minQuadPolynomialFrom3values(const double e0, const double e1, const double e12, const bool print_flag, std::ostream& os); void getkvector(const int index, const int kmax, int kvector[3]); +double getCharge(double* rho); #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc205167..6de34971 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -209,6 +209,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +add_executable(testIons + ${CMAKE_SOURCE_DIR}/tests/testIons.cc) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -334,6 +336,10 @@ add_test(NAME testBatchLaph4 add_test(NAME testtMGkernels COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMGkernels) +add_test(NAME testIons + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testIons + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) @@ -541,6 +547,7 @@ target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -550,6 +557,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testIons PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg index d543b626..12002703 100644 --- a/tests/WFEnergyAndForces/mgmol.cfg +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H type=QUENCH [Quench] max_steps=50 -atol=1.e-9 +atol=1.e-8 num_lin_iterations=2 [Orbitals] initial_type=Gaussian diff --git a/tests/testIons.cc b/tests/testIons.cc new file mode 100644 index 00000000..30b2ad0c --- /dev/null +++ b/tests/testIons.cc @@ -0,0 +1,104 @@ +#include "Control.h" +#include "Ions.h" +#include "MGmol_MPI.h" +#include "Mesh.h" +#include "Species.h" + +#include + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + + MPI_Comm comm = MPI_COMM_WORLD; + + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + MGmol_MPI::setup(comm, std::cout); + Control::setup(comm, false, 0.); + + // create a domain [0.10.]^3 + const double origin[3] = { 0., 0., 0. }; + const double ll = 10.; + const double lattice[3] = { ll, ll, ll }; + const unsigned ngpts[3] = { 32, 24, 20 }; + short lap_type = 0; + + Mesh::setup(comm, ngpts, origin, lattice, lap_type); + + const double h[3] = { ll / (double(ngpts[0])), ll / (double(ngpts[1])), + ll / (double(ngpts[2])) }; + + // random number generator + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_real_distribution<> dis(0.0, 1.0); + + // create one species + Species sp(MPI_COMM_WORLD); + + // read species info from pseudopotential file + std::string file_path = argv[1]; + std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); + std::cout << "Potential = " << filename << std::endl; + + sp.read_1species(filename); + sp.set_dim_nl(h[0]); + sp.set_dim_l(h[0]); + sp.initPotentials('f', h[0], true); + + // put species into a vector + std::vector vsp; + vsp.push_back(sp); + + Ions ions(lattice, vsp); + ions.setupListIonsBoundaries(10000.); + + double velocity[3] = { 0., 0., 0. }; + + // set "na" atoms coordinates and add them to "ions" + const int na = 10; + for (int i = 0; i < na; i++) + { + double x[3] = { origin[0] + lattice[0] * dis(gen), + origin[1] + lattice[1] * dis(gen), + origin[2] + lattice[2] * dis(gen) }; + if (myrank == 0) + std::cout << "x,y,z = " << x[0] << ", " << x[1] << ", " << x[2] + << std::endl; + + // set all x to the values of PE0 + MPI_Bcast(&x[0], 3, MPI_DOUBLE, 0, comm); + + // make a name for atom based on species and order of reading in + std::string stri = std::to_string(i); + std::string aname("C" + stri); + + ions.addIonToList(sp, aname, &x[0], velocity, false); + } + + ions.setup(); + + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + return 1; + } + + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + + return 0; +} From 23db1eeb5e76bdda3cc20b181e6ac5839205d008 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 15 Oct 2024 21:01:23 -0700 Subject: [PATCH 24/50] Make user defined options --- src/Control.cc | 8 ++++++++ src/md.cc | 17 +++++++---------- src/read_config.cc | 4 ++++ src/rom_Control.h | 6 ++++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 2231813e..8e86f47b 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2056,6 +2056,8 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); + rom_pri_option.compare_md = vm["ROM.basis.compare_md"].as(); + rom_pri_option.num_orbbasis = vm["ROM.basis.number_of_orbital_basis"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); } // onpe0 @@ -2105,6 +2107,12 @@ void Control::syncROMOptions() rom_pri_option.rom_stage = static_cast(rom_stage); rom_pri_option.variable = static_cast(rom_var); + mpirc = MPI_Bcast(&rom_pri_option.compare_md, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.num_orbbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + mpirc = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); bcast_check(mpirc); } diff --git a/src/md.cc b/src/md.cc index 920ac1f3..314fd0dc 100644 --- a/src/md.cc +++ b/src/md.cc @@ -494,20 +494,17 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) #ifdef MGMOL_HAS_LIBROM // TODO: cleanup - int rdim = 39; - std::string basis_filename = "PinnedH2O_orbitals_basis_1_50"; - bool ROM_md = false; - - if (rdim > 0) + if (ct.getROMOptions().num_orbbasis > 0) { if (onpe0) { - os_ << "Projecting orbitals onto ROM subspaces" << std::endl; - os_ << "Loading ROM basis " << basis_filename << std::endl; - os_ << "ROM basis dimension = " << rdim << std::endl; + os_ << "Projecting orbitals onto ROM subspaces to compare " + << ((ct.getROMOptions().compare_md) ? "MD dynamics" : "force") << std::endl; + os_ << "Loading ROM basis " << ct.getROMOptions().basis_file << std::endl; + os_ << "ROM basis dimension = " << ct.getROMOptions().num_orbbasis << std::endl; } - project_orbital(basis_filename, rdim, **orbitals); - if (ROM_md) + project_orbital(ct.getROMOptions().basis_file, ct.getROMOptions().num_orbbasis, **orbitals); + if (ct.getROMOptions().compare_md) { force(**orbitals, ions); } diff --git a/src/read_config.cc b/src/read_config.cc index 40194678..e50977fb 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -430,6 +430,10 @@ void setupROMConfigOption(po::options_description &rom_cfg) "Frequency of saving libROM snapshot file at FOM simulation.") ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") + ("ROM.basis.compare_md", po::value()->default_value(false), + "Compare MD or single-step force.") + ("ROM.basis.number_of_orbital_basis", po::value()->default_value(-1), + "Number of orbital POD basis.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), "Number of potential POD basis to build Hartree potential ROM operator."); } diff --git a/src/rom_Control.h b/src/rom_Control.h index 8c3b4b2c..dddbc1b4 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -43,13 +43,15 @@ struct ROMPrivateOptions int restart_file_minidx = -1; int restart_file_maxidx = -1; std::string basis_file = ""; - ROMVariable variable=ROMVariable::NONE; + ROMVariable variable = ROMVariable::NONE; - /* save librom snapshot matrix at FOM simulation. */ + /* save librom orbital snapshot matrix at FOM simulation. */ bool save_librom_snapshot = false; int librom_snapshot_freq = -1; /* options for ROM building */ + bool compare_md = false; + int num_orbbasis = -1; int num_potbasis = -1; }; From a27efbab7c8cf4e0cccb6c837abbe80e93c6f3d9 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 15 Oct 2024 21:07:00 -0700 Subject: [PATCH 25/50] Add scripts --- .../PinnedH2O/{job.basis => job.basis_1_50} | 4 +- examples/PinnedH2O/{job.rom => job.offline} | 8 ++-- examples/PinnedH2O/job.rom_1_50 | 39 ++++++++++++++++++ .../{mgmol_rom.cfg => mgmol_offline.cfg} | 2 +- examples/PinnedH2O/mgmol_rom_1_50.cfg | 40 +++++++++++++++++++ 5 files changed, 86 insertions(+), 7 deletions(-) rename examples/PinnedH2O/{job.basis => job.basis_1_50} (88%) rename examples/PinnedH2O/{job.rom => job.offline} (73%) create mode 100644 examples/PinnedH2O/job.rom_1_50 rename examples/PinnedH2O/{mgmol_rom.cfg => mgmol_offline.cfg} (97%) create mode 100644 examples/PinnedH2O/mgmol_rom_1_50.cfg diff --git a/examples/PinnedH2O/job.basis b/examples/PinnedH2O/job.basis_1_50 similarity index 88% rename from examples/PinnedH2O/job.basis rename to examples/PinnedH2O/job.basis_1_50 index bf02a621..a5069570 100644 --- a/examples/PinnedH2O/job.basis +++ b/examples/PinnedH2O/job.basis_1_50 @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240919 +set maindir = /p/lustre2/cheung26/mgmol-20241012 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -27,6 +27,6 @@ echo "Snapshot files: $snapshot_files" set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}" -srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_Pinned_H2O.out +srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_PinnedH2O.out date diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.offline similarity index 73% rename from examples/PinnedH2O/job.rom rename to examples/PinnedH2O/job.offline index 8de3d6b7..0f8ef90e 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20240815 +set maindir = /p/lustre2/cheung26/mgmol-20241012 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -20,8 +20,8 @@ cp $maindir/install_quartz/bin/$exe . set datadir = $maindir/examples/PinnedH2O -set cfg_rom = mgmol_rom.cfg -cp $datadir/$cfg_rom . +set cfg_offline = mgmol_offline.cfg +cp $datadir/$cfg_offline . cp $datadir/coords.in . @@ -30,6 +30,6 @@ ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . source $maindir/scripts/modules.quartz -srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_PinnedH2O.out +srun -n $ncpus $exe -c $cfg_offline -i coords.in > offline_PinnedH2O.out date diff --git a/examples/PinnedH2O/job.rom_1_50 b/examples/PinnedH2O/job.rom_1_50 new file mode 100644 index 00000000..aa6df335 --- /dev/null +++ b/examples/PinnedH2O/job.rom_1_50 @@ -0,0 +1,39 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241012 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set increment_md_steps = 1 +set num_md_steps = 50 +set basis_file = PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps} + +set cfg_rom = mgmol_rom_${increment_md_steps}_${num_md_steps}.cfg +cp $datadir/$cfg_rom . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_${increment_md_steps}_${num_md_steps}_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/mgmol_rom.cfg b/examples/PinnedH2O/mgmol_offline.cfg similarity index 97% rename from examples/PinnedH2O/mgmol_rom.cfg rename to examples/PinnedH2O/mgmol_offline.cfg index 1f724546..332dd77d 100644 --- a/examples/PinnedH2O/mgmol_rom.cfg +++ b/examples/PinnedH2O/mgmol_offline.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=50 +num_steps=500 dt=40. thermostat=ON [Thermostat] diff --git a/examples/PinnedH2O/mgmol_rom_1_50.cfg b/examples/PinnedH2O/mgmol_rom_1_50.cfg new file mode 100644 index 00000000..337a86eb --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_1_50.cfg @@ -0,0 +1,40 @@ +verbosity=1 +xcFunctional=PBE +FDtype=Mehrstellen +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +basis_file=PinnedH2O_orbitals_basis_1_50 +[ROM.basis] +compare_md=false +number_of_orbital_basis=39 From f2f1821485b731fd1a6daa04c7f00d010c1de4e8 Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Tue, 22 Oct 2024 13:54:06 -0700 Subject: [PATCH 26/50] Ion density sampled evaluation (#274) * save/load poisson rom operator. * test routine for ion density * ion density hyper-reduction verified. * added rom ion test in ci workflow. * fixed ci workflow * setting ion configuration using Ions::setPositions. --- .github/workflows/ci.yml | 6 +- src/Control.cc | 6 + src/Electrostatic.h | 3 + src/MGmol.h | 1 + src/Potentials.cc | 91 ++++++++ src/Potentials.h | 7 + src/read_config.cc | 4 +- src/rom_Control.h | 3 + src/rom_main.cc | 14 ++ src/rom_workflows.cc | 216 +++++++++++++++++- src/rom_workflows.h | 6 + tests/ROM/test_rom_poisson/carbyne.ion.cfg | 63 +++++ .../{carbyne.cfg => carbyne.poisson.cfg} | 0 13 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 tests/ROM/test_rom_poisson/carbyne.ion.cfg rename tests/ROM/test_rom_poisson/{carbyne.cfg => carbyne.poisson.cfg} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e94113f..43953585 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,11 @@ jobs: cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom . ln -s ${GITHUB_WORKSPACE}/potentials/* . - mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.cfg -i carbyne.in + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.poisson.cfg -i carbyne.in + - name: test ROM ion density evaluation + run: | + cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson + mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.ion.cfg -i carbyne.in # code-style: # runs-on: ubuntu-latest # needs: [docker-image] diff --git a/src/Control.cc b/src/Control.cc index f1dcef23..bd8f5ced 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -2034,10 +2034,14 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::ONLINE; else if (str.compare("build") == 0) rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("online_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE_POISSON; else if (str.compare("test_poisson") == 0) rom_pri_option.rom_stage = ROMStage::TEST_POISSON; else if (str.compare("test_rho") == 0) rom_pri_option.rom_stage = ROMStage::TEST_RHO; + else if (str.compare("test_ion") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_ION; else if (str.compare("none") == 0) rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; @@ -2058,6 +2062,7 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.librom_snapshot_freq = vm["ROM.offline.librom_snapshot_freq"].as(); rom_pri_option.num_potbasis = vm["ROM.basis.number_of_potential_basis"].as(); + rom_pri_option.pot_rom_file = vm["ROM.potential_rom_file"].as(); } // onpe0 // synchronize all processors @@ -2073,6 +2078,7 @@ void Control::syncROMOptions() mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); mmpi.bcast(rom_pri_option.basis_file, comm_global_); + mmpi.bcast(rom_pri_option.pot_rom_file, comm_global_); auto bcast_check = [](int mpirc) { if (mpirc != MPI_SUCCESS) diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 7ae977e1..a8eb4709 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -47,6 +47,9 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + const bool isDielectric() { return diel_flag_; } + pb::GridFunc* getRhoc() { return grhoc_; } + Poisson* getPoissonSolver() { return poisson_solver_; } void setup(const short max_sweeps); diff --git a/src/MGmol.h b/src/MGmol.h index a9bca4ba..a6c383f3 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -187,6 +187,7 @@ class MGmol : public MGmolInterface return hamiltonian_; } std::shared_ptr> getRho() { return rho_; } + std::shared_ptr getIons() { return ions_; } void run() override; diff --git a/src/Potentials.cc b/src/Potentials.cc index 380f05a0..a54f0c47 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -899,6 +899,97 @@ void Potentials::initBackground(Ions& ions) if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; } +void Potentials::evalIonDensityOnSamplePts( + Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc) +{ + Mesh* mymesh = Mesh::instance(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const pb::Grid& mygrid = mymesh->grid(); + + const char flag_filter = pot_type(0); + if (flag_filter == 's') + { + if (onpe0) + { + cout << "Potentials::evalIonDensityOnSamplePts - flag_filter s is not supported" + << endl; + } + mmpi.abort(); + } + + // initialize output vector + sampled_rhoc.resize(local_idx.size()); + for (int d = 0; d < sampled_rhoc.size(); d++) + sampled_rhoc[d] = 0.0; + + // Loop over ions + for (auto& ion : ions.overlappingVL_ions()) + { + const Species& sp(ion->getSpecies()); + + Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + + initializeRadialDataOnSampledPts(position, sp, local_idx, sampled_rhoc); + } + + return; +} + +void Potentials::initializeRadialDataOnSampledPts( + const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc) +{ + assert(local_idx.size() == sampled_rhoc.size()); + + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + const int dim0 = mygrid.dim(0); + const int dim1 = mygrid.dim(1); + const int dim2 = mygrid.dim(2); + + const double start0 = mygrid.start(0); + const double start1 = mygrid.start(1); + const double start2 = mygrid.start(2); + + const double h0 = mygrid.hgrid(0); + const double h1 = mygrid.hgrid(1); + const double h2 = mygrid.hgrid(2); + + Vector3D point(0., 0., 0.); + + const double lrad = sp.lradius(); + + const RadialInter& lpot(sp.local_pot()); + const Vector3D lattice(mygrid.ll(0), mygrid.ll(1), mygrid.ll(2)); + + for(int k = 0; k < local_idx.size(); k++) + { + /* + local_idx provides offset. + offset = iz + iy * dim2 + ix * dim1 * dim2; + evaluate x,y,z indices backward. + */ + int iz = local_idx[k] % dim2; + int ix = local_idx[k] / dim2; + int iy = ix % dim1; + ix /= dim1; + + /* compute grid point position */ + point[0] = start0 + ix * h0; + point[1] = start1 + iy * h1; + point[2] = start2 + iz * h2; + + /* accumulate ion species density */ + const double r = position.minimage(point, lattice, ct.bcPoisson); + if (r < lrad) + sampled_rhoc[k] += sp.getRhoComp(r); + } + + return; +} + template void Potentials::setVxc( const double* const vxc, const int iterativeIndex); template void Potentials::setVxc( diff --git a/src/Potentials.h b/src/Potentials.h index 205fb538..0ddd10fa 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -95,6 +95,9 @@ class Potentials void initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp); + void initializeRadialDataOnSampledPts( + const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc); + public: Potentials(const bool vh_frozen = false); @@ -159,6 +162,8 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + const double getBackgroundCharge() const { return background_charge_; } + /*! * initialize total potential as local pseudopotential */ @@ -196,6 +201,8 @@ class Potentials void initBackground(Ions& ions); void addBackgroundToRhoComp(); + void evalIonDensityOnSamplePts(Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc); + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/read_config.cc b/src/read_config.cc index 40194678..4f3b34c1 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -431,6 +431,8 @@ void setupROMConfigOption(po::options_description &rom_cfg) ("ROM.offline.variable", po::value()->default_value(""), "FOM variable to perform POD: either orbitals or potential.") ("ROM.basis.number_of_potential_basis", po::value()->default_value(-1), - "Number of potential POD basis to build Hartree potential ROM operator."); + "Number of potential POD basis to build Hartree potential ROM operator.") + ("ROM.potential_rom_file", po::value()->default_value(""), + "File name to save/load potential ROM operators."); } #endif diff --git a/src/rom_Control.h b/src/rom_Control.h index 8c3b4b2c..8218bbb4 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -22,8 +22,10 @@ enum class ROMStage ONLINE, RESTORE, // TODO(kevin): what stage is this? BUILD, + ONLINE_POISSON, TEST_POISSON, TEST_RHO, + TEST_ION, UNSUPPORTED }; @@ -51,6 +53,7 @@ struct ROMPrivateOptions /* options for ROM building */ int num_potbasis = -1; + std::string pot_rom_file = ""; }; #endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc index fa956221..7eb323a7 100644 --- a/src/rom_main.cc +++ b/src/rom_main.cc @@ -118,6 +118,13 @@ int main(int argc, char** argv) buildROMPoissonOperator(mgmol); break; + case (ROMStage::ONLINE_POISSON): + if (ct.isLocMode()) + runPoissonROM(mgmol); + else + runPoissonROM(mgmol); + break; + case (ROMStage::TEST_POISSON): if (ct.isLocMode()) testROMPoissonOperator(mgmol); @@ -130,6 +137,13 @@ int main(int argc, char** argv) testROMRhoOperator(mgmol); else testROMRhoOperator(mgmol); + + case (ROMStage::TEST_ION): + if (ct.isLocMode()) + testROMIonDensity(mgmol); + else + testROMIonDensity(mgmol); + break; default: diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 5dc3718e..52ad6e47 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -218,7 +218,7 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) // write the file from PE0 only if (MPIdata::onpe0) { - std::string rom_oper = "pot_rom_oper.h5"; + std::string rom_oper = rom_options.pot_rom_file; CAROM::HDFDatabase h5_helper; h5_helper.create(rom_oper); h5_helper.putInteger("number_of_potential_basis", num_pot_basis); @@ -242,6 +242,67 @@ void buildROMPoissonOperator(MGmolInterface *mgmol_) } } +template +void runPoissonROM(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "runPoissonROM error: ROM variable must be POTENTIAL to run this stage!\n" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix *pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* initialize rom operator variables */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rom_inv(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + CAROM::Vector pot_rhs_rescaler(num_pot_basis, false); + + /* Load ROM operator */ + // read the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = rom_options.pot_rom_file; + CAROM::HDFDatabase h5_helper; + h5_helper.open(rom_oper, "r"); + int num_pot_basis_file = -1; + h5_helper.getInteger("number_of_potential_basis", num_pot_basis_file); + CAROM_VERIFY(num_pot_basis_file == num_pot_basis); + + h5_helper.getDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* load the inverse as well */ + h5_helper.getDoubleArray("potential_rom_inverse", pot_rom_inv.getData(), + num_pot_basis * num_pot_basis, false); + + /* load right-hand side hyper-reduction operator */ + h5_helper.getDoubleArray("potential_rhs_rom_inverse", pot_rhs_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* load right-hand side rescaling operator */ + h5_helper.getDoubleArray("potential_rhs_rescaler", pot_rhs_rescaler.getData(), + num_pot_basis, false); + + h5_helper.close(); + } +} + /* test routines */ template @@ -721,14 +782,167 @@ void computeRhoOnSamplePts(const CAROM::Matrix &dm, // rescaleTotalCharge(); } +template +void testROMIonDensity(MGmolInterface *mgmol_) +{ + /* random number generator */ + static std::random_device rd; // Will be used to obtain a seed for the random number engine + static std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd(){} + static std::uniform_real_distribution<> dis(0.0, 1.0); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol *mgmol = static_cast *>(mgmol_); + Poisson *poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + std::shared_ptr ions = mgmol->getIons(); + + /* get the extent of global domain */ + const double origin[3] = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + if (rank == 0) + { + printf("origin: (%.3e, %.3e, %.3e)\n", origin[0], origin[1], origin[2]); + printf("lattice: (%.3e, %.3e, %.3e)\n", lattice[0], lattice[1], lattice[2]); + } + + /* get global atomic numbers */ + const int num_ions = ions->getNumIons(); + std::vector atnumbers(num_ions); + ions->getAtomicNumbers(atnumbers); + + assert(!(mgmol->electrostat_->isDielectric())); + assert(pot.getBackgroundCharge() <= 0.0); + + const int num_snap = 3; + + /* 3 fictitious ion configurations */ + std::vector> cfgs(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + cfgs[idx].resize(3 * num_ions); + if (rank == 0) + for (int k = 0; k < num_ions; k++) + for (int d = 0; d < 3; d++) + cfgs[idx][3 * k + d] = origin[d] + lattice[d] * dis(gen); + + mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); + } + + /* Collect fictitious ion density based on each configuration */ + std::vector> fom_rhoc(num_snap); + /* Sanity check for overlappingVL_ions */ + std::vector>> fom_overlap_ions(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + /* set ion positions */ + ions->setPositions(cfgs[idx], atnumbers); + + /* save overlapping ions for sanity check */ + fom_overlap_ions[idx].resize(ions->overlappingVL_ions().size()); + for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + { + fom_overlap_ions[idx][k].resize(3); + for (int d = 0; d < 3; d++) + fom_overlap_ions[idx][k][d] = ions->overlappingVL_ions()[k]->position(d); + } + + /* compute resulting ion density */ + /* NOTE: we exclude rescaling for the sake of verification */ + pot.initialize(*ions); + + mgmol->electrostat_->setupRhoc(pot.rho_comp()); + fom_rhoc[idx].resize(dim); + mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); + } + + /* Initialize libROM classes */ + /* + In practice, we do not produce rhoc POD basis. + This rhoc POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_snap, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rhoc"); + for (int idx = 0; idx < num_snap; idx++) + basis_generator.takeSample(&fom_rhoc[idx][0]); + + const CAROM::Matrix rhoc_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix *rhoc_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix *proj_rhoc = rhoc_basis->transposeMult(rhoc_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rhoc_basis_inv(num_snap, num_snap, false); + std::vector global_sampled_row(num_snap), sampled_rows_per_proc(nprocs); + DEIM(rhoc_basis, num_snap, global_sampled_row, sampled_rows_per_proc, + rhoc_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank+1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* test one solution */ + std::uniform_int_distribution<> distrib(0, num_snap-1); + int test_idx = distrib(gen); + mmpi.bcastGlobal(&test_idx); + if (rank == 0) printf("test index: %d\n", test_idx); + + /* set ion positions */ + ions->setPositions(cfgs[test_idx], atnumbers); + + /* Sanity check for overlapping ions */ + CAROM_VERIFY(fom_overlap_ions[test_idx].size() == ions->overlappingVL_ions().size()); + for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + for (int d = 0; d < 3; d++) + CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + + /* eval ion density on sample grid points */ + std::vector sampled_rhoc(sampled_row.size()); + pot.evalIonDensityOnSamplePts(*ions, sampled_row, sampled_rhoc); + + for (int d = 0; d < sampled_row.size(); d++) + { + printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); + CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-12); + } +} + template void readRestartFiles(MGmolInterface *mgmol_); template void readRestartFiles(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template void runPoissonROM(MGmolInterface *mgmol_); +template void runPoissonROM(MGmolInterface *mgmol_); + template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); + +template void testROMIonDensity(MGmolInterface *mgmol_); +template void testROMIonDensity(MGmolInterface *mgmol_); diff --git a/src/rom_workflows.h b/src/rom_workflows.h index fd00c687..9222249f 100644 --- a/src/rom_workflows.h +++ b/src/rom_workflows.h @@ -45,12 +45,18 @@ void readRestartFiles(MGmolInterface *mgmol_); template void buildROMPoissonOperator(MGmolInterface *mgmol_); +template +void runPoissonROM(MGmolInterface *mgmol_); + template void testROMPoissonOperator(MGmolInterface *mgmol_); template void testROMRhoOperator(MGmolInterface *mgmol_); +template +void testROMIonDensity(MGmolInterface *mgmol_); + void computeRhoOnSamplePts(const CAROM::Matrix &dm, const CAROM::Matrix &phi_basis, const CAROM::Matrix &rom_phi, const std::vector &local_idx, CAROM::Vector &sampled_rho); diff --git a/tests/ROM/test_rom_poisson/carbyne.ion.cfg b/tests/ROM/test_rom_poisson/carbyne.ion.cfg new file mode 100644 index 00000000..d98232ad --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.ion.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_ion + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/ROM/test_rom_poisson/carbyne.cfg b/tests/ROM/test_rom_poisson/carbyne.poisson.cfg similarity index 100% rename from tests/ROM/test_rom_poisson/carbyne.cfg rename to tests/ROM/test_rom_poisson/carbyne.poisson.cfg From bc5992dedefc33d3e556ec0831deefb4df099f67 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 31 Oct 2024 12:09:42 -0700 Subject: [PATCH 27/50] Add get_result.sh --- examples/PinnedH2O/get_result.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 examples/PinnedH2O/get_result.sh diff --git a/examples/PinnedH2O/get_result.sh b/examples/PinnedH2O/get_result.sh new file mode 100644 index 00000000..443e4cdd --- /dev/null +++ b/examples/PinnedH2O/get_result.sh @@ -0,0 +1,27 @@ +filename="offline_PinnedH2O.out" # FOM +#filename="rom39_PinnedH2O.out" # compare MD +#filename="39_force_PinnedH2O.out" # compare force + +# Extracting H1, H2, F1, F2 from MGmgol output log +# if FOM, these files contain the FOM results +# if compare MD, these files contain the results with projected orbitals +awk '/H1 / {print $3, $4, $5}' $filename > H1_$filename +awk '/H2 / {print $3, $4, $5}' $filename > H2_$filename +awk '/F1 / {print $6, $7, $8}' $filename > F1_$filename +awk '/F2 / {print $6, $7, $8}' $filename > F2_$filename + +# if compare force, files with "_fom" contain the FOM results +# files with "_rom" contain the results with projected orbitals +if [[ "$filename" == *"force_"* ]]; then + sed -n '1~2p' H1_$filename > H1_rom$filename + sed -n '1~2p' H2_$filename > H2_rom$filename + sed -n '1~2p' F1_$filename > F1_rom$filename + sed -n '1~2p' F2_$filename > F2_rom$filename + + sed -n '2~2p' H1_$filename > H1_fom$filename + sed -n '2~2p' H2_$filename > H2_fom$filename + sed -n '2~2p' F1_$filename > F1_fom$filename + sed -n '2~2p' F2_$filename > F2_fom$filename +fi + +rm -rf snapshot0_* From da6e19a03b2b9a860b6adaa7ff1b333efa4696bd Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 31 Oct 2024 12:25:00 -0700 Subject: [PATCH 28/50] Add visualization scripts --- examples/PinnedH2O/plot_PinnedH2O_force.m | 84 +++++++++++++++++++++++ examples/PinnedH2O/plot_PinnedH2O_md.m | 59 ++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 examples/PinnedH2O/plot_PinnedH2O_force.m create mode 100644 examples/PinnedH2O/plot_PinnedH2O_md.m diff --git a/examples/PinnedH2O/plot_PinnedH2O_force.m b/examples/PinnedH2O/plot_PinnedH2O_force.m new file mode 100644 index 00000000..a41457a5 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_force.m @@ -0,0 +1,84 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +plot_rom = 0; +rdim = 77; + +%% +load F_fom.mat +fprintf(1, 'Force statistics using FOM orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_fom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_fom)); +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_fom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_fom)); + +if plot_fom + plotForce(F1_fom, 'F_H1_fom'); + plotForce(F2_fom, 'F_H2_fom'); + plotForceHistograms(F1_fom, 'H1_fom'); + plotForceHistograms(F2_fom, 'H2_fom'); +end + +%% +load(['F_rom' int2str(rdim) '.mat']) +fprintf(1, 'Force statistics using projected orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_rom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_rom)); +%H1_correlation = sum(F1_fom(:) .* F1_rom(:)) / (norm(F1_fom(:)) * norm(F1_rom(:))) +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_rom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_rom)); +%H2_correlation = sum(F2_fom(:) .* F2_rom(:)) / (norm(F2_fom(:)) * norm(F2_rom(:))) + +if plot_rom + plotForce(F1_rom, ['F_H1_rom' int2str(rdim)]); + plotForce(F2_rom, ['F_H2_rom' int2str(rdim)]); + plotForceHistograms(F1_rom, ['H1_rom' int2str(rdim)]); + plotForceHistograms(F2_rom, ['H2_rom' int2str(rdim)]); + plotForceHistogram(abs(F1_fom - F1_rom), ['H1_rom' int2str(rdim)], 'Fdiff'); + plotForceHistogram(abs(F2_fom - F2_rom), ['H2_rom' int2str(rdim)], 'Fdiff'); +end + +%% +function plotForce(F, suffix) + figure; + imagesc(F'); + axis tight; + axis equal; + colorbar; + saveas(gcf, suffix, 'jpg'); +end + +function plotForceHistogram(F, suffix, var) + figure; + if strcmp(var,'Fx') + X = F(:,1); + var_name = 'x-directional Force'; + elseif strcmp(var,'Fy') + X = F(:,2); + var_name = 'y-directional Force'; + elseif strcmp(var,'Fz') + X = F(:,3); + var_name = 'z-directional Force'; + elseif strcmp(var,'Fmag') + X = sqrt(sum(F.^2, 2)); + var_name = 'Force Magitude'; + elseif strcmp(var,'Fdiff') + X = sqrt(sum(F.^2, 2)); + var_name = 'Magitude of Difference in Force'; + else + error('Invalid type'); + end + histogram(X, 20); + xlabel(var_name); + ylabel('Frequency'); + title(['Histogram of ' var_name]); + saveas(gcf, [var '_' suffix], 'jpg'); +end + +function plotForceHistograms(F, suffix) + plotForceHistogram(F, suffix, 'Fx'); + plotForceHistogram(F, suffix, 'Fy'); + plotForceHistogram(F, suffix, 'Fz'); + plotForceHistogram(F, suffix, 'Fmag'); +end diff --git a/examples/PinnedH2O/plot_PinnedH2O_md.m b/examples/PinnedH2O/plot_PinnedH2O_md.m new file mode 100644 index 00000000..47c309b2 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_md.m @@ -0,0 +1,59 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +rdims = [77, 39]; + +%% +load md_fom.mat +if plot_fom + plotAngleHistogram(H1_fom, H2_fom, 'fom'); +end + +%% + +all_H1_rom = zeros(length(rdims), size(H1_fom, 1), 3); +all_H2_rom = zeros(length(rdims), size(H2_fom, 1), 3); +k = 0; + +for rdim = rdims + k = k + 1; + load(['md_rom' int2str(rdim) '.mat']) + plotAngleHistogram(H1_rom, H2_rom, ['rom' int2str(rdim)]); + all_H1_rom(k, :, :) = H1_rom; + all_H2_rom(k, :, :) = H2_rom; +end + +plotAtomTrajectory(H1_fom(:,1), all_H1_rom(:,:,1), rdims, 'x', 1) +plotAtomTrajectory(H1_fom(:,2), all_H1_rom(:,:,2), rdims, 'y', 1) +plotAtomTrajectory(H1_fom(:,3), all_H1_rom(:,:,3), rdims, 'z', 1) + +plotAtomTrajectory(H2_fom(:,1), all_H2_rom(:,:,1), rdims, 'x', 2) +plotAtomTrajectory(H2_fom(:,2), all_H2_rom(:,:,2), rdims, 'y', 2) +plotAtomTrajectory(H2_fom(:,3), all_H2_rom(:,:,3), rdims, 'z', 2) + +%% +function plotAtomTrajectory(X_fom, all_X_rom, rdims, var, idx) + figure; + hold on; + plot(X_fom, 'Linewidth', 2, 'DisplayName', 'FOM'); + k = 0; + for rdim = rdims + k = k + 1; + X_rom = all_X_rom(k, :); + plot(X_rom, 'Linewidth', 2, 'DisplayName', ['ROM dim = ' int2str(rdim)]); + end + title([var '-coordinate of H' int2str(idx)]) + legend; + saveas(gcf, [var '_H' int2str(idx)], 'jpg'); +end + +function plotAngleHistogram(X1, X2, suffix) + figure; + A = acosd(sum(X1.*X2,2) ./ sqrt(sum(X1.^2,2)) ./ sqrt(sum(X2.^2,2))); + histogram(A, 20); + xlabel('Angle'); + ylabel('Frequency'); + title('Histogram of angle'); + saveas(gcf, ['angle_' suffix], 'jpg'); +end \ No newline at end of file From bcdd6a11269c427feeb6f4c328c89cd66ffefd35 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 31 Oct 2024 12:27:13 -0700 Subject: [PATCH 29/50] Add Python script for plane action --- examples/PinnedH2O/generate_coord.py | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 examples/PinnedH2O/generate_coord.py diff --git a/examples/PinnedH2O/generate_coord.py b/examples/PinnedH2O/generate_coord.py new file mode 100644 index 00000000..c239eb87 --- /dev/null +++ b/examples/PinnedH2O/generate_coord.py @@ -0,0 +1,63 @@ +import numpy as np +import os + +# coords.in +O1 = np.array([0.00, 0.00, 0.00]) +ref_H1 = np.array([-0.45, 1.42, -1.07]) +ref_H2 = np.array([-0.45, -1.48, -0.97]) + +# factors and increments for bond lengths and bond angle +bondlength1_factor = np.linspace(0.95, 1.05, 11) +bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondangle_increment = np.linspace(-5, 5, 11) + +# output directory +output_dir = "PinnedH2O_3dof_coords" + +# utilities +def calculate_bondlength(atom1, atom2): + return np.linalg.norm(atom1 - atom2) + +def calculate_bondangle(atom1, atom2, atom3): + vector1 = atom1 - atom2 + vector2 = atom3 - atom2 + dot_product = np.dot(vector1, vector2) + magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) + angle_rad = np.arccos(dot_product / magnitude_product) + angle_deg = np.degrees(angle_rad) + return angle_deg + +# Rodrigues' rotation formula +def rotation_matrix(axis, angle_degrees): + angle = np.radians(angle_degrees) + cos_theta = np.cos(angle) + sin_theta = np.sin(angle) + ux, uy, uz = axis + return np.array([ + [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], + [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], + [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] + ]) + +# generation +os.makedirs(output_dir, exist_ok=True) + +ref_bondlength1 = calculate_bondlength(ref_H1, O1) +ref_bondlength2 = calculate_bondlength(ref_H2, O1) +ref_bondangle = calculate_bondangle(ref_H1, O1, ref_H2) + +normal_vector = np.cross(ref_H1, ref_H2) +normal_unit_vector = normal_vector / np.linalg.norm(normal_vector) + +for d_bondangle in bondangle_increment: + Q = rotation_matrix(normal_unit_vector, d_bondangle) + Q_ref_H2 = np.dot(Q, ref_H2) + for f_bondlength1 in bondlength1_factor: + for f_bondlength2 in bondlength2_factor: + H1 = f_bondlength1 * ref_H1 + H2 = f_bondlength2 * Q_ref_H2 + filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" + with open(filename, "w") as file: + file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") + file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") + file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") From a733973bc917d0fd02a6d0d56dee9179e039374f Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 10:14:51 -0800 Subject: [PATCH 30/50] Add new files --- examples/PinnedH2O/generate_coord_simple.py | 32 +++++++++++ examples/PinnedH2O/rotation_test.py | 62 +++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 examples/PinnedH2O/generate_coord_simple.py create mode 100644 examples/PinnedH2O/rotation_test.py diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py new file mode 100644 index 00000000..181691a5 --- /dev/null +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -0,0 +1,32 @@ +import numpy as np +import os + +O1 = np.array([0.00, 0.00, 0.00]) + +ref_bondlength = 1.83 +ref_bondangle = 104.5 + +# factors and increments for bond lengths and bond angle +bondlength1_factor = np.linspace(0.95, 1.05, 11) +bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondangle_increment = np.linspace(-5, 5, 11) + +# output directory +output_dir = "PinnedH2O_3dof_coords" + +# generation +os.makedirs(output_dir, exist_ok=True) + +for d_bondangle in bondangle_increment: + bondangle = ref_bondangle + d_bondangle + x = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.cos(np.radians(bondangle / 2)) + for f_bondlength1 in bondlength1_factor: + for f_bondlength2 in bondlength2_factor: + H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) + H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0]) + filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" + with open(filename, "w") as file: + file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") + file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") + file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py new file mode 100644 index 00000000..7f74c55b --- /dev/null +++ b/examples/PinnedH2O/rotation_test.py @@ -0,0 +1,62 @@ +import numpy as np + +O1 = np.array([0.00, 0.00, 0.00]) +H1 = np.array([-0.45, 1.42, -1.07]) +H2 = np.array([-0.45, -1.48, -0.97]) + +def calculate_bondlength(atom1, atom2): + return np.linalg.norm(atom1 - atom2) + +def calculate_bondangle(atom1, atom2, atom3, radian): + vector1 = atom1 - atom2 + vector2 = atom3 - atom2 + dot_product = np.dot(vector1, vector2) + magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) + angle = np.arccos(dot_product / magnitude_product) + if not radian: + angle = np.degrees(angle) + return angle + +def rotation_matrix(axis, angle): + cos_theta = np.cos(angle) + sin_theta = np.sin(angle) + ux, uy, uz = axis + return np.array([ + [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], + [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], + [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] + ]) + +plane_normal = np.cross(H1, H2) +plane_normal = plane_normal / np.linalg.norm(plane_normal) + +target_plane_normal = np.array([0, 0, 1]) +axis_to_align = np.cross(plane_normal, target_plane_normal) +axis_to_align /= np.linalg.norm(axis_to_align) +angle_to_align = np.arccos(np.clip(np.dot(plane_normal, target_plane_normal), -1.0, 1.0)) + +rot_matrix_align_plane = rotation_matrix(axis_to_align, angle_to_align) +H1_rotated = np.dot(rot_matrix_align_plane, H1) +H2_rotated = np.dot(rot_matrix_align_plane, H2) + +bondlength1 = calculate_bondlength(H1, O1) +bondlength2 = calculate_bondlength(H2, O1) +bondangle = calculate_bondangle(H1, O1, H2, False) + +print('Original system') +print(f'H1 = {H1}') +print(f'H2 = {H2}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') + +bondlength1 = calculate_bondlength(H1_rotated, O1) +bondlength2 = calculate_bondlength(H2_rotated, O1) +bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) + +print('Aligned system') +print(f'H1 = {H1_rotated}') +print(f'H2 = {H2_rotated}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') From 3e5cf8fd8c405a25bce149e1cbdccfdcb9c88d6b Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 13:18:46 -0800 Subject: [PATCH 31/50] Fix mistake --- examples/PinnedH2O/generate_coord_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py index 181691a5..9c1ef06a 100644 --- a/examples/PinnedH2O/generate_coord_simple.py +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -20,7 +20,7 @@ for d_bondangle in bondangle_increment: bondangle = ref_bondangle + d_bondangle x = ref_bondlength * np.cos(np.radians(bondangle / 2)) - y = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.sin(np.radians(bondangle / 2)) for f_bondlength1 in bondlength1_factor: for f_bondlength2 in bondlength2_factor: H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) From 7982658b4e091366d68635383ed0faaeb617d6a5 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 5 Nov 2024 13:28:40 -0800 Subject: [PATCH 32/50] Minor fix to rotation orientation --- examples/PinnedH2O/rotation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py index 7f74c55b..63b81d4e 100644 --- a/examples/PinnedH2O/rotation_test.py +++ b/examples/PinnedH2O/rotation_test.py @@ -27,7 +27,7 @@ def rotation_matrix(axis, angle): [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] ]) -plane_normal = np.cross(H1, H2) +plane_normal = np.cross(H2, H1) plane_normal = plane_normal / np.linalg.norm(plane_normal) target_plane_normal = np.array([0, 0, 1]) From 5ff4a66b9e161019fb284c69cbc83d9f0b73f7ec Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 14 Nov 2024 09:02:18 -0800 Subject: [PATCH 33/50] Make system unique --- examples/PinnedH2O/generate_coord_simple.py | 7 +++---- examples/PinnedH2O/rotation_test.py | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/PinnedH2O/generate_coord_simple.py b/examples/PinnedH2O/generate_coord_simple.py index 9c1ef06a..7e799bdd 100644 --- a/examples/PinnedH2O/generate_coord_simple.py +++ b/examples/PinnedH2O/generate_coord_simple.py @@ -7,8 +7,7 @@ ref_bondangle = 104.5 # factors and increments for bond lengths and bond angle -bondlength1_factor = np.linspace(0.95, 1.05, 11) -bondlength2_factor = np.linspace(0.95, 1.05, 11) +bondlength_factor = np.linspace(0.95, 1.05, 11) bondangle_increment = np.linspace(-5, 5, 11) # output directory @@ -21,8 +20,8 @@ bondangle = ref_bondangle + d_bondangle x = ref_bondlength * np.cos(np.radians(bondangle / 2)) y = ref_bondlength * np.sin(np.radians(bondangle / 2)) - for f_bondlength1 in bondlength1_factor: - for f_bondlength2 in bondlength2_factor: + for i, f_bondlength1 in enumerate(bondlength_factor): + for f_bondlength2 in bondlength_factor[:(i+1)]: H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0]) filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" diff --git a/examples/PinnedH2O/rotation_test.py b/examples/PinnedH2O/rotation_test.py index 63b81d4e..79f1aaef 100644 --- a/examples/PinnedH2O/rotation_test.py +++ b/examples/PinnedH2O/rotation_test.py @@ -53,8 +53,10 @@ def rotation_matrix(axis, angle): bondlength1 = calculate_bondlength(H1_rotated, O1) bondlength2 = calculate_bondlength(H2_rotated, O1) bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) +if bondlength1 < bondlength2: + H1_rotated, H2_rotated, bondlength1, bondlength2 = H2_rotated, H1_rotated, bondlength2, bondlength1 -print('Aligned system') +print('Rotated system in z=0 plane about x=0 axis, with longer bondlength in H1') print(f'H1 = {H1_rotated}') print(f'H2 = {H2_rotated}') print(f'Bondlength of O1-H1 = {bondlength1}') From 1067aaf1554802d8556f8ef0c229adaaa9296d8a Mon Sep 17 00:00:00 2001 From: "Kevin\" Seung Whan Chung" Date: Sat, 21 Dec 2024 08:54:12 -0800 Subject: [PATCH 34/50] Merge release (#293) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * remove a redundant assignment in DensityMatrix::DensityMatrix --------- Co-authored-by: Jean-Luc Fattebert --- src/DensityMatrix.cc | 54 +++++- src/DensityMatrix.h | 10 +- src/ExtendedGridOrbitals.cc | 28 +-- src/ExtendedGridOrbitals.h | 3 +- src/HDFrestart.cc | 8 +- src/HDFrestart.h | 32 ++-- src/LBFGS_IonicStepper.cc | 13 +- src/LocGridOrbitals.cc | 27 +-- src/LocGridOrbitals.h | 3 +- src/MGmol.h | 2 +- src/ProjectedMatrices.cc | 115 +++--------- src/ProjectedMatrices.h | 13 +- src/ProjectedMatricesInterface.h | 26 ++- src/hdf_tools.cc | 292 ++++++++++++++++++++----------- src/hdf_tools.h | 6 +- src/md.cc | 49 ++++-- src/restart.cc | 7 +- tests/CMakeLists.txt | 13 +- tests/MD_MVP/li2.xyz | 4 + tests/MD_MVP/md.cfg | 39 +++++ tests/MD_MVP/quench.cfg | 34 ++++ tests/MD_MVP/test.py | 89 ++++++++++ 22 files changed, 566 insertions(+), 301 deletions(-) create mode 100644 tests/MD_MVP/li2.xyz create mode 100644 tests/MD_MVP/md.cfg create mode 100644 tests/MD_MVP/quench.cfg create mode 100755 tests/MD_MVP/test.py diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index e2c7b473..2f5e7c1c 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -12,6 +12,8 @@ #include "DistMatrix.h" #include "MGmol_MPI.h" #include "ReplicatedMatrix.h" +#include "ReplicatedWorkSpace.h" +#include "hdf_tools.h" #include #include @@ -22,16 +24,25 @@ const double factor_kernel4dot = 10.; #define PROCRUSTES 0 +#define MGMOL_DENSITYMATRIX_FAIL(X) \ + { \ + std::cerr << "DensityMatrix failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } + // occupations in [0,1] // DM eigenvalues in [0,orbital_occupation] template DensityMatrix::DensityMatrix(const int ndim) + : dim_(ndim), + orbitals_index_(-1), + occ_uptodate_(false), + uniform_occ_(false), + stripped_(false) { assert(ndim > 0); - dim_ = ndim; - occ_uptodate_ = false; stripped_ = false; uniform_occ_ = false; @@ -39,8 +50,6 @@ DensityMatrix::DensityMatrix(const int ndim) MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; - orbitals_index_ = -1; - dm_ = new MatrixType("DM", ndim, ndim); kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); @@ -438,6 +447,43 @@ void DensityMatrix::mix( orbitals_index_ = new_orbitals_index; } +template +int DensityMatrix::write(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + + wspace.initSquareMatrix(*dm_); + + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); +} + +template +int DensityMatrix::read(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + int ierr = 0; + if (mmpi.instancePE0()) + { + hid_t file_id = h5f_file.file_id(); + ierr = mgmol_tools::read_matrix(file_id, name, work_matrix); + } + mmpi.bcast(&ierr, 1); + + if (ierr >= 0) wspace.mpiBcastSquareMatrix(); + if (ierr >= 0) initMatrix(work_matrix); + + return ierr; +} + template class DensityMatrix>; #ifdef HAVE_MAGMA template class DensityMatrix; diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 84804a86..1d09e79e 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -10,6 +10,7 @@ #ifndef MGMOL_DENSITYMATRIX_H #define MGMOL_DENSITYMATRIX_H +#include "HDFrestart.h" #include "MGmol_MPI.h" #include "global.h" @@ -23,7 +24,7 @@ template class DensityMatrix { - int dim_; + const int dim_; std::vector occupation_; MatrixType* dm_; @@ -35,6 +36,10 @@ class DensityMatrix bool occ_uptodate_; bool uniform_occ_; bool stripped_; + + /*! + * Max. occupation of an orbital: 1 with spin, 2 otherwise + */ double orbital_occupation_; DensityMatrix(); @@ -146,6 +151,9 @@ class DensityMatrix double getExpectation(const MatrixType& A); void mix( const double mix, const MatrixType& matA, const int new_orbitals_index); + + int write(HDFrestart& h5f_file, std::string& name); + int read(HDFrestart& h5f_file, std::string& name); }; #endif diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 12e5b04c..19c0d26b 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -584,7 +584,6 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -603,7 +602,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -618,28 +617,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int ExtendedGridOrbitals::write_hdf5( - HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int ExtendedGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int ExtendedGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { if (onpe0) (*MPIdata::sout) << "ExtendedGridOrbitals::write_func_hdf5()...\n"; @@ -829,7 +807,7 @@ int ExtendedGridOrbitals::read_func_hdf5( const std::string datasetname(getDatasetName(name, icolor)); // check if dataset exists... - int err_id = h5f_file.dset_exists(datasetname); + int err_id = h5f_file.checkDataExists(datasetname); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index fb7a4b42..c1a718d0 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -374,8 +374,7 @@ class ExtendedGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, ExtendedGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index f95292ac..08018d47 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -933,12 +933,12 @@ int HDFrestart::getLRCenters(std::multimap& centers, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) @@ -1052,12 +1052,12 @@ int HDFrestart::getLRs(std::shared_ptr lrs, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) { // dataset does not exists diff --git a/src/HDFrestart.h b/src/HDFrestart.h index fef634c2..7ffadff7 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -155,28 +155,30 @@ class HDFrestart return 0; } - hid_t dset_exists(const std::string& datasetname) const + hid_t checkDataExists(const std::string& datasetname) const { - if (active_) - { - return dset_exists(datasetname.c_str()); - } - return 0; + return checkDataExists(datasetname.c_str()); } - herr_t dset_exists(const char* const datasetname) const + herr_t checkDataExists(const char* const datasetname) const + { + herr_t err_id = checkDataExistsLocal(datasetname); + + short id = (short)err_id; + MPI_Bcast(&id, 1, MPI_SHORT, 0, comm_data_); + return (herr_t)id; + } + + hid_t checkDataExistsLocal(const std::string& datasetname) const + { + return checkDataExistsLocal(datasetname.c_str()); + } + + herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; if (active_) { err_id = H5LTfind_dataset(file_id_, datasetname); - - if (err_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::dset_exists() failed for dataset " - << datasetname << std::endl; - } } return err_id; } diff --git a/src/LBFGS_IonicStepper.cc b/src/LBFGS_IonicStepper.cc index caea5f84..93ce03f5 100644 --- a/src/LBFGS_IonicStepper.cc +++ b/src/LBFGS_IonicStepper.cc @@ -328,11 +328,11 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) short check_data = 0; std::vector attr_d(16); + // Open an existing dataset. + std::string datasetname("/LBFGS"); + int err_id = h5f_file.checkDataExists(datasetname); if (onpe0) { - // Open an existing dataset. - std::string datasetname("/LBFGS"); - int err_id = h5f_file.dset_exists(datasetname); if (err_id < 0) { // dataset does not exists (*MPIdata::sout) << "Warning: no dataset " << datasetname @@ -343,10 +343,9 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) dataset_id = H5Dopen2(file_id, "/LBFGS", H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " - "no restart info for LBFGS" - << std::endl; + (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " + "no restart info for LBFGS" + << std::endl; } else { diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index ae8a4d1d..4a1b9442 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -966,7 +966,6 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -984,7 +983,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -997,27 +996,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int LocGridOrbitals::write_hdf5(HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int LocGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int LocGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { Control& ct = *(Control::instance()); hid_t file_id = h5f_file.file_id(); @@ -1249,7 +1228,7 @@ int LocGridOrbitals::read_func_hdf5( const std::string key(itcenter->first); // checkif dataset exists... - int err_id = h5f_file.dset_exists(key); + int err_id = h5f_file.checkDataExistsLocal(key); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/LocGridOrbitals.h b/src/LocGridOrbitals.h index 5616c6b6..951e418f 100644 --- a/src/LocGridOrbitals.h +++ b/src/LocGridOrbitals.h @@ -410,8 +410,7 @@ class LocGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, LocGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/MGmol.h b/src/MGmol.h index f7525600..1eb8b2d3 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -133,7 +133,7 @@ class MGmol : public MGmolInterface KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij); void computeHnlPhiAndAdd2HPhi(Ions& ions, OrbitalsType& phi, OrbitalsType& hphi, const KBPsiMatrixSparse* const kbpsi); - int dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, + int dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count); diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 3606aa40..74ff01ed 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -26,6 +26,7 @@ #include "SparseDistMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include "fermi.h" +#include "hdf_tools.h" #include #include @@ -732,114 +733,50 @@ double ProjectedMatrices::checkCond( (*MPIdata::sout) << " CONDITION NUMBER OF THE OVERLAP MATRIX EXCEEDS TOL: " << rcond << "!!!" << std::endl; - Control& ct = *(Control::instance()); if (flag) mmpi.abort(); } return rcond; } -////// TEMPLATE THIS FOR FLOAT OPTION ?? template -int ProjectedMatrices::writeDM_hdf5(HDFrestart& h5f_file) +int ProjectedMatrices::writeDM(HDFrestart& h5f_file) { - hid_t file_id = h5f_file.file_id(); + std::string name("/Density_Matrix"); + return dm_->write(h5f_file, name); +} + +template +int ProjectedMatrices::writeSavedDM(HDFrestart& h5f_file) +{ + std::string name("/Density_Matrix_WF"); ReplicatedWorkSpace& wspace( ReplicatedWorkSpace::instance()); - wspace.initSquareMatrix(dm_->getMatrix()); - - if (file_id < 0) return 0; - - hsize_t dims[2] = { dim_, dim_ }; - - // filespace identifier - hid_t dataspace = H5Screate_simple(2, dims, nullptr); - - hid_t dset_id = H5Dcreate2(file_id, "/Density_Matrix", H5T_NATIVE_DOUBLE, - dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5: " - "H5Dcreate2 failed!!!" - << std::endl; - return -1; - } - - hid_t memspace = dataspace; - hid_t filespace = dataspace; + const MatrixType* matrix = mat_X_old_.get(); + wspace.initSquareMatrix(*matrix); DISTMATDTYPE* work_matrix = wspace.square_matrix(); - herr_t status = H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, - H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) << "Orbitals: H5Dwrite failed!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Dclose failed!!!" - << std::endl; - return -1; - } - status = H5Sclose(dataspace); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Sclose failed!!!" - << std::endl; - return -1; - } - return 0; + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); } -////// TEMPLATE THIS FOR FLOAT OPTION ?? + template -int ProjectedMatrices::read_dm_hdf5(hid_t file_id) +int ProjectedMatrices::readDM(HDFrestart& h5f_file) { - ReplicatedWorkSpace& wspace( - ReplicatedWorkSpace::instance()); - DISTMATDTYPE* work_matrix = wspace.square_matrix(); + std::string name("/Density_Matrix"); + return dm_->read(h5f_file, name); +} - int ierr = 0; +template +int ProjectedMatrices::readWFDM(HDFrestart& h5f_file) +{ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.instancePE0()) - { - hid_t dset_id = H5Dopen2(file_id, "/Density_Matrix", H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) - << "H5Dopen failed for /Density_Matrix!!!" << std::endl; - } - else - { - ierr = 1; - herr_t status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) - << "H5Dread failed for /Density_Matrix!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - } - mmpi.bcast(&ierr, 1); - if (ierr >= 0) wspace.mpiBcastSquareMatrix(); - if (ierr >= 0) dm_->initMatrix(work_matrix); - - return ierr; + mmpi.barrier(); + if (mmpi.PE0()) std::cout << "ProjectedMatrices::readWFDM..." << std::endl; + std::string name("/Density_Matrix_WF"); + return dm_->read(h5f_file, name); } template diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 6725b827..f2881763 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -93,7 +93,14 @@ class ProjectedMatrices : public ProjectedMatricesInterface std::unique_ptr matHB_; std::unique_ptr matH_; + /*! + * Density Matrix + */ std::unique_ptr> dm_; + + /*! + * Gram matrix of orbitals overlaps + */ std::unique_ptr> gm_; // work matrix for tmp usage @@ -311,8 +318,10 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropy() override; double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; - int writeDM_hdf5(HDFrestart& h5f_file) override; - int read_dm_hdf5(hid_t file_id) override; + int writeDM(HDFrestart& h5f_file) override; + int writeSavedDM(HDFrestart& h5f_file); + int readDM(HDFrestart& h5f_file) override; + int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; void updateDM(const int iterative_index) override; void updateDMwithEigenstates(const int iterative_index); diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index 653b386d..c52595e1 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -265,19 +265,35 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction exitWithErrorMessage("updateDMwithRelax"); } - virtual int read_dm_hdf5(hid_t file_id) + virtual int readDM(HDFrestart& h5f_file) { - (void)file_id; + (void)h5f_file; + + exitWithErrorMessage("readDM"); + + return 0; + } + virtual int readWFDM(HDFrestart& h5f_file) + { + (void)h5f_file; + + exitWithErrorMessage("readWFDM"); + + return 0; + } + virtual int writeDM(HDFrestart& h5f_file) + { + (void)h5f_file; - exitWithErrorMessage("read_dm_hdf5"); + exitWithErrorMessage("writeDM"); return 0; } - virtual int writeDM_hdf5(HDFrestart& h5f_file) + virtual int writeSavedDM(HDFrestart& h5f_file) { (void)h5f_file; - exitWithErrorMessage("writeDM_hdf5"); + exitWithErrorMessage("writeSavedDM"); return 0; } diff --git a/src/hdf_tools.cc b/src/hdf_tools.cc index 69a105dc..e56c0478 100644 --- a/src/hdf_tools.cc +++ b/src/hdf_tools.cc @@ -14,12 +14,16 @@ #include #include -using namespace std; +#define MGMOL_HDF5_FAIL(X) \ + { \ + std::cerr << "MGMOL_HDF5 failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } namespace mgmol_tools { -void write1d( - hid_t file_id, const string& datasetname, vector& data, size_t length) +void write1d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t length) { assert(file_id >= 0); @@ -31,7 +35,7 @@ void write1d( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "write1d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write1d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -40,7 +44,7 @@ void write1d( dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write1d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write1d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -49,24 +53,24 @@ void write1d( dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write1d(), H5Dwrite failed!!!" << endl; + std::cerr << "write1d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write1d(), H5Dclose failed!!!" << endl; + std::cerr << "write1d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d( - hid_t file_id, const string& datasetname, vector& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -175,7 +179,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -184,28 +188,28 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d(hid_t file_id, const string& datasetname, vector& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -224,14 +228,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -239,7 +244,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, tc.push_back(t); } - string attname("String_Length"); + std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; hid_t dataspaceA_id = H5Screate_simple(1, dimsA, nullptr); hid_t attribute_id = H5Acreate2(dataset_id, attname.c_str(), H5T_NATIVE_INT, @@ -248,15 +253,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, = H5Awrite(attribute_id, H5T_NATIVE_USHORT, &IonData_MaxStrLength); if (status < 0) { - cerr << "write2d(), Attribute: " << attname << " --- H5Awrite failed!!!" - << endl; + std::cerr << "write2d(), Attribute: " << attname + << " --- H5Awrite failed!!!" << std::endl; } status = H5Dwrite(dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tc[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -265,25 +270,25 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, status = H5Sclose(dataspaceA_id); if (status < 0) { - cerr << "write2d(), H5Sclose failed!!!" << endl; + std::cerr << "write2d(), H5Sclose failed!!!" << std::endl; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "write2d(), H5Aclose failed!!!" << endl; + std::cerr << "write2d(), H5Aclose failed!!!" << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; } } #ifdef MGMOL_USE_HDF5P -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -298,15 +303,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -317,8 +324,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -333,7 +340,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -345,7 +353,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -355,8 +363,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -371,15 +379,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -390,8 +400,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -406,7 +416,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -418,7 +429,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_USHORT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -428,8 +439,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -444,15 +455,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -463,8 +476,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -479,7 +492,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -491,7 +505,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -501,13 +515,13 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); @@ -521,15 +535,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -540,8 +556,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -556,7 +572,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -565,8 +582,9 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -576,7 +594,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -589,7 +607,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, #endif void addAttribute2Dataset( - hid_t dset_id, const char* attname, const vector& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -600,30 +618,30 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); - if (status < 0) cerr << "H5Sclose failed!!!" << endl; + if (status < 0) std::cerr << "H5Sclose failed!!!" << std::endl; //(*MPIdata::sout)<<"Write attribute "<& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -634,21 +652,21 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_INT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); if (status < 0) { - cerr << "H5Sclose failed!!!" << endl; + std::cerr << "H5Sclose failed!!!" << std::endl; return; } @@ -656,14 +674,14 @@ void addAttribute2Dataset( status = H5Awrite(attribute_id, H5T_NATIVE_INT, &attr_data[0]); if (status < 0) { - cerr << "H5Awrite failed!!!" << endl; + std::cerr << "H5Awrite failed!!!" << std::endl; return; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "H5Aclose failed!!!" << endl; + std::cerr << "H5Aclose failed!!!" << std::endl; } } @@ -676,7 +694,7 @@ int whatisopen(hid_t fid) if (cnt <= 0) return cnt; - if (cnt > 1) cout << "HDF5 file: " << cnt << " object(s) open\n"; + if (cnt > 1) std::cout << "HDF5 file: " << cnt << " object(s) open\n"; // objs = malloc(cnt * sizeof(hid_t)); hid_t* objs = new hid_t[cnt]; @@ -699,4 +717,80 @@ int whatisopen(hid_t fid) return howmany; } + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim) +{ + if (file_id < 0) return 0; + + hsize_t dims[2] = { (hsize_t)dim, (hsize_t)dim }; + + // filespace identifier + hid_t dataspace = H5Screate_simple(2, dims, nullptr); + + hid_t dset_id = H5Dcreate2(file_id, name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dcreate2 failed!!!"); + return -1; + } + + hid_t memspace = dataspace; + hid_t filespace = dataspace; + + herr_t status = H5Dwrite( + dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dwrite failed!!!"); + return -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + return -1; + } + status = H5Sclose(dataspace); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Sclose failed!!!"); + return -1; + } + + return 0; +} + +int read_matrix(hid_t file_id, std::string& name, double* matrix) +{ + int ierr = 0; + hid_t dset_id = H5Dopen2(file_id, name.c_str(), H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dopen failed!!"); + ierr = -1; + } + else + { + herr_t status = H5Dread( + dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dread failed!!"); + ierr = -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + ierr = -1; + } + } + + return ierr; } + +} // namespace diff --git a/src/hdf_tools.h b/src/hdf_tools.h index e1a0ed1f..7a64bdce 100644 --- a/src/hdf_tools.h +++ b/src/hdf_tools.h @@ -6,8 +6,8 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE - #include "hdf5.h" + #include #include #include @@ -39,4 +39,8 @@ void addAttribute2Dataset( void addAttribute2Dataset( hid_t dset_id, const char* attname, const std::vector& attr_data); int whatisopen(hid_t fid); + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim); +int read_matrix(hid_t file_id, std::string& name, double* matrix); } diff --git a/src/md.cc b/src/md.cc index 314fd0dc..36236f85 100644 --- a/src/md.cc +++ b/src/md.cc @@ -226,7 +226,7 @@ void checkMaxForces(const std::vector& fion, } template -int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, +int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -245,9 +245,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); - OrbitalsType previous_orbitals("ForDumping", **orbitals, false); + OrbitalsType previous_orbitals("ForDumping", orbitals, false); if (!orbitals_extrapol_->getRestartData(previous_orbitals)) - previous_orbitals.assign(**orbitals); + previous_orbitals.assign(orbitals); + + // write all restart info in HDF5 file int ierr = write_hdf5(h5file, rho.rho_, ions, previous_orbitals, lrs_); mmpi.allreduce(&ierr, 1, MPI_MIN); @@ -259,12 +261,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, << std::endl; return ierr; } - // write_hdf5(h5file, rho.rho_, ions, *orbitals_minus1); - // stepper->write_hdf5(h5file); if (write_extrapolated_wf && ct.out_restart_info > 2) { - ierr = (*orbitals)->write_func_hdf5(h5file, "ExtrapolatedFunction"); + // write extra info needed for seamless MD restart + ierr = orbitals.write(h5file, "ExtrapolatedFunction"); mmpi.allreduce(&ierr, 1, MPI_MIN); if (ierr < 0) { @@ -274,6 +275,10 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, << std::endl; return ierr; } + + // write DM associated with non-extrapolated wavefunctions + // (last computed solution of KS equations) + proj_matrices_->writeSavedDM(h5file); } ierr = h5file.close(); @@ -354,16 +359,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.restart_info > 1) { - int flag_extrapolated_data = 0; - if (onpe0) - { + int flag_extrapolated_data + = h5f_file_->checkDataExists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0000"); - if (flag_extrapolated_data == 0) - flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0"); - } - MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + = h5f_file_->checkDataExists("ExtrapolatedFunction0"); if (ct.restart_info > 2) { @@ -377,7 +377,6 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) // need to reset a few things as we just read new orbitals (*orbitals)->computeGramAndInvS(); - dm_strategy_->update(*current_orbitals_); } DFTsolver::setItCountLarge(); @@ -622,6 +621,13 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) lrs_->clearOldCenters(); } + // save DM for possible restart write + // note: extrapolation is going to modify it! + if ((ct.out_restart_info > 2) + && (((md_iteration_ % ct.checkpoint) == 0) + || (mdstep == ct.num_MD_steps))) + proj_matrices_->saveDM(); + preWFextrapolation(); if (ct.dt > 0. @@ -653,7 +659,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) std::cout @@ -692,7 +698,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) @@ -712,6 +718,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) template void MGmol::loadRestartFile(const std::string filename) { + if (onpe0) std::cout << "loadRestartFile..." << std::endl; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); @@ -734,6 +741,12 @@ void MGmol::loadRestartFile(const std::string filename) global_exit(0); } + if (!ct.fullyOccupied()) + { + // overwrite DM with restart data in dataset Density_Matrix_WF + if (h5file.checkDataExists("Density_Matrix_WF")) + ierr = proj_matrices_->readWFDM(h5file); + } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); diff --git a/src/restart.cc b/src/restart.cc index 944b1d8e..2fa76ecb 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -163,9 +163,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, // Write wavefunctions and old centers. if (ct.out_restart_info > 2) { - int ierr = orbitals.write_hdf5(h5f_file); + int ierr = orbitals.write(h5f_file); if (ierr < 0) return ierr; + if (!ct.fullyOccupied()) + { + ierr = proj_matrices_->writeDM(h5f_file); + if (ierr < 0) return ierr; + } if (ct.isLocMode() && ct.WFExtrapolation() == WFExtrapolationType::Reversible) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6de34971..53793986 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -247,6 +247,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc + ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc + ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) @@ -491,6 +493,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMD_MVP + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/quench.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/li2.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testLBFGS COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/LBFGS/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -544,7 +554,7 @@ target_include_directories(testConditionDistMatrix PRIVATE ${Boost_INCLUDE_DIRS} target_include_directories(testConditionDistMatrixPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) @@ -558,6 +568,7 @@ target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) +target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/MD_MVP/li2.xyz b/tests/MD_MVP/li2.xyz new file mode 100644 index 00000000..fce3cd72 --- /dev/null +++ b/tests/MD_MVP/li2.xyz @@ -0,0 +1,4 @@ +2 + +Li 1.33 0.0 0.0 +Li -1.33 0.0 0.0 diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg new file mode 100644 index 00000000..83d5aa4c --- /dev/null +++ b/tests/MD_MVP/md.cfg @@ -0,0 +1,39 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=MD +[MD] +num_steps=5 +dt=15. +[Quench] +solver=PSD +max_steps=30 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +input_filename=wave.out +input_level=3 +output_level=3 diff --git a/tests/MD_MVP/quench.cfg b/tests/MD_MVP/quench.cfg new file mode 100644 index 00000000..135dbde7 --- /dev/null +++ b/tests/MD_MVP/quench.cfg @@ -0,0 +1,34 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py new file mode 100755 index 00000000..489c9c8c --- /dev/null +++ b/tests/MD_MVP/test.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string +import shutil + +print("Test MD with MVP solver...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-5): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp1 = sys.argv[nargs-4] +inp2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst = 'pseudo.Li_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run quench +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp1,coords) +print("Run command: {}".format(command)) +output1 = subprocess.check_output(command,shell=True) +lines=output1.split(b'\n') + +#analyse output of quench +for line in lines: + num_matches = line.count(b'%%') + if num_matches: + print(line) + +#run MD +for i in range(2): + command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + print(command) + restart_file = subprocess.check_output(command,shell=True) + restart_file=str(restart_file[:-1],'utf-8') + print(restart_file) + + os.rename(restart_file, 'wave.out') + + #run MGmol + command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) + output2 = subprocess.check_output(command,shell=True) + + #remove used restart files + shutil.rmtree('wave.out') + + #analyse mgmol standard output + lines=output2.split(b'\n') + + print("Check energy conservation...") + tol = 1.e-4 + energy = 0. + count = 0 + for line in lines: + if line.count(b'%%'): + print(line) + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +#remove last restart files +command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +restart_file = subprocess.check_output(command,shell=True) +restart_file=str(restart_file[:-1],'utf-8') +shutil.rmtree(restart_file) + +sys.exit(0) From c6bec674eb8b341eed4ffc8ecce2e69c03d2bc38 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 16 Jan 2025 08:45:06 -0800 Subject: [PATCH 35/50] Merge release to ROMFPMD (#298) --- src/Control.cc | 72 ------ src/Control.h | 4 +- src/HDFrestart.cc | 15 ++ src/HDFrestart.h | 3 + src/Hamiltonian.cc | 7 +- src/MVPSolver.cc | 4 +- src/setup.cc | 17 +- tests/CMakeLists.txt | 12 + tests/FIRE/test.py | 6 +- tests/LBFGS/test.py | 8 +- tests/MD_D72/test.py | 6 +- tests/RestartEnergyAndForces/h2o.xyz | 6 + tests/RestartEnergyAndForces/mgmol.cfg | 33 +++ tests/RestartEnergyAndForces/restart.cfg | 32 +++ tests/RestartEnergyAndForces/test.py | 88 ++++++++ .../testRestartEnergyAndForces.cc | 206 ++++++++++++++++++ tests/ShortSighted/test.py | 6 +- 17 files changed, 440 insertions(+), 85 deletions(-) create mode 100644 tests/RestartEnergyAndForces/h2o.xyz create mode 100644 tests/RestartEnergyAndForces/mgmol.cfg create mode 100644 tests/RestartEnergyAndForces/restart.cfg create mode 100755 tests/RestartEnergyAndForces/test.py create mode 100644 tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc diff --git a/src/Control.cc b/src/Control.cc index 3f8a11cc..83fe2093 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1119,78 +1119,6 @@ void Control::printThermostatInfo(std::ostream& os) const } } -int Control::readOccupations(std::ifstream* tfile) -{ - int count = 0; - float nel = 0.; - do - { - float t1 = 0.; - int nst = 0; - if (mype_ == 0) - { -#ifdef DEBUG - (*MPIdata::sout) << " Occupations of states..." << std::endl; -#endif - (*tfile) >> nst; - if (nst <= 0) - { - (*MPIdata::sout) - << "Control::readOccupations: numst=" << numst << std::endl; - (*MPIdata::sout) << "Control::readOccupations: nst=" << nst - << ", count=" << count << std::endl; - (*MPIdata::sout) << "Control::readOccupations: Bad repeat " - "count for state occupations" - << std::endl; - return -1; - } - if ((count + nst) > numst) - { - (*MPIdata::sout) << "Control::readOccupations: Occupations " - "specified for too many states" - << std::endl; - return -1; - } - - (*tfile) >> t1; - if (t1 < 0.) - { - (*MPIdata::sout) - << "Control::readOccupations: occupation=" << t1 - << std::endl; - (*MPIdata::sout) << "Control::readOccupations: occupation " - "should be a positive number" - << std::endl; - return -1; - } - finishRead(*tfile); - } - int mpirc = MPI_Bcast(&nst, 1, MPI_INT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation numbers failed!!!" << std::endl; - return -1; - } - mpirc = MPI_Bcast(&t1, 1, MPI_FLOAT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation failed!!!" << std::endl; - return -1; - } - nel += nst * t1; - count += nst; - - } while (count < numst); - - nel_ = (int)nel; - - nempty_ = (2 * numst - (int)nel) / 2; - - return count; -} - void Control::setLocMode(const float radius, const float lx, const float ly, const float lz, const float mind_centers) { diff --git a/src/Control.h b/src/Control.h index 43da0be7..162a68cc 100644 --- a/src/Control.h +++ b/src/Control.h @@ -249,6 +249,7 @@ class Control const float total_spin, std::string run_directory = "."); void setDefaultValues(); + bool withSpin() { return with_spin_; } bool globalColoring() const { return (coloring_algo_ / 10 == 0); } @@ -270,6 +271,8 @@ class Control float getSpin() const { return total_spin_; } + void setNempty(const int nempty) { nempty_ = nempty; } + short getMGlevels() { return mg_levels_; } bool withPreconditioner() const { return (mg_levels_ >= 0); } @@ -301,7 +304,6 @@ class Control const float dtol, const short kim, const short itmax, const short lfil, const short maxfill, const short ilutype); void setSpreadRadius(); - int readOccupations(std::ifstream* tfile); bool checkTimeout() { return timeout_.check(); } bool occupationWidthIsZero() { return occ_width < 1.e-12; } diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 08018d47..ef658955 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -2725,6 +2725,21 @@ void HDFrestart::gatherDataXdir(std::vector& data) } } +int HDFrestart::countFunctionObjects(std::string& name) const +{ + int count = 0; + int found = 0; + do + { + std::string datasetname(getDatasetName(name, count)); + // check if dataset exists... + found = checkDataExists(datasetname); + if (found) count++; + } while (found); // dataset exists + + return count; +} + template int HDFrestart::read_1func_hdf5(float*, const std::string&); template int HDFrestart::read_1func_hdf5(double*, const std::string&); diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 7ffadff7..72c5c1f5 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -173,6 +173,7 @@ class HDFrestart return checkDataExistsLocal(datasetname.c_str()); } + // Returns zero (false), a positive (true) or a negative (failure) value. herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; @@ -282,6 +283,8 @@ class HDFrestart int getMDstepFromFile() const; int getFromFile(const std::string& attname) const; + int countFunctionObjects(std::string& name) const; + hid_t createPlist() { hid_t plist_id = H5P_DEFAULT; diff --git a/src/Hamiltonian.cc b/src/Hamiltonian.cc index ba0aa6c4..5a5d2a53 100644 --- a/src/Hamiltonian.cc +++ b/src/Hamiltonian.cc @@ -59,9 +59,9 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) { (*MPIdata::sout) << "Hamiltonian::applyLocal(), new_index =" - << new_index << endl; + << new_index << std::endl; (*MPIdata::sout) << "Hamiltonian::applyLocal(), itindex_ =" - << itindex_ << endl; + << itindex_ << std::endl; } #endif if (force || new_index != itindex_) @@ -76,7 +76,8 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) (*MPIdata::sout) << "Hamiltonian::hlphi up to date, itindex_=" << itindex_ - << endl; + << ", Potential index=" << pot_->getIterativeIndex() + << std::endl; #endif } return *hlphi_; diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6115f933..7ab9f665 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -143,7 +143,6 @@ void MVPSolver::buildTarget_MVP( proj_mat_work_->setHB2H(); proj_mat_work_->updateDM(orbitals_index); - target = proj_mat_work_->dm(); if (ct.verbose > 2) @@ -286,7 +285,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) std::cout << "MVP --- Target energy..." << std::endl; proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) current_proj_mat->printOccupations(os_); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); const double nel = proj_mat_work_->getNel(); if (onpe0 && ct.verbose > 1) os_ << "MVP --- Number of electrons at beta=1 : " << nel @@ -346,7 +345,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "Number of electrons for interpolated DM = " << pnel << std::endl; } - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, *work_); } diff --git a/src/setup.cc b/src/setup.cc index f022678c..9963bfc5 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -50,6 +50,7 @@ int MGmol::setupFromInput(const std::string filename) if (ct.isLocMode()) mymesh->subdivGridx(ct.getMGlevels()); const pb::PEenv& myPEenv = mymesh->peenv(); + if (ct.restart_info > 0) h5f_file_.reset( new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type)); @@ -58,8 +59,20 @@ int MGmol::setupFromInput(const std::string filename) if (status == -1) return -1; const short myspin = mmpi.myspin(); - const int nval = ions_->getNValenceElectrons(); - ct.setNumst(myspin, nval); + const int nel = ions_->getNValenceElectrons(); + // for the case of extended wavefunctions, we can determine the number + // of empty states from the number of wavefunctions in restart file + if (ct.restart_info > 2 && !ct.short_sighted) + { + std::string name = "Function"; + int count = h5f_file_->countFunctionObjects(name); + std::cout << "found " << count << " functions in restart file..." + << std::endl; + int nempty = ct.withSpin() ? count - nel : count - int(0.5 * nel); + ct.setNempty(nempty); + } + ct.setNumst(myspin, nel); + ct.setTolEnergy(); ct.setSpreadRadius(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53793986..b29c1aad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -256,6 +256,8 @@ add_executable(testWFEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) add_executable(testDMandEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) +add_executable(testRestartEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -371,6 +373,15 @@ add_test(NAME testDMandEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRestartEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRestartEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -567,6 +578,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) diff --git a/tests/FIRE/test.py b/tests/FIRE/test.py index eaf380d9..6866e3ac 100755 --- a/tests/FIRE/test.py +++ b/tests/FIRE/test.py @@ -49,7 +49,11 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) print(command) diff --git a/tests/LBFGS/test.py b/tests/LBFGS/test.py index 2ff69c9a..324fd129 100755 --- a/tests/LBFGS/test.py +++ b/tests/LBFGS/test.py @@ -49,7 +49,12 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') + +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp2,coords,lrs) print(command) @@ -75,6 +80,7 @@ os.remove('wave.out') if force>tol: + print("Force larger than tol {}".format(tol)) sys.exit(1) sys.exit(0) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index a664b305..17b70de2 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz new file mode 100644 index 00000000..cdc906f6 --- /dev/null +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -0,0 +1,6 @@ +3 +https://pubchem.ncbi.nlm.nih.gov/compound/Water +O 2.5369 -0.1550 0.0 +H 3.0739 0.1550 0.0 +H 2.0000 0.1550 0.0 + diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..e590f810 --- /dev/null +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -0,0 +1,33 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +nempty=2 +[Restart] +output_level=3 +output_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=1 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg new file mode 100644 index 00000000..99bc77d8 --- /dev/null +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -0,0 +1,32 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +input_level=3 +input_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=20 diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py new file mode 100755 index 00000000..349434e8 --- /dev/null +++ b/tests/RestartEnergyAndForces/test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test RestartEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-6] +test_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +ref_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + ref_energy=energy + break + +#sys.exit(0) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +test_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'Eks'): + print(line) + words=line.split() + print(words) + test_energy = words[2] + break + + +tol = 1.e-6 +diff=eval(test_energy)-eval(ref_energy) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(ref_energy,test_energy)) + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc new file mode 100644 index 00000000..e323afbc --- /dev/null +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -0,0 +1,206 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + HDFrestart h5file(ct.restart_file, myPEenv, ct.restart_file_type); + std::string name = "Function"; + int count = h5file.countFunctionObjects(name); + if (count != ct.numst) + { + std::cerr << "The number of functions in the restart file, " + << count << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + // read numst_ wavefunction + int nread = orbitals.read_func_hdf5(h5file, name); + if (nread != ct.numst) + { + std::cerr << "The number of functions read from the restart file, " + << nread << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + // set the iterative index to 1 to differentiate it from first instance + // in MGmol initial() function. This is not very clean and could be + // better designed, but works for now + orbitals.setIterativeIndex(1); + + // set initial DM with uniform occupations + projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->printDM(std::cout); + + // + // evaluate energy and forces with wavefunctions just read + // + std::vector forces; + double eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks : " << eks << std::endl; + std::cout << "Forces :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 9b731f4c..27da7f7d 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) From 80c6bfdcf17dac0cddf7cee11812b1c488b2caf5 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 6 Feb 2025 09:10:43 -0800 Subject: [PATCH 36/50] ROM calculations of Pinned H2O with 3 DOF (#289) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * set up offline procedure * Parametric file * Add basis script * Add postprocessing scripts * Add files via upload * Add restore check. Make H1 shorter than H2 in original system for verification * Minor change * Add C++ translation * Add restoring test * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Set up test * Update cfg * Fix conversions * Add new script * Add CMake * Restore * Updates * Changes * Relocate set * Refactor * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Merge release to ROMFPMD * updates * Updates * Fixes * Add computeGramAndInvS * Add * Debug * Clean up * Turn on MVP solver for nempty > 0 case * Add getForces for evaluateDMandEnergyAndForces * Add new tests * Remove redundant line * Fix filename * Add quench and MVP comparison for main executable * minor changes * Changes in setting of Quench and MVP comparison * Add number of quench iterations for convergence * Update job scripts * Update config files * Update toy example * Remove redundant script * Updates * Update scripts * Add functional script * Templatize scripts * Add postprocessing script * Add units to plot --------- Co-authored-by: Jean-Luc Fattebert --- examples/PinnedH2O/generate_coord.py | 63 ----- examples/PinnedH2O/get_ROM_table.py | 28 +++ src/DensityMatrix.cc | 4 - src/ExtendedGridOrbitals.h | 3 + src/MGmol.cc | 2 + src/MGmol.h | 4 +- src/md.cc | 1 - src/rom.cc | 20 ++ tests/CMakeLists.txt | 11 + tests/PinnedH2O_3DOF/README.md | 1 + tests/PinnedH2O_3DOF/assemble_results.sh | 31 +++ tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png | Bin 0 -> 23439 bytes tests/PinnedH2O_3DOF/aux/get_ROM_table.py | 31 +++ .../PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py | 69 ++++++ tests/PinnedH2O_3DOF/aux/rotation_test.cc | 141 +++++++++++ .../PinnedH2O_3DOF/aux}/rotation_test.py | 32 ++- .../compare_energy_and_forces.py | 233 ++++++++++++++++++ tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in | 3 + tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in | 3 + .../PinnedH2O_3DOF/generate_coord.py | 2 +- tests/PinnedH2O_3DOF/job.basis | 47 ++++ tests/PinnedH2O_3DOF/job.basis_test1 | 34 +++ tests/PinnedH2O_3DOF/job.basis_test2 | 39 +++ tests/PinnedH2O_3DOF/job.offline | 61 +++++ tests/PinnedH2O_3DOF/job.online | 70 ++++++ tests/PinnedH2O_3DOF/job.online_test1 | 30 +++ tests/PinnedH2O_3DOF/job.online_test2 | 30 +++ tests/PinnedH2O_3DOF/mgmol_offline.cfg | 30 +++ tests/PinnedH2O_3DOF/mgmol_online.cfg | 36 +++ tests/PinnedH2O_3DOF/mgmol_online_test1.cfg | 32 +++ tests/PinnedH2O_3DOF/mgmol_online_test2.cfg | 36 +++ tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc | 203 +++++++++++++++ tests/RestartEnergyAndForces/job.test | 28 +++ tests/WFEnergyAndForces/job.test | 27 ++ 34 files changed, 1306 insertions(+), 79 deletions(-) delete mode 100644 examples/PinnedH2O/generate_coord.py create mode 100644 examples/PinnedH2O/get_ROM_table.py create mode 100644 tests/PinnedH2O_3DOF/README.md create mode 100644 tests/PinnedH2O_3DOF/assemble_results.sh create mode 100644 tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png create mode 100644 tests/PinnedH2O_3DOF/aux/get_ROM_table.py create mode 100644 tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py create mode 100644 tests/PinnedH2O_3DOF/aux/rotation_test.cc rename {examples/PinnedH2O => tests/PinnedH2O_3DOF/aux}/rotation_test.py (73%) create mode 100644 tests/PinnedH2O_3DOF/compare_energy_and_forces.py create mode 100644 tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in create mode 100644 tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in rename examples/PinnedH2O/generate_coord_simple.py => tests/PinnedH2O_3DOF/generate_coord.py (96%) create mode 100644 tests/PinnedH2O_3DOF/job.basis create mode 100644 tests/PinnedH2O_3DOF/job.basis_test1 create mode 100644 tests/PinnedH2O_3DOF/job.basis_test2 create mode 100644 tests/PinnedH2O_3DOF/job.offline create mode 100644 tests/PinnedH2O_3DOF/job.online create mode 100644 tests/PinnedH2O_3DOF/job.online_test1 create mode 100644 tests/PinnedH2O_3DOF/job.online_test2 create mode 100644 tests/PinnedH2O_3DOF/mgmol_offline.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_online.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_online_test1.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_online_test2.cfg create mode 100644 tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc create mode 100644 tests/RestartEnergyAndForces/job.test create mode 100644 tests/WFEnergyAndForces/job.test diff --git a/examples/PinnedH2O/generate_coord.py b/examples/PinnedH2O/generate_coord.py deleted file mode 100644 index c239eb87..00000000 --- a/examples/PinnedH2O/generate_coord.py +++ /dev/null @@ -1,63 +0,0 @@ -import numpy as np -import os - -# coords.in -O1 = np.array([0.00, 0.00, 0.00]) -ref_H1 = np.array([-0.45, 1.42, -1.07]) -ref_H2 = np.array([-0.45, -1.48, -0.97]) - -# factors and increments for bond lengths and bond angle -bondlength1_factor = np.linspace(0.95, 1.05, 11) -bondlength2_factor = np.linspace(0.95, 1.05, 11) -bondangle_increment = np.linspace(-5, 5, 11) - -# output directory -output_dir = "PinnedH2O_3dof_coords" - -# utilities -def calculate_bondlength(atom1, atom2): - return np.linalg.norm(atom1 - atom2) - -def calculate_bondangle(atom1, atom2, atom3): - vector1 = atom1 - atom2 - vector2 = atom3 - atom2 - dot_product = np.dot(vector1, vector2) - magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) - angle_rad = np.arccos(dot_product / magnitude_product) - angle_deg = np.degrees(angle_rad) - return angle_deg - -# Rodrigues' rotation formula -def rotation_matrix(axis, angle_degrees): - angle = np.radians(angle_degrees) - cos_theta = np.cos(angle) - sin_theta = np.sin(angle) - ux, uy, uz = axis - return np.array([ - [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], - [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], - [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] - ]) - -# generation -os.makedirs(output_dir, exist_ok=True) - -ref_bondlength1 = calculate_bondlength(ref_H1, O1) -ref_bondlength2 = calculate_bondlength(ref_H2, O1) -ref_bondangle = calculate_bondangle(ref_H1, O1, ref_H2) - -normal_vector = np.cross(ref_H1, ref_H2) -normal_unit_vector = normal_vector / np.linalg.norm(normal_vector) - -for d_bondangle in bondangle_increment: - Q = rotation_matrix(normal_unit_vector, d_bondangle) - Q_ref_H2 = np.dot(Q, ref_H2) - for f_bondlength1 in bondlength1_factor: - for f_bondlength2 in bondlength2_factor: - H1 = f_bondlength1 * ref_H1 - H2 = f_bondlength2 * Q_ref_H2 - filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" - with open(filename, "w") as file: - file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") - file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") - file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") diff --git a/examples/PinnedH2O/get_ROM_table.py b/examples/PinnedH2O/get_ROM_table.py new file mode 100644 index 00000000..fa4289f3 --- /dev/null +++ b/examples/PinnedH2O/get_ROM_table.py @@ -0,0 +1,28 @@ +import subprocess +import re + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" + +print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +for t in range(10): + k = 50*(t+1) + snapshots = 4*k + grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 2f5e7c1c..78d7008e 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -43,10 +43,6 @@ DensityMatrix::DensityMatrix(const int ndim) { assert(ndim > 0); - occ_uptodate_ = false; - stripped_ = false; - uniform_occ_ = false; - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index c1a718d0..74af1982 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -400,6 +400,9 @@ class ExtendedGridOrbitals : public Orbitals const pb::Grid& mygrid = mymesh->grid(); return mygrid.maxDomainSize(); } +#ifdef MGMOL_HAS_LIBROM + void set(std::string file_path, int rdim); +#endif }; #endif diff --git a/src/MGmol.cc b/src/MGmol.cc index acc2b944..0a5c4ad4 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1511,6 +1511,8 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, force(*dorbitals, *ions_); + ions_->getForces(forces); + return eks; } diff --git a/src/MGmol.h b/src/MGmol.h index 1eb8b2d3..315a0ff9 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -359,8 +359,8 @@ class MGmol : public MGmolInterface } #ifdef MGMOL_HAS_LIBROM - int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); - void project_orbital(std::string snapshot_dir, int rdim, OrbitalsType& orbitals); + int save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals); + void project_orbital(std::string file_path, int rdim, OrbitalsType& orbitals); #endif }; // Instantiate static variables here to avoid clang warnings diff --git a/src/md.cc b/src/md.cc index 36236f85..01e34a87 100644 --- a/src/md.cc +++ b/src/md.cc @@ -492,7 +492,6 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) force(**orbitals, ions); #ifdef MGMOL_HAS_LIBROM - // TODO: cleanup if (ct.getROMOptions().num_orbbasis > 0) { if (onpe0) diff --git a/src/rom.cc b/src/rom.cc index 19ff66a3..986e136f 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -11,6 +11,7 @@ #ifdef MGMOL_HAS_LIBROM #include "LocGridOrbitals.h" +#include "ExtendedGridOrbitals.h" #include "MGmol.h" #include "librom.h" @@ -91,6 +92,25 @@ void MGmol::project_orbital(std::string file_path, int rdim, Orbit } } +void ExtendedGridOrbitals::set(std::string file_path, int rdim) +{ + const int dim = getLocNumpt(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + pb::GridFunc gf_psi(mymesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector psi; + for (int i = 0; i < rdim; ++i) + { + orbital_basis->getColumn(i, psi); + gf_psi.assign(psi.getData()); + setPsi(gf_psi, i); + } +} + template class MGmol; template class MGmol; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b29c1aad..1855eb1e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -258,6 +258,8 @@ add_executable(testDMandEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) add_executable(testRestartEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) +add_executable(testPinnedH2O_3DOF + ${CMAKE_SOURCE_DIR}/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -382,6 +384,14 @@ add_test(NAME testRestartEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testPinnedH2O_3DOF + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testPinnedH2O_3DOF + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -579,6 +589,7 @@ target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testPinnedH2O_3DOF PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) diff --git a/tests/PinnedH2O_3DOF/README.md b/tests/PinnedH2O_3DOF/README.md new file mode 100644 index 00000000..90551b47 --- /dev/null +++ b/tests/PinnedH2O_3DOF/README.md @@ -0,0 +1 @@ +python3 test.py srun -p debug -n 64 ../../build_quartz/tests/testPinnedH2O_3DOF mgmol_online.cfg coords_1.00_1.00_0.0.in ../../potentials diff --git a/tests/PinnedH2O_3DOF/assemble_results.sh b/tests/PinnedH2O_3DOF/assemble_results.sh new file mode 100644 index 00000000..6a9a4d0d --- /dev/null +++ b/tests/PinnedH2O_3DOF/assemble_results.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +rm -rf PinnedH2O_3DOF_assembled_results +mkdir PinnedH2O_3DOF_assembled_results + +for d in results_*; do + echo "Processing $d" + rm $d/energy_and_forces_*.out + rm $d/*.png + + suffix=$(echo "$d" | cut -d'_' -f2,3,4) + N_l=$(echo "$suffix" | cut -d'_' -f1) + N_theta=$(echo "$suffix" | cut -d'_' -f2) + rdim=$(echo "$suffix" | cut -d'_' -f3) + python3 compare_energy_and_forces.py --N_l "$N_l" --N_theta "$N_theta" --rdim "$rdim" + + cp "$d/Eks_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/Eks_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_H1_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_H1_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_H2_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_H2_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_O1_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_O1_difference_histogram_${suffix}_reproductive.png" + + cp "$d/Eks_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/Eks_difference_histogram_${suffix}_predictive.png" + cp "$d/f_H1_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_H1_difference_histogram_${suffix}_predictive.png" + cp "$d/f_H2_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_H2_difference_histogram_${suffix}_predictive.png" + cp "$d/f_O1_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_O1_difference_histogram_${suffix}_predictive.png" +done + +tar -cvf PinnedH2O_3DOF_assembled_results.tar PinnedH2O_3DOF_assembled_results +mv PinnedH2O_3DOF_assembled_results.tar /p/lustre2/cheung26/scp_local + +echo "Finished assembling results." diff --git a/tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png b/tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png new file mode 100644 index 0000000000000000000000000000000000000000..b484f4c9eba4bfa6f73c8b448826f601c213b4c1 GIT binary patch literal 23439 zcmeFZcTiMYv_5zm5Cl}rC<-V+Bumagy$Vg1oH2o9keoqGS5S~9G#S(eBr7?Cq9Qrx zs3ZZ&p^43`RrAkG)!g!yHk>}^?7i1s;rqU|UudW)9HwEQK@j9HR`D-Q z1R)DR5X{&?O8ALL`_KgZCE;>I-$l#8!o~fLlR2V#$Hmds!Nu0++pM%o_{=Pdam2eYkM@0i?1fjcw{*QStmu7_^hP&9mu4;S4 z&kwu%XzP&nmx+YeWY~Zwub+gRJfMA4%SHCqRAiX+(WQ)7+Lv6Dx5@aJAld zY_p}57;lr$(|8X@`Sy@x9RF(F7p)#qbxOK%pIw8my zmuO=ij~+dyPU7LuC-BHZzdf&E$J)c}3#)6L>T~(6vYjK9-n2ch7^QUgiT?X(&2eH< zncDD8mNlg3N_u)ajqc^Wmnyvpw0y*n{}-T!s*MWpA#cwJ~Oo!oA+L)F$4ZpZZe zz{(=UW!@%qCgIz;USSAQ?6kRbN5p=puiLz){$PN}mv^DE8LhHAD(6IPOD6ok-@0{6 zNllF}Z((zJd}Vc2%frLtMV(5bw804uH6F{f)YJ^C&eZLhw4{cLq2Knu-O77%OfqGC zx4(~$j&@IK|DD#OHr#S!xMe!MFiFf!A8QLZNjxSGG~H z8n0nN$%MdgxqEN*{@xwP{=@Shd z9q~QCb-lAqn*H=!`{~t*fs~Hi-DHdLx7E+RC&Cm8?$;mZH7V1+c<~~PLf6SDuO&&& z>_h8BeYl8q*ZaVYzK{1@f3;mir{9~n(CxignRbBov}FeYo3nWeHgWhdL}qArYc(=9 zHUk!$TT-%YtkAYUFVGi$lmrzkRBkYntsk*OPv)F2rm3EB*Sna}Jij+TQ$o zi-**GwUG}P>YmclpEl~v)Z?G;`}mH53rut&gh?J;1^;z~0zsApDgIyoVSIWPUb}>? z5ho~=vO1koAzfdmKtqEgBWI6m?(W7gJx2~Ajt(&~9WPFF^yh2Z=TQ55dg5|(wbMen zk{?kAA`P$5E6m`E_IH}PQPkdk^t%q+912S6_`Lj-o39Q&l0V=`cLAx9F^HPkYHYyG z_V6;0*gA8JdTr$%e*R6DY=~^3puCvj5ff#Fiv6 z;;zZis=I)h+02l{OrHQJ&r1x8yaeil7dZV391P&?wbBTqenO7~xP$-mS9N(R(NfOe zC?w;WDN1R**Ws#(`$7lVF34w?jSii-EFe!&La~6!!id`TC4YKM#eM10P0={jj~_pp zFI3wPOZM9q-Ocsg*^o%Pja*i?NOhU`+hw=JWrZ*^yfZJBztAU;Kb)u1YUWmS@;py> zhK}$A-0|`WnE;WRQs-%x^=`enM^sF0ZEvnO`Wj!m7I^LYb=rZ$48re2R*>XfK_S(p z#s)zdFA0e$QdOIVv)ovzGj7=av&qe8+T}$CdgX5Ql|J-Gj#AS&ZRe7gaFg9We8q80 z{_uJ651;F9+`O4#PntZAI2Hz{`Gyge98YX^bf~DCS-v+=QxbgCmZ!34=Hao7VU98n zk?|c^JNWFmkT}IABR-%~KTR#+BqL*XoP8PArZ>mcRRXDqS9dF#X)bphz}Lugay6_i zCCKro5TeKM5%(9CGff6BZvQA%&Z_4@&70@vW&qWJTnu5*qqjz&3_0>cD zwyti}X@1iz^yYIt*`0^QhBST(7#J8dG&FEorlq7{ao|(RC7zo%ICK5|4T?ui&yeNk(NI07 zpo-H=kknNCyOiKF^|I9G{E6|`pDrf)wEU>FjryAKQRGFlMQ4UirVGRt?O!FZ!i^UB zGvMdBAMO-bbqf1#uic%)^O%N9(^9!E@Ty7_+TcrMoaS_0^-DBJ)2)qp z{rB^;0+Tk9FU)(5VHg*O3@~ZxjbcJtrJp|KO-?m%NEyW(JJyh?lQ~wo+mfiGrtw;k zm8D|C$?@jRXE$G-dDGR^#X@{sD|HmHiEf~*IV)sgS&w5MD0g>wF{nI?vF#wd0RMB6 ziODn)m(XBe%+1HA28I_|U0t2on0{0}JE1J0k#(+!r@ymveST$fyVe|EFDKZs)^vd4 zBNEW6gQ6@9VpH!OcJ6404A1c*apX=4@ zOjQ-}T)EFEuN2U7w9X3z(desuP12fbOe-DK&#dW-NPJ0Z-H<_>=Pmb3if z8!%Ka;zD+w{iJb;*?nkR zQCW!fg^(Z(hhIy*Ksc6)BD(ZIr3=lO0OcA4ndY`ao|(stl>N5t^v37jdbd)U)@Wc{E1Ig5Y5Je&*I8YJnf2BXS-BWZ><$0s~dfG}d#L9I|a}5q&iOdC7f+ zAZ*i%jHTCHOCqoSq)l<$4w9a4*m#Y38#XGP2=fQ@BjN5pd>=_jbQn3raGJ1jvRvC9BR@eSw{N_?=-+(s$M$-PX=RhIKf4K+ zMFGDC3;O0FWaO%rJA2p*u<@1C(=Soj4_c5V-5+qCik=)Yb`CYG_M1w)DkmqTv^i9Y zOY@QN?pb4Ss%F-pK;J*-#F@(?_BEir|-9U4ka&EP~7lpI4YOASX?94hX zo!C|WD<2#S93XaW7sSY2x%3R4coq$~#cdOFxsyAjoz!DzEI*s6h0GjQQRnyJQuOd> zH?7=lui9eusr`Ir4P#6O1<#n5l>6+g`v|9P^=e30b+zYxWBC9Byj;z@EYr%4#4BXU z2hq7J)YP1CCk|V%SYjEZbQwe%c&3TJCnut*6B=p!Rkp|?&iP-$AjTa0>I@=EMP~jY z%`FLF_w{<83GM8X-aWtTlIg~9=-H&23fBxh$aXgfke`~5SNm%kijmkVO zT+2mDs|q>wjr}$fV6(fpAFEJ_&9fD?ixai^tR6C50u@9&AnFH ztj6RgRbw=(?yaWzq??UOGne@7eV#ExBF>?AG**}8v>UOFiW3Px&2KS#x~6`U{|GWB z&Fs%px%|u^M@M;kO-tyb5R%4%kRkE+TA5}(QPB=MPb-;}mE~?Ptg@**pfAY4^>Z`Nsaf^@X2)fzEdUOaDz1$V6Cc3&eDOYB54!nifj75Qt`ZSvr{nJyU^3 zKR*jZ@jk)EhD?J3odfY5Vw?nNrE(*->)rFl28@OvY77D`2}0pAX(=%q!|uwVva$I+`jInT&%*Ea?o z^!(o-)=~7PuY0lt8pFamIFb}eq?~MO`BDhq0dVciY-VoHP^oic;r=6XDq+jE3+uo8 zwIG7V3YrHodH)v3*8li6mivIly)09cO-nImW~+X|)h~1i)1&E?E`*%wNLS4x^2W;j zL5J$>RMj|Zh;KLocTUN*!6LiP%-pJsH`kx*YDFT}FUhG0I}6;lt?``e3-ftSi1v9* z8?D_)g~8tTI(J@<|31m^!|g|qP3NBN+->S^k-GcrNaTFeS*+%WT`{NoXNvTor0Mi&qm`innQT+>)9?g#4vx$_ zwa;$!h}iaNNJvOXdae-OzIzwUDC9h)qM9UI*|%M2)5|3;UNT^v%?E{e4&R#&=|ZCeL?%~-M4JcTW($NZyneP1&rbO5yp-$FHRJ?FPc^oN4(L6yrddC5wwCS$*+)} zA?>7qhn&wbN$E9nw^KzjUMkm!)D+nbcBQE&1-g+cy@;d!)u;K4-xCOgz$=%rP;R%) zc4w|{j)&TeRCorxc+q~&q2klYD@c&{mN|G+(Z(M?I?|Gq*oTgg)#Rhb3?(rAp`VxQ zKR=}n&YtVe#Jm0OyY)&3v*cms&$~D8V9{?8F1xE?J!3F4^Z=&LmHTKe>P127#Cm5oC+>u5mW) zfkU+U;npSZm}CfY`=#4*55{lK)N{}dGW%RGaWh!==R@$|p99M48^irJua-^4&hEue z+_&0v7T8ImASW^>cHR=++_A|Jf=!p!` zfNvUV4q?P4CAoC#bQ8~Nr7F(##!cRQMMzO*HyfSZkw)yF!h9`Q)E!ATk>b4aVY5h% zbSpzoy==9mp?bd|vO|t^Cu(n(Fxi;${d+?-GZKmd3hw#$6lU=L%-&6hy546c`QdJc z^4Yc~^13dXZV}1H!eExh)OeH!O7qt^9M@)8pFYuEsBHb?gVW+RWQdC6b$Sbxh=(vb z5#eTjVVYWua>$oF!pD0lCgm#SCQ>uQ?lHg5E@}GZ@F9rfBSxtvb+ggWA1+}@B+bF1 zNdM8|(WodMfkvp%>l5d?(sy?-OO`O*jvUL&QxA&mZ}hs)U9uZ7E+r1--PoQpqa9qV zf)NWr@b!qqOmy)8(i2Y-jbrF>%hw6IbBPtrbCUB~= zlVyQO2_sBN7h6os?Dk4yUV_BEf$Pr{!!Mx$#U8b6k-rz8P-pDS<)*fm6~?SQtxv=l zKS6)~)qAmix{}C-)zx?%^IL+a>FW)QqXN9yd+ML#DGE%SmQz*hQ7)5_d^`sOq*%dr z(g2SDJC5PRN?y4gkmbGT`tTMqbQWwz+Q2kX*g9fu$JgJ}q};kUT7Zv_CZHVs5=Wef z^7gKC{&}XHp-@>(rf?bZ5(;=_kBohip4Wy!zHL5M8Ly-ohk75TNAPBezrG)vN%U@F zzg;iAk!>ai8A8oH*o`%v_a8v0#B!*|ZX{pr=t_sM7NBqueKF6U)HCIaO-`#*9Y4R~ z=**^(hERt?FvoCb=jqtV0A!|9J!N~{1kwkv$E=GZUUR&KmhIObJ$f`dRGKepKcqif z1r>jIvjz7Clxj&QxG6R6l)Ed_$!PZ_O6koG^}DVkOZpTw47dC9%ww$k^QU zK%*xFMR1UJW9r!1*e-K&rjK=(x!Q2(97_J&(LuO)o$&Hdr{5Un#TE(CJ`RyQTE@HR z+sm`>sF-~;^o$=qoob4C2hP)yw7X@_M1S^d`uq3q1M#`Jx#3)t8W*G3_h!^P)a7E_ z>OwjyJS}GFk@pmGdh`Jq^(`$u8}ozfO+u|F4Mua#~%Pa6$zB}yFscKkkpI~3|NiZs!4Y{{E})!L4}y5=z$ z49&rVmy1RKdwy57wBl(!jGG4h`PBVlW#TZ$MtE{8;_ENxsQyzwVJAmOS zqECK^S=Jbkw+>qViev(T4RGqKnR>ap`4-J=06Ac9Oi?s}H4?DlE^EJT9XN0x9T1@H zpTYcSf~mJhgRJbWg$sMR-0|M9O*az5N0KL7upm80$G^SmEO|JtG*smH<1@=VpbARr zGV2j1GRQP8PPbsMKUbQWN%d`&>0k{l0skO!oCw=p>I2UVzke8-gm28J_>;oHr)kro zk>T;yY6_j4FgSTdZe7Jfn{xC!r@=uP&+jazP@kX~T&(6Vw3~xuw95x|{wXIo2vUnY zK3RibUZfsIy3rgz2N%aZ?41jMsTGhQ!Zy0~JASLLy3)e|C>?eRK6`}>i9}O%y%R zr*kVSgqoV!>|^q;ZUQ1Lu4R!5PMiH}!Sh35i5_p)_NqyS4fSD$g56XL-3C1 zb)B2bnm!c<_QL}zTbe!hTVK4GdU1i1^J#AMnRI`Dj+3;>sc7zy&!d0a6nD0{+$rTG zgW(21o))7%b3HNCZ*xmi2O20eR-x3ZIs0=wShklq?4ELjkI z{5TjfvlZBZq{mXSprBw^@>O!z*)Htf4zU-&XCPa~W(6@N*Zu^Uvs>KQUQ_PoL=)1u zuTPV2;9|9z%sOUC$(SrB5Ov1$gL-;qzJ(|zo&$mC^Al>6192VoCDsG4l_04jARyqi zHHnXm-0zh`76qJB*S4lDB&3a@(0w6y5L0qSCP2p#AlF~@97$by=8f9G&}k9}D?IZ} z{r6leHwJ9bI_U&A#hB*9haLH?vF35AP-v!OPe7PLSPL8+IyM+BXJ*n27zmNJe__&4x~Gb234yYXnmPz+&_d|=HSEVUG>|{ADo}DEghClM~@IugWiFtUzbYkXu|t*pK(t7>|o& z$&X(GoI$(7;c;NB60+r3u~r@`3Uv3Z&+3r&tb3ypz11Lve)4^1QH(X1xah7@iB!Ng zlvH$cDN8e?6R z7?OJpad`~^t*nd!$+>HAX=!qQxi>vr*gfZ-T&+ERp-kDjH)n-h2^n)Af=#3&;$2kQ zs}6LHNVp8u0f~jZ)uc;5=eT+GHLB9@AjyZ;5XtYrasZ~sPSo{^24sxKeYwiyvvd`uds!7i zv|U_;iF=rl3!GR6S>uLgk@{~W2mu4UY@WPnm5c#hQ>$Bb5tDZH8 zb##4r%ByqDPUGJ?I#Q;XWRN9R6f7d`t<6 z62b+Ds7_nDRvHU&dvABO6*%2Dv0@H<{{H(cI}3duYx^fL?~hPZTQuN(a;y1V=cWLx zNIrh>pY2cr6W`*p5D&{FGhCqX^7Xe@&QR8AP*c=!d62fIO-#s;2OoJ>ql<^jRm*+7 z(QG`NE#?*I2Rp5!&tY)k!)a@=bB=K&|T8e(XrNdFSN8w8kt<` ze~`op=__jij&yp*Ai)d2Ee(tRQ8)z)fw@W3bzl zb*x2p=eN-C!#YMKEH;b~SSK;HuM*}`PH6R7bHw^0V^9cE)i*YBi>?Q=ZSCyPd-HvG z$7@uia^jLoTUWZ)DQ4zePzqR&)jUGc{)hXJfuh*{Y=3^*&+%~)^M;c$8-4c-!}hOu z*EHU@u&`eH)sAD!j7YQB$KB7jT6;{#B_z~oRfv&yy#RaRmDtGpYS_$@Zolw^a=^uXAx~W%17eg0f}=t zT%EE8BFR=X?)Zrlu9J~^*1$1!ZS$*bdUc<{^Ho&U$Zpd!Edwb&e@gNzI6oY(=472Jg zwRrJ2u!~Rj>!ztO7TOORmX?-o|9H+EROtXP$Hd}du9@FfN^+481P^rE0OIuy!qXhy z%&$F)*9g6!rluyaaRZXHuOq~jMt(gFjn_vQB?(}1^!=wHqrdrzfm24tL~eiAIn2(_ z&#!EyftwvGN7{Vn(HGMdF|RYGVy#d9FZ zh#a=Zv&|2e7wfBQuTC}V9jhfGohY+$buB`GNS{9>%KR0;m_u;OE-f?gA?<^ZWNf0`ulONI+6U z7^ReO8^8PCWo7Z8=n&-M_(K7>iShAwU=EqEgkOtHeAl~ImY3Vr<@N{wOP=b7q2`2Y<%<(3vFq`b`?3@z&2KCk4^HMC;3%mZKIol!P9!s~w>?XHD zmR46gm&fa9tMpbRcBEL$NW|YMRje>*t>5|{$7qif0?sxE`|1N}+MiSx6datn(JJM& z_MW|d8&)hhD@owqvu>zej)z(emlcZK`}UG$u4-??NEzq)yYJ)iP*3oxyfK-#&8xiD zXUzfbH3YZA|IpjkHtW>6b2f!Q|MeA7pSYwCjDJtk{_gbMLD;>P=!MF7KHgFD^2_JBMrKz{*-b9&HiwOPyveJ*_YmTg>8lG&%7A`qhVZNTk2f?mBU zWbzn$ba$}KZQdHackpRmuI2Nfp6Nn=hdLSCL3rf{+@hjAP)c9>Xa}R=eVxN?Q?jV!r#4+@lUZBPava=bjszjf~YgSe9g}B#S zP5N_R8Lzd=Me|iV3;F1HIki$x4x)8U;Hb#u%kTR(;6cskC@7Ddhf&xS4_lxkggS_; zh*wy+V9fX3+qXe>{;+!4eSP{in@c~w7A-UW@Ebh_8my2l=Ht>q&3O&W9Q;E)AMe!x zp2`YUI~FQ7eto%n-wyGbR+N0HJ~CKf^^S_kLmo-ohKY59Haq*!{0POZR;6}9m%bC z+F&0dt%2B(o_=8d-~oQlSxZZc7uy3sI=5a)@cjo5 z&{dx#$qc_p>sg?7^@pHxY8GQ$;qft3nJjxs9sFg7+0TSw9PTMaI!(D^Ug zj}@`$N>RoiU)(3RE(2gA-L%px&*bMvdl1%uMRq2*kIWDnkYul;NaimG|DCvJhtH*g zXHkv5a<%q3T`zzDXr==WSQ$KjZ-wV-M&UD{=V#6KxA7z1i!$K4^+M%HB@p98U>9IJ z{zLng$w zHRk86H70olGSd6`{XxcpWhIAe@+l}OXJM~)F8!!2^jfzDffDdtB&2bX*Wo*wXvj{E)qBU5vc{H(R~z~sizEN9^jI5y~@@5l~f4R zV?_T3J(WBnf)5QX@c7tOn(wldO#N70`XX=!ncU<7L3e=H(3aoZMhOiJCRN5gr@+N( z-G4!BR53vk`{`2$Z?Pi1j3oua^#svBKu&q64I%|!sTpW0P5rhet&`}E1~|Jc*an(a zJ!itXE#_3U{#WpU0I0@j8ZSkF(vRbgrw*MHRRQmSM-vQiSm_WRfL^%;St^>yKzjEp z&Y`Lu9IoqJcJVCp{^s*cWy-M&_r58{iDCgSF~6?}kg`#+QunfA_lpxL97z#g>w@!u zYuXK--v{{>73KpCif2MfFx#mv2YGJeTB|=v3k(a_67dtltCN#h>6aApItVRaY;?mht>(nTzk z0^M7y)2Eo2TAW=UG4NehH2nD4__!KYfJj+IDx3M}9Fbcc1(mLDHcc-`z*OU=N?MUI z-x8{}PoF=3K6&cYjZN~3zZ#=DsF+U{C3z_gl(auwOp3OgI`szNGIa{fG$j`knt*@wg)!lV89VvVq%Wp-gY`vQeHDDcr?A# z!&B7UVDkL(aww9p=3@0rgK3zurt&Dlcpo;OB;+UGLYZ^XBn=XB264w^*}XNb{&_b& zJXH-O=TO$AxXr70_wrf&*00awBAVj8F!DE!A&i$n{p$~}P7#72tWordo41cR)VW`2D{k8lUf@h(-{ie;7n%1&~B#8|2@t zs;XiTwtS6Nac=ViGf<_wyLuw=V1RE!9pkQC-G&UIQb8@G>+Mr&UrtU7hUh_ip;l(f zfCeZ){~osQ1-X?ASofdEP~}s~Qv0flB`9K}67zZ!(&B7SOIGqm8OlRr9muU0ezPGjM5@AzA|WJ`2I~qc19{E_0oe ztQ19ZL~No8%!AjO#X7=RJ1U7vW`0&vN3!1g?lk2G@Rv6&yOJ$2p*RF2vpJyEwF1_Y z{h1s#LoYY+$a(Rxq0dN3e0B9&J%?uPvs6ZU4%H;nWn%rmz)10DZ(2;UAgr+<^ zDa*LxT-9?K+ykr64PoKtM-#&h< zx?{F4_~rv!b8Jj~Vrvo}bW%R+zh^4EFkRB1vv`lT5j0R4A?ZUJR-L!a4=0owH$x&$){vwDDSxB*h$)5eXI2q zi|=#Z-JCCL-O>*Xk>ckoaG?sg_9D-8SHj-+LYmV6y-6_pyOe z=d`NrS#Btd^w59b3_0&vKTdh*3;^zgz+Q0B2^zcsD5`V# z9eRwW7G?lY!P-ip$sms4Pg+fy9jPn_jI|4xcg@hl2M=mMftRmzK;!Bk?;fM@#r!@j zbg_Npwd*D(X9HqEfks#4xoYwB>C;nBNx)(3_?ThMEAC1Dyv@w49S!Yr77h znTIQ$HaQSRBIzdFD7VK3MAcMj4Wgm5hIoCLn79u*-_u6SQ2tI#OlXFl19yTZOcu?t zGaw=H+US4aI{r$`8nlTt->a8@vHZ_ETkli;C#LIjyTh$f&AyiV%8282G!`IrqlfaAsgy$tZ-R+|Ew-k&HqwI zn`z&6ffJ~=e2XnE{#FHOrJ(ZN_YHabE_iWH%~kKoKXQ;XwPl4TVyy5sfB$_DN@`=L zYN&0wMMQLrifr2ptL0QlknW>dy%$7;g;pR8BaSf3X8ppE99GJqwc$8F-OkRAcNe`W zFoUSHCcWI6L~<882(mj+p~o$hx0->*B2qWDyvyVkN|604GogIs@d1?rp&83{IvE1m z=>ph$0+ZLTi^E=X**9(?fGbP_su0OG^SeJEhKv(+7|DV(%zC)Yl|kC$15j#fb#kP3 zoobo@<)w9zR$Ax>DG;+4RzHFXXt)6MBQ%j{=$U!~ojaEX(QO8B4q$zQgMWv=ex2Oa zrHw^V55L`Ig<-#)dF#&9SEx3&(5mxhoS1_y^m}mo9Y7|b0ieLg^G@z)W^OF@9VYp0 z3B~Upv%R^w?8;FQrL9lLeF6gm(E=;2tdUGZwahi#)Zc_bZlr+OLw>gHD%ZUMb>#9h z&}%MT8UKc5y%{~Z<|2e$D|JRm=?5q&{rqD;!F@DxpjZsHX7l*Z=q_4$Pt?bfcjwqG z>^D}sy3c4@k$g~k+Zm;|uNT^JSMK}W@%~IPv$JT&mq`^cjk>RMv+cFUJ(Uq}uBUJ$ zUTJhIhnQ2Pq?)Z$`4zM-sOuE+^z5LMQ|vX?s@i+d65+KW+^7)JC+zL^U~)>=%omYn zMm<+km9K@igC}`%b#;MlOvFCLcYswCE0@U377?X7F%WXbFiNq>$-#s+_^Y@SKLtyK~bV%`vR={jA`DJH|ullMt`ZPo!Pv+*EK8IF@10G#8Rj zA^uQJEjkyk^rH%#)4EEXm+jiyHS0nwBm^}x%*-Tx%YMm<#sPM_Y~9^pO!H6tF@$vW z5&C8~{kCo>saCEbuk8k#EPq^54RWA8($db6G;vwHuYjKC3P18)0rmTN_nvRf%yG{T z=xiz}ULKn6;nJzF*S{Ojx3SE+ASRg7Uz;r$SaY@S0&vd{B}S7zDGvZ1`R*w3S6N2! zDnmHq6lh0PNvJ%;;{}igM)m@YX=>QoXEC2HA{BIC5f%SuEOY2*ky(DCtV}k>g1!{X z`Ye`PH~P~>Tnx)pEgdSGm`24M_|hqcne1YkL4Q(yj4eX|5w%bH?am~Ez`Z7?o6m&r z{~ZxE1&T%SKlks7b5KfC>{AjEF#-4gqll94OvQ?g+D3(k zr<5;1V+KBoBAEe|q^_(-M)5^5h>6ZSR0)9J&=5j@iX%`SZc081bLTfN(n+bY3Xen% zQi#jtJvHb6J&M|hb47D$2<+zA`= zs@|UQk*efhkUohD`AkH{tcsPSUS;Z(Hdo$NJIhkN>-=cb4l?7OR|1WKg}? zuOAjo(QlY!ye^B2>$|QdO}kJ>nOhAMSfO0f>a2!y81pqwNjgSr+~F_-BX1 z)#4pdF_!c9&`=Wv3$bOxwYqinuQr+YOiLK%*Rp4s$y?J2O9OB8k>h zDo|Q9tw16wLyYElhAbIV$kU1WZo9~kHpfvcy0Ng@+#puyK?tg9xp@8Ad%z+L3oOHc zkS%bT>9Ec@9l)`&8ZJxH4+;{Thbk-4e+1247@!3MZC%8Lhoen}fvzBXfJ6l^YBw

(qUGzl>*%G&-{KIx{ONgr|pak&nhMCaN}XXDTV z!2f4EQj}BVYAy0lSs6v?*bYCnXo)wts|xKyXpB#pbBru05!b_OSp8|{(R1wvL>4q9 zdUhk{-$9;1c|O=P|(Fq1^KM_ieg#IjYNgeGEdBXX z*FgGUY5Xns)Le$1$F{DdnpzbnSTPW8#ck1Uqa#!^P$;sK{sHj@(i@R( zTttZuE-ob$)&ju-n-Iq+rm(KKD~6wDv%xktYxyr zC|9E#l`?-N0ZKrt-W(GKC`bbzKNh+B<)Gdk10t>R{78q~zCoYg#4)S7Be2Vp@BYH9 zIYJ#34{xvC*0R`bRPL4ydw>@UshWnhwe=@iR1^Z;lDUA~qa0G%LfITz8$O+oTct)C z?tD3DwQ6CtV9d$gsKQX`TdrMQ1vrZ%C*+|e(COHamvuLHcZnkg!vqC0pXH#fC~y#} zpj&BZYU+YKqWjHts%d(5*W=0kGlt%auuFQ6K zDD(f(YojHEHmqfb0y9>&dY^f+befGW?%07PN${cak;6XupsTd#&udx@JaGcG{0`I& z9a(M?>c}pRorI>pJGD(?N0Eg;jfw}EnP))z4lKQQow3cV@f8A=6!% ze&s*MYgK<7tx^6+P2Whm3Vz}rg*sqw^WEGmM&msA^f5>O?eMj+#}-?=o{*0mSZybF zM4MWY4J)=%U7AHKJoo)(9#J(Nyo2PRp-?-xMjqA1mAR^VcXO>y9R2slai@8Mj*xjy zoweHYLOn4!dL*M8U@1U74O*HbpV8q%7=Z!~D2BLzWDFNrX^vJ|Y=Q9QV8Nh?;qkF< z)caXa1LI02o;?g_#V)b8v& zq9Qz@0Ev(XFM4r0lb?RsqHMm;MizvZE$SMmmk=Z^ub%*lPDy~5WV{K_nK#UFmNhTX z4rZigrz?jyTmPc#3`GmWr*lpHEG~C)mR7m z_V`!{t=WDdb)3uEdsX7lJ-+oR0Ged{IXN$_9iXC@krnUDQpXDHn0W96;83ICo1EtG z-Mo`C+uaz!wud&2)G^9%*^Tn~EWo@BhJf>6RviRbI|et0-RCNlS~H^xaWeQdYtpVZ@2r z;fm}q`+fnw+!W=+hN% zTIiPWoVYTU{m015J)EZF&!?*;*a8N!gqIr|AFxrS8wq2bnvQcgT^S!2Yfb6vvy(0C znjw5!ez|pJU1X6i`6$}VrG}5Uy=J1K@}mp{7c`U}6TvSdpsSVUknnu);$%M0st{|f z3e(;s{zfr00KATgvRL4i_ODtxQxk_y1p0NpInzyjXaI4R5Mv`OnZj(X5?-%Jk2L2I z@TP(n`MH(1=fyguqN6Ni=lTR@#;%@_781&w*CUrE`{xZ3>;i|s&i3K~9w5IO!XzT& zE#)4I=rKXB;r@<)2Q%q*2mo5KA}GFw5#p$8VvJ6(yGcDw7_YbrU{*M=~|QiW^v7jcqH6gR4^wPxJ=w)sQq!&36M@QeLlfaa{kcb4LEE8)Swz5 zojeKLFMuymbJ6YZ6rh9llqVGa* zr7KgdbbW6gd&Uoao$V4x`kHdvgPSb*c_U~Xp^68&8=c;-5)3)zAGI$K53|-8N0Ff5 zAVJJx-^b6co}N#M(NW;&_a63oXeB{jCOdb*YVS#_0fbEkAeZ61M1 zMrF~|G|`hw(TcdWwYl`#0=Y_{!O`CXFUuc!;5833@faBICFu(v=YQyn`4^X@#a^3ba&*VH z{^@&JDJZM0P4U|!)IOgF(TbL(Wk{^Bz+7!DE%ThHXP#=S%=FGq@+({pl@a)MdeiSZ z@&PIF=T9(ncBquR4B_)SE5T8>c6xg&(*MarE9v?8>D7OLp1X%4%40bL=G`mRN7TgvL%R5Sys7<@@3fvEJb(;*H6V08KCPmWlpD*bU`pwJo6l%$sUWoT%(Fwu?jd=fWopLkTWIH#+;`0SbE@I+j? zP9q$x5o2DzYdh4ZLKV);(3&WM4~G`n3*nT9&&yF4q$N22$usA3nfM^V{I}`I$=SKk zd0Gwau6_XhB7M{Uk!WJ!V>j^f1%OsEN_%L)uI?QB^!OB{!g8eL(41`%L>?HEk!nfT zCi&M*Dv&mQUc!g7va`oLV&XN5y+Y4m?X|6zz$KaPd^>sOhYh?I=ZUjc(1{x7DvhBS zfujNFh#;v$NrD6Opj*S@(Bnj)>qr>~kSd^lT-cTM^=`ln--9+v9@s9JDP7R271;I* zK7H~;ywVRT(EzWNre-mKXH`%cqoomiEaBs;8eRTdP)K1cjMahd4{?OtdT*Iovl zl#}E4ymnR=7omOn>e5f-GNj=bFRpE3=>NpkkkC+SZ@}H(|W8S;rqUKQ=IEJ;be zaALyAsfm3IM?o2jVe<2vdLSP_{pjjdi*qvm@^Ekqbm*QuV|kMzEe-E;R(!P7dCKoo z^cgZ{4O10za>P+tG_Lf?xudr)1-8_4mdS9`WlhVOPpOJpMHyV<_opZdsZ?0 zΜWW}ZWax07S)_=U72{YRe2Ev%AfJOehv(Hm`_R+%~^P6?u81o1cv`63d4M+^A> zk9_d@fI^>5(E`?Ratk1iF8${C8KA)R+}zw^>I|Y%0C`8niC3>)odww!5Mc9!3=9&G z2}i;7fu;o&N4bx9DwIR(1WF_okNO$G*+HDEC4i9L0NEGX>RISIrwfqw5{BLl8A`VJ3$Y{=4&vc}0H=2=t zMhw7P%%Mj@0OE?4{$2AR!P0<3ZJ?ae`+ydkm0s&;bB5C4fPc&S6$zZn_L>!y&cbyL)>kCTdST{&aYD&BKum-XvnNxdC4eMGi5JM0yQDK`8R)I1EtDXw_RoWBC|pBxz`Aon6m53X&zCi-qX$;?CSP zqhfm_e?U{E;EndG(4U4@6x*j-K z=hrQ>^B}1}1&Sz64%#q1yw=4(U;(|W6f|Kb0tEm=jq>b1C_;~EJN$}S&J>OmagW&% zb{MI+y#?eTTJV>-FXq6Hi<`zl#={nqSOe!M1+LeHGV}Lvd%S`8e}IDGq!IJ!(^I$1 zJhmWq$mF4f#=qwYbq)CXoS~b2buhsGG7HPKW5~O`)T5AGd$87}}Zu z|3RY$ICeHHC@2l9{VGx209Wr31Q}xfIQkfEo`I$rUC@`T!Fta^^9b6-Q_{*mI*B*S zfzll-i1*`AiB86wdSOA%-T}HAVXrmI15{Ktz*XgJioi}odtg4H2P45L7zT3tyZN9$ zF}^?e^j#S^NSd4TN*(A?5ujVM87g@w={)r^r+h&Y<#XW(q)F0m7_iL#?kcg(NOGst%U&T9GGe*dXmon_SCXTr=_v0s6G`{-U4>U2fh`(YeP#5->P_H zo-j~AU%!00eEG5>47dk~kOb{Q0Z>}4|7yPu$2MhvQIv+91(3p4)F@03*);K@Qq^4XrwbfJ+Dx;dF&c zbRlYLL&@PSn{9m+^8_s1}4{yN`g5pPevW=wD?~a2@D%+mT1h8_* zX?~G!y8Nx{wDbCwDNG-#(DQ``6V!U)tTB}00U2kc*UNGk{fh(I9o)GUccF3bZP&b~ zr>8EknLz4g1-iz>#OMO|igv(Dz5Bv4@_%%4=218OQ*hB}Z?6$TU$ zA(ChjCT*#YP@&YOKYD6>A!iSKq8>|_E{Z8ZRN?$yui_OjN)68ly=jBOU z#X2N!d5rY|^ozLGF7AvFT+(3Y;zMopBID+s*POH@Tjau}D^?ld{{}%4H1m%Vo|Bw9u-TaL!qy>R)4-NXq_!);EWMdOS{tOSH2+H2OTgRGx6rxb)EpI69I`gKEImxiUU+IMTdI=m(T zNT`XvB~Y~urH_3@nIjR7vrmUNdsFynHqH ziMHim2g8`)BcEWswSrPh97cL9s9U|i*o>Fe{$JB&x1cXejmZc8sYvWpsYFaBGx*Uo zXmeQ8Y4bLD^79S6?*xiB@AhxZ^(r{D<8c2(Iuux=5J4Qk%HAq+%&0t*Xgqx2ds{tO zuiMHA!xD^%M+e(?p!aLW_ z_nu%!Q)snUf$cvh(2#I(f$Z4G$PQDGT>Hp!aRV<<$zLP3c7>vOJ=q7u>5?~;MM!8Q zN1GBMqneo>Z_H8&f$JC&2j9uan2A-4>^r5XWrMmmCB-}4axzwzUVS7L=9TSOAq%22 z3#D*JiBA^KIWWh#vE_PjsefujziDW1Z&$A> zO8FX-oE$Mcq2drB_~X`YHSqr$F^E;?0SPiLbFd&{KYTFm-ADK#twR`;BXQZXu7}c$ zsTWOoV0WwbT%Q|<2y6cB4}Poyi7Wj6VrRWas}8t=W8CBB?0NPX#C<+MOL9yQO2(d5 zz6A*%q$0xN^vMHNrl&^?Q4p-fp@CusVjqof(wg=pKgNS=1`AaXmYkNBcD->g*NmwV z-GBvJI~VitY%s2Eok+(ap~jgPhpD_zgBH_{`D2GWMGgZi871KqU3mdVwF|g93a?)z zp$#z#Ns$m~2Okz(;v0+Bhmf`*Fb-(4ou{Hm>0&O-&fW{dlD62toc$cd?4fm^-0v{p z&($Dkt2GaFU42rhY+DQv45&|@T;k4V-=v?jb{>7Wc0d9=ie3N}*n!Z_aM8|e*I&?F zA0pI9pRp*QDJDhWDlB>!A2Cu5Oi&>Cy^G@gZqLRkofuhlD5Tzeq@*gO$={MPwIEV7 z{c<&{5m%Z15!}hfzdL}F5A<}h(xa|gHfDR}o8+6?E^UM98n7`^+cxM4(6_3^)b zxaRHwj29tU^mqF%%p8z-sh-oED;8SqD~Ecl&(AP=18~*YI*29EtQ8R#v%pOAiTD1f z?xoy=ls>q^E`=+yYGiER0J0p;ZF_@^Nal2;z)oy8v<)4|_TRC>t_84r`2VJrf90Dj71C}!Z zC+yf?Vql|8p>1H`B3^i^BuMw zp0prz8<>#^2L}h|J)rR>+EQei^3{%V z2=ezc2)q|?hqJIR6vHibjfa2~(_}B-RtVh>dT13$vtl1yQL~}-ByH6u*`yw4uNK3- zS2)0lh|c~n1N8)m{UJ!lIzjYmup^N{g!+tyV%#}3HHB_^&k%Ae*N_ZU9I^O6m>ec! zx67=rgxfxdtkTd|gGJAH1?Gpy|2*EhvNl&PFUieaiac}a0~q>jsX)6xe*W08g~;MQ zhF`$^MT<_c!9C2F$N?26e>uiRau^;C3jxr_*RVwSDZjIA!v3t#K2LnZ|$SwIfb5+Y-;&C9Hk>8Vt%+u>q7G%|6a zNgei@1_wq6f_yuE&E`T36^zX^|LbBl>1EIsU6argiM3jQ-(YrW0nP5?t<*a=pFO)0 z>Ym)b>NgopsI8xGXq=u@LgRCxu5Po_ro{5wvKu#U0CP)(k-@$Pv8IYLyoL(Y5Hy3t zCW4dZGP1#w!_onD*0ZZwotQzg?!{lehkF)8M<6`#R5pxj5gNATH@-nk-IC9CkiP+z zUrA@REn}t@jB((jv&W=(909dS!+Lap66zbZ+C$s4d+`~CS~)0u*1?|3F((271I4$P zf}mm6L^nzo97n4%>sX9j7F{?WN+A3NvPONXxLEWJ#C+#a66B`(fm0p5v%Y7kkWv%O zaN#9H?CPYwXGTM$H1f;S>~87E9m7zCk~TpsO3C}W{aLt0*s-NWB<$v4)sf}F&p6!=3)@nw9O(O06Y~n+b*L8hBti3=w(Z6w$Is`dJiW>cm#%f z)wPhuA~ulDAR@x|bJ8^6uqFw!k~gP!KUyxmOmxdKmikRRa)52atnVkZf5~E3M%@z5 k>tKgY{Qv0-J2EpT;`G2grPB|u;1Ue3&#vv7U+wwg9}hn*FaQ7m literal 0 HcmV?d00001 diff --git a/tests/PinnedH2O_3DOF/aux/get_ROM_table.py b/tests/PinnedH2O_3DOF/aux/get_ROM_table.py new file mode 100644 index 00000000..5fea1e38 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/get_ROM_table.py @@ -0,0 +1,31 @@ +import subprocess +import re + +bondlength_num_increments = (2, 5, 10) +bondangle_num_increments = (2, 5, 10) + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" + +print("\\begin{tabular}{|c|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$N_L$ & $N_\\theta$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +for _, N_L in enumerate(bondlength_num_increments): + for _, N_theta in enumerate(bondangle_num_increments): + snapshots = 2*(N_L+1)*(N_L+2)*(N_theta+1) + grep_command = f"grep 'take first' basis_{N_L}_{N_theta}_PinnedH2O_3DOF.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{N_L} & {N_theta} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py b/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py new file mode 100644 index 00000000..e679c691 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py @@ -0,0 +1,69 @@ +import matplotlib +import matplotlib.pyplot as plt +import numpy as np + +ref_bondlength = 1.83 +ref_bondangle = 104.5 + +# factors and increments for bond lengths and bond angle +bondlength_factor = np.linspace(0.95, 1.05, 3) +bondangle_increment = np.linspace(-5, 5, 3) + +fig, ax = plt.subplots() + +for d_bondangle in bondangle_increment: + bondangle = ref_bondangle + d_bondangle + x = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.sin(np.radians(bondangle / 2)) + for i, f_bondlength1 in enumerate(bondlength_factor): + H1_x = f_bondlength1*x + H1_y = f_bondlength1*y + ax.plot(H1_x, H1_y, 'ro', markersize=8, alpha=0.1) + for f_bondlength2 in bondlength_factor[:(i+1)]: + H2_x = f_bondlength2*x + H2_y = -f_bondlength2*y + ax.plot(H2_x, H2_y, 'bo', markersize=8, alpha=0.1) + +ref_bondangle = np.radians(ref_bondangle) +x = ref_bondlength * np.cos(ref_bondangle / 2) +y = ref_bondlength * np.sin(ref_bondangle / 2) + +ax.plot([0, x], [0, y], 'r-', lw=2) +ax.plot([0, x], [0, -y], 'b-', lw=2) +ax.plot(0, 0, 'ko', markersize=8) +ax.plot(x, y, 'ro', markersize=8) +ax.plot(x, -y, 'bo', markersize=8) + +arc1 = matplotlib.patches.Arc(xy=(0, 0), + width=0.5, + height=0.5, + angle=0, + theta1=0, + theta2=ref_bondangle/2, + color='red', + linewidth=2) +ax.add_patch(arc1) +ax.text(0.4, 0.25, r"$\frac{\theta}{2}$", ha='center', va='center', fontsize=12, color='red') + +arc2 = matplotlib.patches.Arc(xy=(0, 0), + width=0.5, + height=0.5, + angle=0, + theta1=-ref_bondangle/2, + theta2=0, + color='blue', + linewidth=2) +ax.add_patch(arc2) +ax.text(0.4, -0.25, r"$\frac{\theta}{2}$", ha='center', va='center', fontsize=12, color='blue') + +ax.text(x / 2 - 0.1, y / 2, r"$L_1$", ha='right', color='red') +ax.text(x / 2 - 0.1, -y / 2, r"$L_2$", ha='right', color='blue') + +ax.plot([-2, 2], [0, 0], 'k--', lw=1) +ax.set_xlim(-2.0, 2.0) +ax.set_ylim(-2.0, 2.0) +ax.set_aspect('equal') +ax.set_xlabel("x (Bohr)") +ax.set_ylabel("y (Bohr)") +ax.grid(True) +plt.savefig("PinnedH2O_3DOF.png") diff --git a/tests/PinnedH2O_3DOF/aux/rotation_test.cc b/tests/PinnedH2O_3DOF/aux/rotation_test.cc new file mode 100644 index 00000000..0e5641ee --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.cc @@ -0,0 +1,141 @@ +#include +#include +#include +#include + +using namespace std; + +double calculate_bondlength(const double atom1[3], const double atom2[3]) { + return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + pow(atom1[2] - atom2[2], 2)); +} + +double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3], bool radian) { + double vector1[3] = {atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2]}; + double vector2[3] = {atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2]}; + + double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + vector1[2] * vector2[2]; + double magnitude_product = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) * + sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); + double angle = acos(dot_product / magnitude_product); + + if (!radian) { + angle = angle * 180.0 / M_PI; + } + return angle; +} + +void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) { + double cos_theta = cos(angle); + double sin_theta = sin(angle); + double ux = axis[0], uy = axis[1], uz = axis[2]; + + matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); + matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; + matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; + + matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; + matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); + matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; + + matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; + matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; + matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); +} + +void normalize(double vec[3]) { + double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); + if (norm > 0) { + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; + } +} + +void cross(const double a[3], const double b[3], double result[3]) { + result[0] = a[1] * b[2] - a[2] * b[1]; + result[1] = a[2] * b[0] - a[0] * b[2]; + result[2] = a[0] * b[1] - a[1] * b[0]; +} + +void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) { + result[0] = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; + result[1] = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; + result[2] = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; +} + +void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) { + result[0] = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; + result[1] = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; + result[2] = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; +} + +int main() { + double O1[3] = {0.00, 0.00, 0.00}; + double H1[3] = {-0.45, -1.48, -0.97}; + double H2[3] = {-0.45, 1.42, -1.07}; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + + double target_plane_normal[3] = {0, 0, 1}; + double axis_to_align[3]; + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align = acos(min(max(dot_product, -1.0), 1.0)); + + double rot_matrix_align_plane[3][3]; + rotation_matrix(axis_to_align, angle_to_align, rot_matrix_align_plane); + + double bondlength1 = calculate_bondlength(H1, O1); + double bondlength2 = calculate_bondlength(H2, O1); + double bondangle = calculate_bondangle(H1, O1, H2, false); + + cout << "Original system" << endl; + cout << "H1 = (" << H1[0] << ", " << H1[1] << ", " << H1[2] << ")" << endl; + cout << "H2 = (" << H2[0] << ", " << H2[1] << ", " << H2[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1 << endl; + cout << "Bondlength of O1-H2 = " << bondlength2 << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + + double H1_rotated[3], H2_rotated[3]; + apply_rotation(rot_matrix_align_plane, H1, H1_rotated); + apply_rotation(rot_matrix_align_plane, H2, H2_rotated); + bool flipped_bond = false; + if (bondlength1 < bondlength2) { + flipped_bond = true; + swap(H1_rotated, H2_rotated); + } + bondlength1 = calculate_bondlength(H1_rotated, O1); + bondlength2 = calculate_bondlength(H2_rotated, O1); + bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, false); + + cout << "Reference system (z=0 plane about x=0 axis, with longer bondlength in H1)" << endl; + cout << "H1 = (" << H1_rotated[0] << ", " << H1_rotated[1] << ", " << H1_rotated[2] << ")" << endl; + cout << "H2 = (" << H2_rotated[0] << ", " << H2_rotated[1] << ", " << H2_rotated[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1 << endl; + cout << "Bondlength of O1-H2 = " << bondlength2 << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + + if (flipped_bond) { + swap(H1_rotated, H2_rotated); + } + double H1_restored[3], H2_restored[3]; + apply_transpose_rotation(rot_matrix_align_plane, H1_rotated, H1_restored); + apply_transpose_rotation(rot_matrix_align_plane, H2_rotated, H2_restored); + bondlength1 = calculate_bondlength(H1_restored, O1); + bondlength2 = calculate_bondlength(H2_restored, O1); + bondangle = calculate_bondangle(H1_restored, O1, H2_restored, false); + + cout << "Restored system" << endl; + cout << "H1 = (" << H1_restored[0] << ", " << H1_restored[1] << ", " << H1_restored[2] << ")" << endl; + cout << "H2 = (" << H2_restored[0] << ", " << H2_restored[1] << ", " << H2_restored[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1 << endl; + cout << "Bondlength of O1-H2 = " << bondlength2 << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + + return 0; +} diff --git a/examples/PinnedH2O/rotation_test.py b/tests/PinnedH2O_3DOF/aux/rotation_test.py similarity index 73% rename from examples/PinnedH2O/rotation_test.py rename to tests/PinnedH2O_3DOF/aux/rotation_test.py index 79f1aaef..fbecb532 100644 --- a/examples/PinnedH2O/rotation_test.py +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.py @@ -1,8 +1,8 @@ import numpy as np O1 = np.array([0.00, 0.00, 0.00]) -H1 = np.array([-0.45, 1.42, -1.07]) -H2 = np.array([-0.45, -1.48, -0.97]) +H1 = np.array([-0.45, -1.48, -0.97]) +H2 = np.array([-0.45, 1.42, -1.07]) def calculate_bondlength(atom1, atom2): return np.linalg.norm(atom1 - atom2) @@ -34,10 +34,7 @@ def rotation_matrix(axis, angle): axis_to_align = np.cross(plane_normal, target_plane_normal) axis_to_align /= np.linalg.norm(axis_to_align) angle_to_align = np.arccos(np.clip(np.dot(plane_normal, target_plane_normal), -1.0, 1.0)) - rot_matrix_align_plane = rotation_matrix(axis_to_align, angle_to_align) -H1_rotated = np.dot(rot_matrix_align_plane, H1) -H2_rotated = np.dot(rot_matrix_align_plane, H2) bondlength1 = calculate_bondlength(H1, O1) bondlength2 = calculate_bondlength(H2, O1) @@ -50,15 +47,34 @@ def rotation_matrix(axis, angle): print(f'Bondlength of O1-H2 = {bondlength2}') print(f'Angle between O1-H1 and O1-H2 = {bondangle}') +H1_rotated = np.dot(rot_matrix_align_plane, H1) +H2_rotated = np.dot(rot_matrix_align_plane, H2) +fliped_bond = False +if bondlength1 < bondlength2: + fliped_bond = True + H1_rotated, H2_rotated = H2_rotated, H1_rotated bondlength1 = calculate_bondlength(H1_rotated, O1) bondlength2 = calculate_bondlength(H2_rotated, O1) bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) -if bondlength1 < bondlength2: - H1_rotated, H2_rotated, bondlength1, bondlength2 = H2_rotated, H1_rotated, bondlength2, bondlength1 -print('Rotated system in z=0 plane about x=0 axis, with longer bondlength in H1') +print('Reference system (z=0 plane about x=0 axis, with longer bondlength in H1)') print(f'H1 = {H1_rotated}') print(f'H2 = {H2_rotated}') print(f'Bondlength of O1-H1 = {bondlength1}') print(f'Bondlength of O1-H2 = {bondlength2}') print(f'Angle between O1-H1 and O1-H2 = {bondangle}') + +if fliped_bond: + H1_rotated, H2_rotated = H2_rotated, H1_rotated +H1_restored = np.dot(rot_matrix_align_plane.T, H1_rotated) +H2_restored = np.dot(rot_matrix_align_plane.T, H2_rotated) +bondlength1 = calculate_bondlength(H1_restored, O1) +bondlength2 = calculate_bondlength(H2_restored, O1) +bondangle = calculate_bondangle(H1_restored, O1, H2_restored, False) + +print('Restored system') +print(f'H1 = {H1_restored}') +print(f'H2 = {H2_restored}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') diff --git a/tests/PinnedH2O_3DOF/compare_energy_and_forces.py b/tests/PinnedH2O_3DOF/compare_energy_and_forces.py new file mode 100644 index 00000000..b11dbe91 --- /dev/null +++ b/tests/PinnedH2O_3DOF/compare_energy_and_forces.py @@ -0,0 +1,233 @@ +import numpy as np +import os +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import argparse + +parser = argparse.ArgumentParser(description="Calculate and plot differences in energies and forces.") + +parser.add_argument("--N_l", type=int, default=2, help="Sampling frequency of bond length points") +parser.add_argument("--N_theta", type=int, default=2, help="Sampling frequency of bond angle points") +parser.add_argument("--rdim", type=int, default=18, help="Dimension of the ROM basis") + +args = parser.parse_args() + +N_l = args.N_l +N_theta = args.N_theta +rdim = args.rdim + +bondlength_min = 0.95 +bondlength_max = 1.05 +bondlength_num_increments = 10 + +bondangle_min = -5.0 +bondangle_max = 5.0 +bondangle_num_increments = 10 + +output_dir = f'results_{N_l}_{N_theta}_{rdim}' + +def process_data(s_l1, s_l2, s_theta, N_l, N_theta, rdim): + try: + data_dir = f'data/{s_l1}_{s_l2}_{s_theta}' + offline_file = os.path.join(data_dir, 'offline_PinnedH2O.out') + log_file = f'{output_dir}/{s_l1}_{s_l2}_{s_theta}.log' + + f_O1_fom = None + f_H1_fom = None + f_H2_fom = None + + if not os.path.exists(offline_file): + print(f"Error: File not found: {offline_file}") + return None + + with open(offline_file, 'r') as f: + eks_lines = [line for line in f if " SC ENERGY [Ha]" in line] + if eks_lines: + Eks_fom = float(eks_lines[-1].split()[-1]) + else: + print(f"Error: Could not find 'SC ENERGY [Ha]' line in {offline_file}") + return None + + with open(offline_file, 'r') as f: + for line in f: + if "O1" in line: + f_O1_fom = np.array(line.split()[6:9], dtype=float) + elif "H1" in line: + f_H1_fom = np.array(line.split()[5:8], dtype=float) + elif "H2" in line: + f_H2_fom = np.array(line.split()[5:8], dtype=float) + + if f_O1_fom is not None and f_H1_fom is not None and f_H2_fom is not None: + break # Exit the loop once we found all lines + + if f_O1_fom is None or f_H1_fom is None or f_H2_fom is None: + print(f"Error: Could not find O1, H1, or H2 lines in {offline_file}") + return None + + f_H2_rom = None + f_O1_rom = None + f_H1_rom = None + + if not os.path.exists(log_file): + print(f"Error: File not found: {log_file}") + return None + + with open(log_file, 'r') as f: + lines = f.readlines() + + eks_index = -1 + for i in range(len(lines) - 1, -1, -1): # Search from the end + if "Eks:" in lines[i]: + eks_index = i + break + + if eks_index == -1: + print(f"Error: Could not find 'Eks:' line in {log_file}") + return None + + Eks_rom = float(lines[eks_index].split()[-1]) # Get the last element + + force_index = -1 + for i in range(len(lines) - 1, -1, -1): # Search from the end + if "Forces:" in lines[i]: + force_index = i + break + + if force_index == -1: + print(f"Error: Could not find 'Forces:' line in {log_file}") + return None + + if force_index + 3 >= len(lines): + print(f"Error: Not enough lines after 'Forces:' in {log_file}") + return None + + f_H2_rom = np.array(lines[force_index + 1].split(), dtype=float) + f_O1_rom = np.array(lines[force_index + 2].split(), dtype=float) + f_H1_rom = np.array(lines[force_index + 3].split(), dtype=float) + + return Eks_fom, f_O1_fom, f_H1_fom, f_H2_fom, Eks_rom, f_O1_rom, f_H1_rom, f_H2_rom + + except Exception as e: + print(f"An error occurred: {e}") + return None + +Eks_diff_reproductive = [] +f_O1_diff_reproductive = [] +f_H1_diff_reproductive = [] +f_H2_diff_reproductive = [] + +Eks_diff_predictive = [] +f_O1_diff_predictive = [] +f_H1_diff_predictive = [] +f_H2_diff_predictive = [] + +for i in range(bondlength_num_increments + 1): + bondlength_one = round(bondlength_min + i * (bondlength_max - bondlength_min) / bondlength_num_increments, 2) + for j in range(i + 1): + bondlength_two = round(bondlength_min + j * (bondlength_max - bondlength_min) / bondlength_num_increments, 2) + for k in range(bondangle_num_increments + 1): + bondangle = round(bondangle_min + k * (bondangle_max - bondangle_min) / bondangle_num_increments, 1) + + s_l1 = f"{bondlength_one:.2f}" + s_l2 = f"{bondlength_two:.2f}" + s_theta = f"{bondangle:.1f}" + tag = f'{s_l1}_{s_l2}_{s_theta}' + + output_filename = f'{output_dir}/energy_and_forces_{tag}.out' + + with open(output_filename, 'w') as outfile: + print("s_l1:", s_l1, file=outfile) + print("s_l2:", s_l2, file=outfile) + print("s_theta:", s_theta, file=outfile) + print("N_l:", N_l, file=outfile) + print("N_theta:", N_theta, file=outfile) + print("rdim:", rdim, file=outfile) + + results = process_data(s_l1, s_l2, s_theta, N_l, N_theta, rdim) + + if results: + Eks_fom, f_O1_fom, f_H1_fom, f_H2_fom, Eks_rom, f_O1_rom, f_H1_rom, f_H2_rom = results + + print("Eks_fom:", Eks_fom, file=outfile) + print("Eks_rom:", Eks_rom, file=outfile) + + print("f_O1_fom:", f_O1_fom, file=outfile) + print("f_O1_rom:", f_O1_rom, file=outfile) + + print("f_H1_fom:", f_H1_fom, file=outfile) + print("f_H1_rom:", f_H1_rom, file=outfile) + + print("f_H2_fom:", f_H2_fom, file=outfile) + print("f_H2_rom:", f_H2_rom, file=outfile) + + def calculate_differences(fom, rom, name): + abs_diff = np.linalg.norm(fom - rom) + rel_diff = abs_diff / np.linalg.norm(fom) if np.linalg.norm(fom)!= 0 else float('inf') + print(f"Absolute difference in {name}:", abs_diff, file=outfile) + print(f"Relative difference in {name}:", rel_diff, file=outfile) + return abs_diff + + Eks_diff = calculate_differences(Eks_fom, Eks_rom, "Eks") + f_O1_diff = calculate_differences(f_O1_fom, f_O1_rom, "f_O1") + f_H1_diff = calculate_differences(f_H1_fom, f_H1_rom, "f_H1") + f_H2_diff = calculate_differences(f_H2_fom, f_H2_rom, "f_H2") + + if i * N_l % bondlength_num_increments == 0 and j * N_l % bondlength_num_increments == 0 and k * N_theta % bondangle_num_increments == 0: + Eks_diff_reproductive.append(Eks_diff) + f_O1_diff_reproductive.append(f_O1_diff) + f_H1_diff_reproductive.append(f_H1_diff) + f_H2_diff_reproductive.append(f_H2_diff) + else: + Eks_diff_predictive.append(Eks_diff) + f_O1_diff_predictive.append(f_O1_diff) + f_H1_diff_predictive.append(f_H1_diff) + f_H2_diff_predictive.append(f_H2_diff) + else: + print("Error occurred during data processing. Differences cannot be calculated.", file=outfile) + +def plot_histogram(data, quantity, test_case): + plt.figure(figsize=(8, 6)) + plt.hist(data, bins=20, color='skyblue', edgecolor='black') + + if quantity == "Eks": + quantity_name = "absolute difference in total energy" + elif quantity.startswith("f_"): + quantity_name = f"magnitude of difference in force on {quantity[2:]}" + else: + raise ValueError("Invalid input quantity") + + plt.title(f'Histogram of {quantity_name}') + plt.xlabel('Difference') + plt.ylabel('Frequency') + + min_val, max_val = np.min(data), np.max(data) + plt.xlim(min_val, max_val) + num_ticks = 8 + xticks = np.linspace(min_val, max_val, num_ticks) + plt.xticks(xticks) + formatter = ticker.ScalarFormatter(useMathText=True) + formatter.set_scientific(True) + formatter.set_powerlimits((0, 0)) + plt.gca().xaxis.set_major_formatter(formatter) + + total_count = len(data) + mean_val = np.mean(data) + max_val = np.max(data) + stats_text = (f"Total {test_case} cases: {total_count}\n" + f"Mean: {mean_val:.3e}") + plt.text(0.95, 0.95, stats_text, transform=plt.gca().transAxes, + fontsize=12, verticalalignment='top', horizontalalignment='right', + bbox=dict(facecolor='white', alpha=0.7, edgecolor='black')) + + plt.tight_layout() + plt.savefig(f"{output_dir}/{quantity}_difference_histogram_{test_case}.png") + +plot_histogram(Eks_diff_reproductive, "Eks", "reproductive") +plot_histogram(f_O1_diff_reproductive, "f_O1", "reproductive") +plot_histogram(f_H1_diff_reproductive, "f_H1", "reproductive") +plot_histogram(f_H2_diff_reproductive, "f_H2", "reproductive") + +plot_histogram(Eks_diff_predictive, "Eks", "predictive") +plot_histogram(f_O1_diff_predictive, "f_O1", "predictive") +plot_histogram(f_H1_diff_predictive, "f_H1", "predictive") +plot_histogram(f_H2_diff_predictive, "f_H2", "predictive") diff --git a/tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in b/tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in new file mode 100644 index 00000000..46f5681d --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 1.12 1.45 0.00 1 +H2 2 1.12 -1.45 0.00 1 diff --git a/tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in b/tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in new file mode 100644 index 00000000..4c440443 --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 1.12 1.50 0.00 1 +H2 2 1.07 -1.44 0.00 1 diff --git a/examples/PinnedH2O/generate_coord_simple.py b/tests/PinnedH2O_3DOF/generate_coord.py similarity index 96% rename from examples/PinnedH2O/generate_coord_simple.py rename to tests/PinnedH2O_3DOF/generate_coord.py index 7e799bdd..194c2df7 100644 --- a/examples/PinnedH2O/generate_coord_simple.py +++ b/tests/PinnedH2O_3DOF/generate_coord.py @@ -11,7 +11,7 @@ bondangle_increment = np.linspace(-5, 5, 11) # output directory -output_dir = "PinnedH2O_3dof_coords" +output_dir = "data" # generation os.makedirs(output_dir, exist_ok=True) diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis new file mode 100644 index 00000000..ccc03c9d --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.basis @@ -0,0 +1,47 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set snapshot_files = "" +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 5 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 5 + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" + end + end +end +echo "Snapshot files: $snapshot_files" + +set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${bondlength_num_increments}_${bondangle_num_increments}" +srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${bondlength_num_increments}_${bondangle_num_increments}_PinnedH2O_3DOF.out + +date diff --git a/tests/PinnedH2O_3DOF/job.basis_test1 b/tests/PinnedH2O_3DOF/job.basis_test1 new file mode 100644 index 00000000..3955cdc6 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.basis_test1 @@ -0,0 +1,34 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set basis_tag = "test1" +set snapshot_files = "" + +set bondlength_one = 1.00 +set bondlength_two = 1.00 +set bondangle = 0.0 +echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" +set tag = ${bondlength_one}_${bondlength_two}_${bondangle} +set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" + +echo "Snapshot files: $snapshot_files" + +set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${basis_tag}" +srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${basis_tag}_PinnedH2O_3DOF.out + +date diff --git a/tests/PinnedH2O_3DOF/job.basis_test2 b/tests/PinnedH2O_3DOF/job.basis_test2 new file mode 100644 index 00000000..d0ac6301 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.basis_test2 @@ -0,0 +1,39 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set basis_tag = "test2" +set snapshot_files = "" + +set bondlength_one = 1.00 +set bondlength_two = 1.00 +set bondangle = 0.0 +echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" +set tag = ${bondlength_one}_${bondlength_two}_${bondangle} +set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" + +set bondangle = 1.0 +echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" +set tag = ${bondlength_one}_${bondlength_two}_${bondangle} +set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" + +echo "Snapshot files: $snapshot_files" + +set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${basis_tag}" +srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${basis_tag}_PinnedH2O_3DOF.out + +date diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline new file mode 100644 index 00000000..7fc51dd7 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.offline @@ -0,0 +1,61 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 4:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +rm $exe +cp $maindir/install_quartz/bin/$exe . + +set cfg_offline = mgmol_offline.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 10 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 10 + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + if (-e data/${tag}) then + continue + endif + cp $cfg_offline ${tag}_${cfg_offline} + sed -i "s/CUSTOM_TAG/${tag}/g" ${tag}_${cfg_offline} + srun -n $ncpus $exe -c ${tag}_${cfg_offline} -i data/coords_${tag}.in > ${tag}_offline_PinnedH2O.out + mv data/coords_${tag}.in ${tag}/coords.in + mv ${tag}_${cfg_offline} ${tag}/${cfg_offline} + mv ${tag}_offline_PinnedH2O.out ${tag}/offline_PinnedH2O.out + mv -f ${tag} data + end + end +end + +date diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online new file mode 100644 index 00000000..e703fc29 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.online @@ -0,0 +1,70 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testPinnedH2O_3DOF + +rm $exe +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol_online.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +set sample_bondlength_num_increments = 5 +set sample_bondangle_num_increments = 5 +set num_orbital_basis = 36 + +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 10 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 10 + +set case = ${sample_bondlength_num_increments}_${sample_bondangle_num_increments}_${num_orbital_basis} +mkdir results_${case} +cp ${cfg_online} ${case}_${cfg_online} + +@ num_empty_orbital = $num_orbital_basis - 4 +sed -i "s/CUSTOM_NE/${num_empty_orbital}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NL/${sample_bondlength_num_increments}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NT/${sample_bondangle_num_increments}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NB/${num_orbital_basis}/g" ${case}_${cfg_online} + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + + echo $ncpus $exe $case $cfg_online $tag + srun -p pdebug -n $ncpus $exe -c ${case}_${cfg_online} -i data/${tag}/coords.in >& results_${case}/${tag}.log + end + end +end + +mv ${case}_${cfg_online} results_${case} + +date diff --git a/tests/PinnedH2O_3DOF/job.online_test1 b/tests/PinnedH2O_3DOF/job.online_test1 new file mode 100644 index 00000000..bf69a1ef --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.online_test1 @@ -0,0 +1,30 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testPinnedH2O_3DOF + +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol_online_test1.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -p pdebug -n $ncpus $exe -c $cfg_online -i coords_1.00_1.00_0.0.in + +date diff --git a/tests/PinnedH2O_3DOF/job.online_test2 b/tests/PinnedH2O_3DOF/job.online_test2 new file mode 100644 index 00000000..88665f1e --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.online_test2 @@ -0,0 +1,30 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 64 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testPinnedH2O_3DOF + +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol_online_test2.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -p pdebug -n $ncpus $exe -c $cfg_online -i coords_1.00_1.00_0.0.in + +date diff --git a/tests/PinnedH2O_3DOF/mgmol_offline.cfg b/tests/PinnedH2O_3DOF/mgmol_offline.cfg new file mode 100644 index 00000000..7b2dbcfa --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_offline.cfg @@ -0,0 +1,30 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +output_filename=CUSTOM_TAG +[ROM.offline] +save_librom_snapshot=true diff --git a/tests/PinnedH2O_3DOF/mgmol_online.cfg b/tests/PinnedH2O_3DOF/mgmol_online.cfg new file mode 100644 index 00000000..5bc5b8b8 --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_online.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=CUSTOM_NE +[Restart] +output_level=4 +[DensityMatrix] +solver=MVP +nb_inner_it=100 +[ROM.offline] +basis_file=data/PinnedH2O_3DOF_orbitals_basis_CUSTOM_NL_CUSTOM_NT +[ROM.basis] +compare_md=false +number_of_orbital_basis=CUSTOM_NB diff --git a/tests/PinnedH2O_3DOF/mgmol_online_test1.cfg b/tests/PinnedH2O_3DOF/mgmol_online_test1.cfg new file mode 100644 index 00000000..e032350d --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_online_test1.cfg @@ -0,0 +1,32 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +basis_file=data/PinnedH2O_3DOF_orbitals_basis_test1 +[ROM.basis] +compare_md=false +number_of_orbital_basis=4 diff --git a/tests/PinnedH2O_3DOF/mgmol_online_test2.cfg b/tests/PinnedH2O_3DOF/mgmol_online_test2.cfg new file mode 100644 index 00000000..8ce3e32d --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_online_test2.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=1000 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=4 +[Restart] +output_level=4 +[DensityMatrix] +solver=MVP +nb_inner_it=20 +[ROM.offline] +basis_file=data/PinnedH2O_3DOF_orbitals_basis_test2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=8 diff --git a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc new file mode 100644 index 00000000..e7813777 --- /dev/null +++ b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc @@ -0,0 +1,203 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#ifdef MGMOL_HAS_LIBROM +#include "librom.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + // compute energy and forces again with projected problem onto ROM subspace + const int rdim = ct.getROMOptions().num_orbbasis; + if (rdim != ct.numst) + { + std::cerr << "The number of functions in the ROM basis file, " + << rdim << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + orbitals.set(ct.getROMOptions().basis_file, ct.numst); + orbitals.orthonormalizeLoewdin(); + orbitals.setDataWithGhosts(true); + + // set the iterative index to 1 to differentiate it from first instance + // in MGmol initial() function. This is not very clean and could be + // better designed, but works for now + orbitals.setIterativeIndex(10); + + // set initial DM with uniform occupations + projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->printDM(std::cout); + + // + // evaluate energy and forces with ROM bases just read + // + std::vector forces; + double eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Forces:" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} +#endif // MGMOL_HAS_LIBROM diff --git a/tests/RestartEnergyAndForces/job.test b/tests/RestartEnergyAndForces/job.test new file mode 100644 index 00000000..89b4865f --- /dev/null +++ b/tests/RestartEnergyAndForces/job.test @@ -0,0 +1,28 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testRestartEnergyAndForces + +cp $maindir/build_quartz/tests/$exe . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +python3 test.py srun -n $ncpus ${maindir}/install_quartz/bin/mgmol-opt ${exe} mgmol.cfg restart.cfg h2o.xyz . + +date diff --git a/tests/WFEnergyAndForces/job.test b/tests/WFEnergyAndForces/job.test new file mode 100644 index 00000000..7a9e49ee --- /dev/null +++ b/tests/WFEnergyAndForces/job.test @@ -0,0 +1,27 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testWFEnergyAndForces + +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol.cfg + +source $maindir/scripts/modules.quartz + +python3 test.py srun -p pdebug -n $ncpus $exe $cfg_online sih4.xyz $maindir/potentials + +date From 930d31b121ee261e70138e33a012172b1ca4a193 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 11 Feb 2025 20:13:46 -0800 Subject: [PATCH 37/50] ROM on PinnedH2O with geometric transformation (#304) * Minor change * Fix calculations * Rename * Add rotation * Clean up --- examples/PinnedH2O/mgmol.cfg | 2 +- examples/PinnedH2O/mgmol_offline.cfg | 2 +- .../{ => aux}/coords_1.00_1.00_0.0.in | 0 .../{ => aux}/coords_1.02_0.98_2.0.in | 0 tests/PinnedH2O_3DOF/aux/rotation_test.cc | 126 +++++---- tests/PinnedH2O_3DOF/aux/rotation_test.py | 85 +++--- tests/PinnedH2O_3DOF/coords.in_rotate1 | 3 + tests/PinnedH2O_3DOF/coords.in_rotate2 | 3 + tests/PinnedH2O_3DOF/job.basis | 2 +- tests/PinnedH2O_3DOF/job.basis_test1 | 34 --- tests/PinnedH2O_3DOF/job.basis_test2 | 39 --- tests/PinnedH2O_3DOF/job.offline | 2 +- tests/PinnedH2O_3DOF/job.online | 2 +- .../{job.online_test1 => job.online_rotate} | 22 +- tests/PinnedH2O_3DOF/job.online_test2 | 30 -- ...line_test2.cfg => mgmol_online_rotate.cfg} | 10 +- ..._online_test1.cfg => mgmol_ref_rotate.cfg} | 7 +- tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc | 264 ++++++++++++++++++ 18 files changed, 422 insertions(+), 211 deletions(-) rename tests/PinnedH2O_3DOF/{ => aux}/coords_1.00_1.00_0.0.in (100%) rename tests/PinnedH2O_3DOF/{ => aux}/coords_1.02_0.98_2.0.in (100%) create mode 100644 tests/PinnedH2O_3DOF/coords.in_rotate1 create mode 100644 tests/PinnedH2O_3DOF/coords.in_rotate2 delete mode 100644 tests/PinnedH2O_3DOF/job.basis_test1 delete mode 100644 tests/PinnedH2O_3DOF/job.basis_test2 rename tests/PinnedH2O_3DOF/{job.online_test1 => job.online_rotate} (55%) delete mode 100644 tests/PinnedH2O_3DOF/job.online_test2 rename tests/PinnedH2O_3DOF/{mgmol_online_test2.cfg => mgmol_online_rotate.cfg} (76%) rename tests/PinnedH2O_3DOF/{mgmol_online_test1.cfg => mgmol_ref_rotate.cfg} (70%) diff --git a/examples/PinnedH2O/mgmol.cfg b/examples/PinnedH2O/mgmol.cfg index 32325e47..ef82f07d 100644 --- a/examples/PinnedH2O/mgmol.cfg +++ b/examples/PinnedH2O/mgmol.cfg @@ -1,6 +1,6 @@ verbosity=1 xcFunctional=PBE -FDtype=Mehrstellen +FDtype=4th [Mesh] nx=64 ny=64 diff --git a/examples/PinnedH2O/mgmol_offline.cfg b/examples/PinnedH2O/mgmol_offline.cfg index 332dd77d..f535cff2 100644 --- a/examples/PinnedH2O/mgmol_offline.cfg +++ b/examples/PinnedH2O/mgmol_offline.cfg @@ -1,6 +1,6 @@ verbosity=1 xcFunctional=PBE -FDtype=Mehrstellen +FDtype=4th [Mesh] nx=64 ny=64 diff --git a/tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in b/tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in similarity index 100% rename from tests/PinnedH2O_3DOF/coords_1.00_1.00_0.0.in rename to tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in diff --git a/tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in b/tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in similarity index 100% rename from tests/PinnedH2O_3DOF/coords_1.02_0.98_2.0.in rename to tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in diff --git a/tests/PinnedH2O_3DOF/aux/rotation_test.cc b/tests/PinnedH2O_3DOF/aux/rotation_test.cc index 0e5641ee..21cfd93a 100644 --- a/tests/PinnedH2O_3DOF/aux/rotation_test.cc +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.cc @@ -5,11 +5,13 @@ using namespace std; -double calculate_bondlength(const double atom1[3], const double atom2[3]) { +double calculate_bondlength(const double atom1[3], const double atom2[3]) +{ return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + pow(atom1[2] - atom2[2], 2)); } -double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3], bool radian) { +double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3], bool radian) +{ double vector1[3] = {atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2]}; double vector2[3] = {atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2]}; @@ -24,7 +26,8 @@ double calculate_bondangle(const double atom1[3], const double atom2[3], const d return angle; } -void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) { +void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) +{ double cos_theta = cos(angle); double sin_theta = sin(angle); double ux = axis[0], uy = axis[1], uz = axis[2]; @@ -42,54 +45,41 @@ void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) { matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); } -void normalize(double vec[3]) { +void normalize(double vec[3]) +{ double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); - if (norm > 0) { - vec[0] /= norm; - vec[1] /= norm; - vec[2] /= norm; - } + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; } -void cross(const double a[3], const double b[3], double result[3]) { +void cross(const double a[3], const double b[3], double result[3]) +{ result[0] = a[1] * b[2] - a[2] * b[1]; result[1] = a[2] * b[0] - a[0] * b[2]; result[2] = a[0] * b[1] - a[1] * b[0]; } -void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) { +void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) +{ result[0] = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; result[1] = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; result[2] = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; } -void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) { +void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) +{ result[0] = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; result[1] = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; result[2] = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; } -int main() { +int main() +{ double O1[3] = {0.00, 0.00, 0.00}; double H1[3] = {-0.45, -1.48, -0.97}; double H2[3] = {-0.45, 1.42, -1.07}; - double plane_normal[3]; - cross(H2, H1, plane_normal); - normalize(plane_normal); - - double target_plane_normal[3] = {0, 0, 1}; - double axis_to_align[3]; - cross(plane_normal, target_plane_normal, axis_to_align); - normalize(axis_to_align); - double dot_product = plane_normal[0] * target_plane_normal[0] + - plane_normal[1] * target_plane_normal[1] + - plane_normal[2] * target_plane_normal[2]; - double angle_to_align = acos(min(max(dot_product, -1.0), 1.0)); - - double rot_matrix_align_plane[3][3]; - rotation_matrix(axis_to_align, angle_to_align, rot_matrix_align_plane); - double bondlength1 = calculate_bondlength(H1, O1); double bondlength2 = calculate_bondlength(H2, O1); double bondangle = calculate_bondangle(H1, O1, H2, false); @@ -101,41 +91,75 @@ int main() { cout << "Bondlength of O1-H2 = " << bondlength2 << endl; cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + double H1_temp[3], H2_temp[3]; double H1_rotated[3], H2_rotated[3]; - apply_rotation(rot_matrix_align_plane, H1, H1_rotated); - apply_rotation(rot_matrix_align_plane, H2, H2_rotated); - bool flipped_bond = false; - if (bondlength1 < bondlength2) { - flipped_bond = true; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + double target_plane_normal[3] = {0, 0, 1}; + double axis_to_align[3]; + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align = acos(min(max(dot_product, -1.0), 1.0)); + double out_of_plane_rotation_matrix[3][3]; + rotation_matrix(axis_to_align, angle_to_align, out_of_plane_rotation_matrix); + apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); + apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); + + double theta1 = atan2(H1_temp[1], H1_temp[0]); + double planar_rotation_angle = -theta1 + bondangle / 360.0 * M_PI; + double planar_rotation_matrix[3][3]; + rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); + apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); + + if (bondlength1 < bondlength2) + { swap(H1_rotated, H2_rotated); + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; } - bondlength1 = calculate_bondlength(H1_rotated, O1); - bondlength2 = calculate_bondlength(H2_rotated, O1); - bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, false); - cout << "Reference system (z=0 plane about x=0 axis, with longer bondlength in H1)" << endl; + double bondlength1_rotated = calculate_bondlength(H1_rotated, O1); + double bondlength2_rotated = calculate_bondlength(H2_rotated, O1); + double bondangle_rotated = calculate_bondangle(H1_rotated, O1, H2_rotated, false); + + cout << "Reference system (in z=0 plane, symmetric about x=0 axis, with longer bondlength in Q1)" << endl; cout << "H1 = (" << H1_rotated[0] << ", " << H1_rotated[1] << ", " << H1_rotated[2] << ")" << endl; cout << "H2 = (" << H2_rotated[0] << ", " << H2_rotated[1] << ", " << H2_rotated[2] << ")" << endl; - cout << "Bondlength of O1-H1 = " << bondlength1 << endl; - cout << "Bondlength of O1-H2 = " << bondlength2 << endl; - cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + cout << "Bondlength of O1-H1 = " << bondlength1_rotated << endl; + cout << "Bondlength of O1-H2 = " << bondlength2_rotated << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle_rotated << endl; + + double H1_restored[3], H2_restored[3]; - if (flipped_bond) { + if (bondlength1 < bondlength2) + { swap(H1_rotated, H2_rotated); + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; } - double H1_restored[3], H2_restored[3]; - apply_transpose_rotation(rot_matrix_align_plane, H1_rotated, H1_restored); - apply_transpose_rotation(rot_matrix_align_plane, H2_rotated, H2_restored); - bondlength1 = calculate_bondlength(H1_restored, O1); - bondlength2 = calculate_bondlength(H2_restored, O1); - bondangle = calculate_bondangle(H1_restored, O1, H2_restored, false); + + apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); + apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); + + apply_transpose_rotation(out_of_plane_rotation_matrix, H1_temp, H1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, H2_temp, H2_restored); + + double bondlength1_restored = calculate_bondlength(H1_restored, O1); + double bondlength2_restored = calculate_bondlength(H2_restored, O1); + double bondangle_restored = calculate_bondangle(H1_restored, O1, H2_restored, false); cout << "Restored system" << endl; cout << "H1 = (" << H1_restored[0] << ", " << H1_restored[1] << ", " << H1_restored[2] << ")" << endl; cout << "H2 = (" << H2_restored[0] << ", " << H2_restored[1] << ", " << H2_restored[2] << ")" << endl; - cout << "Bondlength of O1-H1 = " << bondlength1 << endl; - cout << "Bondlength of O1-H2 = " << bondlength2 << endl; - cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + cout << "Bondlength of O1-H1 = " << bondlength1_restored << endl; + cout << "Bondlength of O1-H2 = " << bondlength2_restored << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle_restored << endl; return 0; } diff --git a/tests/PinnedH2O_3DOF/aux/rotation_test.py b/tests/PinnedH2O_3DOF/aux/rotation_test.py index fbecb532..41e20296 100644 --- a/tests/PinnedH2O_3DOF/aux/rotation_test.py +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.py @@ -17,6 +17,17 @@ def calculate_bondangle(atom1, atom2, atom3, radian): angle = np.degrees(angle) return angle +bondlength1 = calculate_bondlength(H1, O1) +bondlength2 = calculate_bondlength(H2, O1) +bondangle = calculate_bondangle(H1, O1, H2, False) + +print('Original system') +print(f'H1 = {H1}') +print(f'H2 = {H2}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') + def rotation_matrix(axis, angle): cos_theta = np.cos(angle) sin_theta = np.sin(angle) @@ -27,6 +38,8 @@ def rotation_matrix(axis, angle): [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] ]) +H1_rotated, H2_rotated = H1, H2 + plane_normal = np.cross(H2, H1) plane_normal = plane_normal / np.linalg.norm(plane_normal) @@ -34,47 +47,53 @@ def rotation_matrix(axis, angle): axis_to_align = np.cross(plane_normal, target_plane_normal) axis_to_align /= np.linalg.norm(axis_to_align) angle_to_align = np.arccos(np.clip(np.dot(plane_normal, target_plane_normal), -1.0, 1.0)) -rot_matrix_align_plane = rotation_matrix(axis_to_align, angle_to_align) - -bondlength1 = calculate_bondlength(H1, O1) -bondlength2 = calculate_bondlength(H2, O1) -bondangle = calculate_bondangle(H1, O1, H2, False) +out_of_plane_rotation_matrix = rotation_matrix(axis_to_align, angle_to_align) +H1_rotated = np.dot(out_of_plane_rotation_matrix, H1) +H2_rotated = np.dot(out_of_plane_rotation_matrix, H2) -print('Original system') -print(f'H1 = {H1}') -print(f'H2 = {H2}') -print(f'Bondlength of O1-H1 = {bondlength1}') -print(f'Bondlength of O1-H2 = {bondlength2}') -print(f'Angle between O1-H1 and O1-H2 = {bondangle}') +theta1 = np.arctan2(H1_rotated[1], H1_rotated[0]) +theta2 = np.arctan2(H2_rotated[1], H2_rotated[0]) +planar_rotation_angle = -theta1 + np.radians(bondangle) / 2 +planar_rotation_matrix = rotation_matrix(target_plane_normal, planar_rotation_angle) +H1_rotated = np.dot(planar_rotation_matrix, H1_rotated) +H2_rotated = np.dot(planar_rotation_matrix, H2_rotated) -H1_rotated = np.dot(rot_matrix_align_plane, H1) -H2_rotated = np.dot(rot_matrix_align_plane, H2) -fliped_bond = False if bondlength1 < bondlength2: - fliped_bond = True - H1_rotated, H2_rotated = H2_rotated, H1_rotated -bondlength1 = calculate_bondlength(H1_rotated, O1) -bondlength2 = calculate_bondlength(H2_rotated, O1) -bondangle = calculate_bondangle(H1_rotated, O1, H2_rotated, False) + H1_rotated, H2_rotated = H2_rotated, H1_rotated + H1_rotated[1] *= -1 + H2_rotated[1] *= -1 -print('Reference system (z=0 plane about x=0 axis, with longer bondlength in H1)') +bondlength1_rotated = calculate_bondlength(H1_rotated, O1) +bondlength2_rotated = calculate_bondlength(H2_rotated, O1) +bondangle_rotated = calculate_bondangle(H1_rotated, O1, H2_rotated, False) + +print('Reference system (z=0 plane, symmetric about x=0 axis, with longer bondlength in Q1)') print(f'H1 = {H1_rotated}') print(f'H2 = {H2_rotated}') -print(f'Bondlength of O1-H1 = {bondlength1}') -print(f'Bondlength of O1-H2 = {bondlength2}') -print(f'Angle between O1-H1 and O1-H2 = {bondangle}') +print(f'Bondlength of O1-H1 = {bondlength1_rotated}') +print(f'Bondlength of O1-H2 = {bondlength2_rotated}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle_rotated}') -if fliped_bond: - H1_rotated, H2_rotated = H2_rotated, H1_rotated -H1_restored = np.dot(rot_matrix_align_plane.T, H1_rotated) -H2_restored = np.dot(rot_matrix_align_plane.T, H2_rotated) -bondlength1 = calculate_bondlength(H1_restored, O1) -bondlength2 = calculate_bondlength(H2_restored, O1) -bondangle = calculate_bondangle(H1_restored, O1, H2_restored, False) +H1_restored, H2_restored = H1_rotated, H2_rotated + +if bondlength1 < bondlength2: + H1_rotated, H2_rotated = H2_rotated, H1_rotated + H1_rotated[1] *= -1 + H2_rotated[1] *= -1 + +H1_restored = np.dot(planar_rotation_matrix.T, H1_restored) +H2_restored = np.dot(planar_rotation_matrix.T, H2_restored) + +H1_restored = np.dot(out_of_plane_rotation_matrix.T, H1_restored) +H2_restored = np.dot(out_of_plane_rotation_matrix.T, H2_restored) + +bondlength1_restored = calculate_bondlength(H1_restored, O1) +bondlength2_restored = calculate_bondlength(H2_restored, O1) +bondangle_restored = calculate_bondangle(H1_restored, O1, H2_restored, False) print('Restored system') print(f'H1 = {H1_restored}') print(f'H2 = {H2_restored}') -print(f'Bondlength of O1-H1 = {bondlength1}') -print(f'Bondlength of O1-H2 = {bondlength2}') -print(f'Angle between O1-H1 and O1-H2 = {bondangle}') +print(f'Bondlength of O1-H1 = {bondlength1_restored}') +print(f'Bondlength of O1-H2 = {bondlength2_restored}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle_restored}') diff --git a/tests/PinnedH2O_3DOF/coords.in_rotate1 b/tests/PinnedH2O_3DOF/coords.in_rotate1 new file mode 100644 index 00000000..7498ff55 --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords.in_rotate1 @@ -0,0 +1,3 @@ +O1 1 0.00000 0.00000 0.00 0 +H1 2 -1.08248 -1.47039 0.00 1 +H2 2 -1.14768 1.43060 0.00 1 diff --git a/tests/PinnedH2O_3DOF/coords.in_rotate2 b/tests/PinnedH2O_3DOF/coords.in_rotate2 new file mode 100644 index 00000000..307e170c --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords.in_rotate2 @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.42 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis index ccc03c9d..eaf8bfde 100644 --- a/tests/PinnedH2O_3DOF/job.basis +++ b/tests/PinnedH2O_3DOF/job.basis @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20241220 +set maindir = /p/lustre2/cheung26/mgmol-20250206 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.basis_test1 b/tests/PinnedH2O_3DOF/job.basis_test1 deleted file mode 100644 index 3955cdc6..00000000 --- a/tests/PinnedH2O_3DOF/job.basis_test1 +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/tcsh -#SBATCH -N 2 -#SBATCH -t 0:10:00 -#SBATCH -p pdebug - -date - -setenv OMP_NUM_THREADS 1 -#setenv KMP_DETERMINISTIC_REDUCTION 1 - -set ncpus = 64 - -set maindir = /p/lustre2/cheung26/mgmol-20241220 - -setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH - -set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples - -set basis_tag = "test1" -set snapshot_files = "" - -set bondlength_one = 1.00 -set bondlength_two = 1.00 -set bondangle = 0.0 -echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" -set tag = ${bondlength_one}_${bondlength_two}_${bondangle} -set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" - -echo "Snapshot files: $snapshot_files" - -set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${basis_tag}" -srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${basis_tag}_PinnedH2O_3DOF.out - -date diff --git a/tests/PinnedH2O_3DOF/job.basis_test2 b/tests/PinnedH2O_3DOF/job.basis_test2 deleted file mode 100644 index d0ac6301..00000000 --- a/tests/PinnedH2O_3DOF/job.basis_test2 +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/tcsh -#SBATCH -N 2 -#SBATCH -t 0:10:00 -#SBATCH -p pdebug - -date - -setenv OMP_NUM_THREADS 1 -#setenv KMP_DETERMINISTIC_REDUCTION 1 - -set ncpus = 64 - -set maindir = /p/lustre2/cheung26/mgmol-20241220 - -setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH - -set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples - -set basis_tag = "test2" -set snapshot_files = "" - -set bondlength_one = 1.00 -set bondlength_two = 1.00 -set bondangle = 0.0 -echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" -set tag = ${bondlength_one}_${bondlength_two}_${bondangle} -set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" - -set bondangle = 1.0 -echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" -set tag = ${bondlength_one}_${bondlength_two}_${bondangle} -set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" - -echo "Snapshot files: $snapshot_files" - -set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${basis_tag}" -srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${basis_tag}_PinnedH2O_3DOF.out - -date diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline index 7fc51dd7..3870a3a3 100644 --- a/tests/PinnedH2O_3DOF/job.offline +++ b/tests/PinnedH2O_3DOF/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20241220 +set maindir = /p/lustre2/cheung26/mgmol-20250206 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online index e703fc29..77b8fde0 100644 --- a/tests/PinnedH2O_3DOF/job.online +++ b/tests/PinnedH2O_3DOF/job.online @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20241220 +set maindir = /p/lustre2/cheung26/mgmol-20250206 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online_test1 b/tests/PinnedH2O_3DOF/job.online_rotate similarity index 55% rename from tests/PinnedH2O_3DOF/job.online_test1 rename to tests/PinnedH2O_3DOF/job.online_rotate index bf69a1ef..11dd7419 100644 --- a/tests/PinnedH2O_3DOF/job.online_test1 +++ b/tests/PinnedH2O_3DOF/job.online_rotate @@ -10,21 +10,27 @@ setenv OMP_NUM_THREADS 1 set ncpus = 64 -set maindir = /p/lustre2/cheung26/mgmol-20241220 +set maindir = /p/lustre2/cheung26/mgmol-20250206 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH -set exe = testPinnedH2O_3DOF - -cp $maindir/build_quartz/tests/$exe . - -set cfg_online = mgmol_online_test1.cfg - ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . source $maindir/scripts/modules.quartz -srun -p pdebug -n $ncpus $exe -c $cfg_online -i coords_1.00_1.00_0.0.in +rm -rf snapshot0_* + +set coord_file = coords.in_rotate2 + +set exe = mgmol-opt +set cfg_online = mgmol_ref_rotate.cfg +cp $maindir/install_quartz/bin/$exe . +srun -p pdebug -n $ncpus $exe -c $cfg_online -i $coord_file + +set exe = testPinnedH2O_3DOF +set cfg_online = mgmol_online_rotate.cfg +cp $maindir/build_quartz/tests/$exe . +srun -p pdebug -n $ncpus $exe -c $cfg_online -i $coord_file date diff --git a/tests/PinnedH2O_3DOF/job.online_test2 b/tests/PinnedH2O_3DOF/job.online_test2 deleted file mode 100644 index 88665f1e..00000000 --- a/tests/PinnedH2O_3DOF/job.online_test2 +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/tcsh -#SBATCH -N 2 -#SBATCH -t 0:10:00 -#SBATCH -p pdebug - -date - -setenv OMP_NUM_THREADS 1 -#setenv KMP_DETERMINISTIC_REDUCTION 1 - -set ncpus = 64 - -set maindir = /p/lustre2/cheung26/mgmol-20241220 - -setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH - -set exe = testPinnedH2O_3DOF - -cp $maindir/build_quartz/tests/$exe . - -set cfg_online = mgmol_online_test2.cfg - -ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . -ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . - -source $maindir/scripts/modules.quartz - -srun -p pdebug -n $ncpus $exe -c $cfg_online -i coords_1.00_1.00_0.0.in - -date diff --git a/tests/PinnedH2O_3DOF/mgmol_online_test2.cfg b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg similarity index 76% rename from tests/PinnedH2O_3DOF/mgmol_online_test2.cfg rename to tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg index 8ce3e32d..72288ac3 100644 --- a/tests/PinnedH2O_3DOF/mgmol_online_test2.cfg +++ b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg @@ -18,19 +18,19 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=quench [Quench] -max_steps=1000 +max_steps=100 atol=1.e-8 [Orbitals] initial_type=Random initial_width=2. -nempty=4 +nempty=68 [Restart] output_level=4 [DensityMatrix] solver=MVP -nb_inner_it=20 +nb_inner_it=100 [ROM.offline] -basis_file=data/PinnedH2O_3DOF_orbitals_basis_test2 +basis_file=data/PinnedH2O_3DOF_orbitals_basis_2_2 [ROM.basis] compare_md=false -number_of_orbital_basis=8 +number_of_orbital_basis=72 diff --git a/tests/PinnedH2O_3DOF/mgmol_online_test1.cfg b/tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg similarity index 70% rename from tests/PinnedH2O_3DOF/mgmol_online_test1.cfg rename to tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg index e032350d..96be6b8d 100644 --- a/tests/PinnedH2O_3DOF/mgmol_online_test1.cfg +++ b/tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg @@ -1,4 +1,4 @@ -verbosity=2 +verbosity=1 xcFunctional=PBE FDtype=4th [Mesh] @@ -25,8 +25,3 @@ initial_type=Random initial_width=2. [Restart] output_level=4 -[ROM.offline] -basis_file=data/PinnedH2O_3DOF_orbitals_basis_test1 -[ROM.basis] -compare_md=false -number_of_orbital_basis=4 diff --git a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc index e7813777..a0cd51d3 100644 --- a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc +++ b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc @@ -22,10 +22,222 @@ #include #include #include +#include +#include +#include #include namespace po = boost::program_options; +double calculate_bondlength(const double atom1[3], const double atom2[3]) +{ + return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + pow(atom1[2] - atom2[2], 2)); +} + +double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3]) +{ + double vector1[3] = {atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2]}; + double vector2[3] = {atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2]}; + + double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + vector1[2] * vector2[2]; + double magnitude_product = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) * + sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); + double angle = acos(dot_product / magnitude_product); + + return angle; +} + +void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) +{ + double cos_theta = cos(angle); + double sin_theta = sin(angle); + double ux = axis[0], uy = axis[1], uz = axis[2]; + + matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); + matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; + matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; + + matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; + matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); + matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; + + matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; + matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; + matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); +} + +void normalize(double vec[3]) +{ + double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; +} + +void cross(const double a[3], const double b[3], double result[3]) +{ + result[0] = a[1] * b[2] - a[2] * b[1]; + result[1] = a[2] * b[0] - a[0] * b[2]; + result[2] = a[0] * b[1] - a[1] * b[0]; +} + +void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) +{ + result[0] = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; + result[1] = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; + result[2] = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; +} + +void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) +{ + result[0] = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; + result[1] = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; + result[2] = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; +} + +void rotate_PinnedH2O(std::vector& positions, std::vector& anumbers, + double out_of_plane_rotation_matrix[3][3], double& planar_rotation_angle, bool& flipped_bond) +{ + int O1_idx = -1; + for (int i = 0; i < 3; i++) + { + if (positions[3*i] == 0.0 && positions[3*i+1] == 0.0 && positions[3*i+2] == 0.0) + { + O1_idx = i; + break; + } + } + if (O1_idx == -1) return; + + int H1_idx = (O1_idx + 1) % 3; + int H2_idx = (O1_idx + 2) % 3; + + double O1[3] = {positions[3*O1_idx], positions[3*O1_idx+1], positions[3*O1_idx+2]}; + double H1[3] = {positions[3*H1_idx], positions[3*H1_idx+1], positions[3*H1_idx+2]}; + double H2[3] = {positions[3*H2_idx], positions[3*H2_idx+1], positions[3*H2_idx+2]}; + + double bondlength1 = calculate_bondlength(H1, O1); + double bondlength2 = calculate_bondlength(H2, O1); + double bondangle = calculate_bondangle(H1, O1, H2); + + double H1_temp[3], H2_temp[3]; + double H1_rotated[3], H2_rotated[3]; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + double target_plane_normal[3] = {0, 0, 1}; + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align = std::acos(std::min(std::max(dot_product, -1.0), 1.0)); + double axis_to_align[3]; + if (abs(dot_product) > 1.0 - 1e-8) + { + axis_to_align[0] = 1.0; + axis_to_align[1] = 0.0; + axis_to_align[2] = 0.0; + } + else + { + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + } + rotation_matrix(axis_to_align, angle_to_align, out_of_plane_rotation_matrix); + apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); + apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); + + double theta1 = std::atan2(H1_temp[1], H1_temp[0]); + planar_rotation_angle = -theta1 + bondangle / 2.0; + double planar_rotation_matrix[3][3]; + rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); + apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); + + flipped_bond = (bondlength1 < bondlength2); + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + std::swap(H1_rotated, H2_rotated); + } + + positions[0] = H2_rotated[0]; + positions[1] = H2_rotated[1]; + positions[2] = H2_rotated[2]; + positions[3] = 0.0; + positions[4] = 0.0; + positions[5] = 0.0; + positions[6] = H1_rotated[0]; + positions[7] = H1_rotated[1]; + positions[8] = H1_rotated[2]; + + anumbers[0] = 1; + anumbers[1] = 8; + anumbers[2] = 1; +} + +void transpose_rotate_PinnedH2O(std::vector& positions, std::vector& forces, + const double out_of_plane_rotation_matrix[3][3], + const double planar_rotation_angle, const bool flipped_bond) +{ + double H2_rotated[3] = {positions[0], positions[1], positions[2]}; + double O1_rotated[3] = {positions[3], positions[4], positions[5]}; + double H1_rotated[3] = {positions[6], positions[7], positions[8]}; + + double f_H2_rotated[3] = {forces[0], forces[1], forces[2]}; + double f_O1_rotated[3] = {forces[3], forces[4], forces[5]}; + double f_H1_rotated[3] = {forces[6], forces[7], forces[8]}; + + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + f_O1_rotated[1] *= -1.0; + f_H1_rotated[1] *= -1.0; + f_H2_rotated[1] *= -1.0; + } + + double planar_rotation_matrix[3][3]; + double target_plane_normal[3] = {0, 0, 1}; + rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + + double H1_temp[3], H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); + apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); + + double f_O1_temp[3], f_H1_temp[3], f_H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, f_O1_rotated, f_O1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H1_rotated, f_H1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H2_rotated, f_H2_temp); + + double H1_restored[3], H2_restored[3]; + apply_transpose_rotation(out_of_plane_rotation_matrix, H1_temp, H1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, H2_temp, H2_restored); + + double f_O1_restored[3], f_H1_restored[3], f_H2_restored[3]; + apply_transpose_rotation(out_of_plane_rotation_matrix, f_O1_temp, f_O1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, f_H1_temp, f_H1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, f_H2_temp, f_H2_restored); + + positions[0] = H2_restored[0]; + positions[1] = H2_restored[1]; + positions[2] = H2_restored[2]; + positions[6] = H1_restored[0]; + positions[7] = H1_restored[1]; + positions[8] = H1_restored[2]; + + forces[0] = f_H2_restored[0]; + forces[1] = f_H2_restored[1]; + forces[2] = f_H2_restored[2]; + forces[3] = f_O1_restored[0]; + forces[4] = f_O1_restored[1]; + forces[5] = f_O1_restored[2]; + forces[6] = f_H1_restored[0]; + forces[7] = f_H1_restored[1]; + forces[8] = f_H1_restored[2]; +} + int main(int argc, char** argv) { int mpirc = MPI_Init(&argc, &argv); @@ -127,6 +339,13 @@ int main(int argc, char** argv) } } + // rotate the molecule to the reference coordinate system + int atom_order[3]; + double out_of_plane_rotation_matrix[3][3]; + double planar_rotation_angle; + bool flipped_bond; + rotate_PinnedH2O(positions, anumbers, out_of_plane_rotation_matrix, planar_rotation_angle, flipped_bond); + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -172,13 +391,58 @@ int main(int argc, char** argv) if (MPIdata::onpe0) { std::cout << "Eks: " << eks << std::endl; + std::cout << "Positions in the reference domain:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + + std::cout << "Forces in the reference domain:" << std::endl; + ita = anumbers.begin(); + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // rotate the forces to the original coordinate system + transpose_rotate_PinnedH2O(positions, forces, out_of_plane_rotation_matrix, planar_rotation_angle, flipped_bond); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } std::cout << "Forces:" << std::endl; + ita = anumbers.begin(); for (std::vector::iterator it = forces.begin(); it != forces.end(); it += 3) { + std::cout << *ita; for (int i = 0; i < 3; i++) std::cout << " " << *(it + i); std::cout << std::endl; + ita++; } } From 488c9610fd84b7acf61804bcf023b5ab68e89bb1 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Wed, 19 Feb 2025 11:02:11 -0800 Subject: [PATCH 38/50] Merge release to ROMFPMD (#307) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Add commit number temporarily * Fix merge --------- Co-authored-by: Jean-Luc Fattebert --- potentials/pseudo.Br_ONCV_PBE_SG15 | 2529 ++++++++++++++++++++++++++ scripts/build_quartz_libROM.sh | 1 + scripts/build_ubuntu22_openmpi.sh | 2 +- src/Control.cc | 25 +- src/Control.h | 2 + src/DistributedIonicData.cc | 13 +- src/HDFrestart.cc | 608 +------ src/HDFrestart.h | 13 +- src/HamiltonianMVP_DMStrategy.cc | 2 +- src/Ion.h | 2 +- src/IonData.cc | 17 +- src/IonicStepper.cc | 83 +- src/Ions.cc | 69 +- src/LocalizationRegions.cc | 3 +- src/MD_IonicStepper.cc | 125 +- src/MD_IonicStepper.h | 13 +- src/MVP_DMStrategy.h | 2 +- src/Potentials.cc | 19 +- src/Potentials.h | 27 +- src/hdf_tools.cc | 30 +- src/md.cc | 3 + src/mgmol_run.cc | 1 - src/restart.cc | 10 +- tests/CMakeLists.txt | 8 + tests/DMandEnergyAndForces/test.py | 3 + tests/HDF5single/h2o.xyz | 6 + tests/HDF5single/md.cfg | 34 + tests/HDF5single/mgmol.cfg | 31 + tests/HDF5single/test.py | 75 + tests/RestartEnergyAndForces/test.py | 5 +- tests/WFEnergyAndForces/test.py | 3 + tests/testSetGhostValues.cc | 4 - 32 files changed, 2933 insertions(+), 835 deletions(-) create mode 100644 potentials/pseudo.Br_ONCV_PBE_SG15 create mode 100644 tests/HDF5single/h2o.xyz create mode 100644 tests/HDF5single/md.cfg create mode 100644 tests/HDF5single/mgmol.cfg create mode 100755 tests/HDF5single/test.py diff --git a/potentials/pseudo.Br_ONCV_PBE_SG15 b/potentials/pseudo.Br_ONCV_PBE_SG15 new file mode 100644 index 00000000..c135617b --- /dev/null +++ b/potentials/pseudo.Br_ONCV_PBE_SG15 @@ -0,0 +1,2529 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# Br 35.00 6 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 6.00 +# 3 0 2.00 +# 3 1 6.00 +# 3 2 10.00 +# 4 0 2.00 +# 4 1 5.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 2 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 2.49793 -0.73993 5 8 5.10687 +# 1 2.49793 -0.28884 5 8 4.46675 +# 2 2.49793 0.09432 5 8 4.91674 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 2.24365 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.26029 +# 1 2 1.41324 +# 2 2 1.05278 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +Br_ONCV_PBE +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +35 +# Atomic mass +79.90399933 +# Number of valence electrons +7 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +4 +# l-value for state which is local, then type of potential format +3 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 -0.3626325499E+01 0.1750463805E+01 +1 2 0.4803824052E+02 0.7506916045E+00 +2 2 0.1514320363E+02 0.8114306630E-01 +# l= 0 +0.0 -3.0708154786 -2.4047930787 +0.01 -3.070772567 -2.4042133705 +0.02 -3.0706536405 -2.4024662354 +0.03 -3.0704488283 -2.3995597251 +0.04 -3.0701482591 -2.3955018914 +0.05 -3.0697381761 -2.3903039657 +0.06 -3.0692009986 -2.3839803178 +0.07 -3.0685154018 -2.376548404 +0.08 -3.0676564136 -2.3680287033 +0.09 -3.0665955275 -2.3584446428 +0.1 -3.0653008322 -2.3478225123 +0.11 -3.063737157 -2.3361913688 +0.12 -3.0618662322 -2.3235829307 +0.13 -3.0596468635 -2.3100314626 +0.14 -3.0570351213 -2.2955736506 +0.15 -3.053984542 -2.2802484691 +0.16 -3.0504463422 -2.2640970395 +0.17 -3.0463696444 -2.2471624809 +0.18 -3.0417017127 -2.2294897538 +0.19 -3.0363881987 -2.2111254979 +0.2 -3.030373396 -2.1921178621 +0.21 -3.0236005018 -2.1725163322 +0.22 -3.0160118859 -2.1523715511 +0.23 -3.0075493645 -2.1317351366 +0.24 -2.9981544792 -2.1106594957 +0.25 -2.987768779 -2.0891976363 +0.26 -2.976334104 -2.0674029769 +0.27 -2.9637928718 -2.0453291551 +0.28 -2.9500883622 -2.0230298363 +0.29 -2.9351650013 -2.0005585216 +0.3 -2.9189686442 -1.9779683581 +0.31 -2.9014468534 -1.9553119497 +0.32 -2.8825491704 -1.9326411716 +0.33 -2.8622273897 -1.910006986 +0.34 -2.840435816 -1.8874592637 +0.35 -2.8171315174 -1.8650466102 +0.36 -2.7922745751 -1.8428161943 +0.37 -2.7658283185 -1.820813584 +0.38 -2.7377595404 -1.7990825945 +0.39 -2.7080387236 -1.7776651304 +0.4 -2.6766402368 -1.7566010457 +0.41 -2.643542518 -1.7359280108 +0.42 -2.6087282508 -1.7156813851 +0.43 -2.5721845223 -1.6958940993 +0.44 -2.5339029669 -1.6765965483 +0.45 -2.4938798925 -1.6578164926 +0.46 -2.4521163913 -1.639578971 +0.47 -2.4086184283 -1.6219062255 +0.48 -2.3633969093 -1.6048176402 +0.49 -2.3164677785 -1.588329662 +0.5 -2.2678520179 -1.5724557793 +0.51 -2.2175756537 -1.5572064995 +0.52 -2.165669807 -1.5425892978 +0.53 -2.1121706776 -1.5286086079 +0.54 -2.0571194038 -1.5152658873 +0.55 -2.0005622031 -1.5025595131 +0.56 -1.9425500828 -1.4904849389 +0.57 -1.8831389418 -1.4790346138 +0.58 -1.8223892906 -1.4681981355 +0.59 -1.760366311 -1.4579621933 +0.6 -1.6971394766 -1.4483107855 +0.61 -1.6327826828 -1.439225116 +0.62 -1.5673737973 -1.4306838573 +0.63 -1.5009946281 -1.4226631485 +0.64 -1.4337307139 -1.415136705 +0.65 -1.365670995 -1.4080760077 +0.66 -1.2969076816 -1.4014503615 +0.67 -1.227536009 -1.3952270271 +0.68 -1.1576539096 -1.3893714093 +0.69 -1.087361756 -1.3838471989 +0.7 -1.0167621481 -1.3786164792 +0.71 -0.94595957813 -1.3736399185 +0.72 -0.87506012463 -1.3688769435 +0.73 -0.80417116303 -1.3642858995 +0.74 -0.7334010578 -1.3598242236 +0.75 -0.66285885089 -1.355448623 +0.76 -0.59265395182 -1.351115238 +0.77 -0.52289580245 -1.3467798445 +0.78 -0.45369356504 -1.3423980267 +0.79 -0.38515580233 -1.337925358 +0.8 -0.31739015855 -1.3333175826 +0.81 -0.25050304259 -1.3285307948 +0.82 -0.18459931451 -1.3235216181 +0.83 -0.11978197659 -1.3182473801 +0.84 -0.056151869857 -1.3126662853 +0.85 0.0061926226578 -1.3067375838 +0.86 0.067155864931 -1.3004217357 +0.87 0.12664524827 -1.2936805697 +0.88 0.18457146187 -1.2864774371 +0.89 0.2408487527 -1.2787773586 +0.9 0.29539517376 -1.2705471649 +0.91 0.34813281968 -1.2617556292 +0.92 0.39898804895 -1.2523735929 +0.93 0.44789169169 -1.2423740812 +0.94 0.49477924217 -1.2317324118 +0.95 0.53959097092 -1.2204262598 +0.96 0.58227215742 -1.2084357833 +0.97 0.6227733871 -1.1957437779 +0.98 0.66105043859 -1.1823356271 +0.99 0.6970644809 -1.1681994044 +1.0 0.73078216308 -1.1533259198 +1.01 0.76217568426 -1.1377087564 +1.02 0.79122266757 -1.1213442058 +1.03 0.81790653051 -1.1042314518 +1.04 0.84221661359 -1.0863726296 +1.05 0.86414774871 -1.0677726042 +1.06 0.88370040536 -1.048439037 +1.07 0.90088046461 -1.0283822657 +1.08 0.91569937292 -1.0076153718 +1.09 0.92817465654 -0.98615441727 +1.1 0.93832889415 -0.96401792656 +1.11 0.9461899162 -0.94122697008 +1.12 0.9517900713 -0.91780480034 +1.13 0.95516808541 -0.8937777179 +1.14 0.95636695802 -0.86917403945 +1.15 0.95543424502 -0.84402421656 +1.16 0.95242155537 -0.81836058881 +1.17 0.94738611915 -0.79221806224 +1.18 0.94038861073 -0.76563307802 +1.19 0.93149314032 -0.73864360126 +1.2 0.9207680708 -0.71128944543 +1.21 0.90828544129 -0.68361195316 +1.22 0.89411988748 -0.65565351703 +1.23 0.87834887992 -0.62745766744 +1.24 0.86105341064 -0.5990692311 +1.25 0.84231617223 -0.57053359643 +1.26 0.82222172706 -0.54189679743 +1.27 0.80085735674 -0.51320561099 +1.28 0.77831122726 -0.48450694828 +1.29 0.75467272985 -0.45584799729 +1.3 0.73003271242 -0.42727593895 +1.31 0.70448226061 -0.39883777816 +1.32 0.6781130858 -0.37058037334 +1.33 0.65101693162 -0.34254982352 +1.34 0.62328542089 -0.31479192232 +1.35 0.59500981379 -0.28735157546 +1.36 0.56628028474 -0.26027246962 +1.37 0.53718687691 -0.23359805009 +1.38 0.50781737223 -0.20736952858 +1.39 0.47825865123 -0.18162739142 +1.4 0.44859623165 -0.15641087651 +1.41 0.41891237101 -0.13175663087 +1.42 0.38928923477 -0.10770120711 +1.43 0.35980439328 -0.084277653647 +1.44 0.33053421696 -0.061518184098 +1.45 0.30155239027 -0.03945302065 +1.46 0.27292797429 -0.0181091649 +1.47 0.24473036233 0.002486186983 +1.48 0.2170211898 0.022311130325 +1.49 0.18986298266 0.041343142397 +1.5 0.16331164708 0.059564125665 +1.51 0.13742115 0.076957214701 +1.52 0.11224181642 0.093507941835 +1.53 0.087818451049 0.10920533268 +1.54 0.064195549675 0.12403864261 +1.55 0.041408563173 0.13800275831 +1.56 0.019495861764 0.15109077964 +1.57 -0.0015158417959 0.16330294016 +1.58 -0.021596235849 0.17463728157 +1.59 -0.040725475422 0.18509813619 +1.6 -0.058881960784 0.19468876976 +1.61 -0.076052534895 0.20341742456 +1.62 -0.092224070602 0.21129232349 +1.63 -0.10739003129 0.21832549573 +1.64 -0.12154529981 0.2245297272 +1.65 -0.13468985191 0.22992068159 +1.66 -0.14682597759 0.23451524024 +1.67 -0.15795999363 0.23833250484 +1.68 -0.16810089293 0.24139293702 +1.69 -0.17726108092 0.24371882479 +1.7 -0.18545548467 0.24533367189 +1.71 -0.19270225049 0.24626263687 +1.72 -0.19902144307 0.24653172825 +1.73 -0.20443649328 0.2461685876 +1.74 -0.20897181658 0.24520121549 +1.75 -0.21265555109 0.24365927067 +1.76 -0.21551572366 0.24157228058 +1.77 -0.21758447021 0.23897137241 +1.78 -0.21889277302 0.23588720289 +1.79 -0.21947591783 0.23235173626 +1.8 -0.21936729817 0.22839644157 +1.81 -0.21860435564 0.224053418 +1.82 -0.2172225127 0.21935470487 +1.83 -0.21526018396 0.21433189433 +1.84 -0.21275469141 0.20901710885 +1.85 -0.2097443342 0.2034411581 +1.86 -0.20626747958 0.19763549074 +1.87 -0.20236211696 0.19163011691 +1.88 -0.19806642786 0.18545493817 +1.89 -0.19341811175 0.17913927614 +1.9 -0.18845384334 0.17271059028 +1.91 -0.18321119098 0.16619768388 +1.92 -0.17772431694 0.15962506074 +1.93 -0.17202997839 0.15302021981 +1.94 -0.1661601512 0.14640550171 +1.95 -0.16014881031 0.13980526812 +1.96 -0.15402698692 0.13324102441 +1.97 -0.14782425499 0.12673280543 +1.98 -0.14157131194 0.12030174533 +1.99 -0.13529270013 0.11396333406 +2.0 -0.12901767217 0.10773726718 +2.01 -0.12276761168 0.10163636831 +2.02 -0.11656673377 0.095675877908 +2.03 -0.1104363775 0.089868634417 +2.04 -0.10439399399 0.084224202689 +2.05 -0.098461395649 0.0787557483 +2.06 -0.09265062254 0.07346847772 +2.07 -0.086979074138 0.068371912909 +2.08 -0.081458820035 0.06347135213 +2.09 -0.076100125576 0.058770653954 +2.1 -0.070916200757 0.054275981844 +2.11 -0.065911720591 0.049986904382 +2.12 -0.061096352253 0.04590680729 +2.13 -0.05647481096 0.04203534513 +2.14 -0.052050288108 0.038371018518 +2.15 -0.047828987027 0.034914562031 +2.16 -0.043809718782 0.031661293642 +2.17 -0.039995100139 0.028609286026 +2.18 -0.036385130803 0.025754761958 +2.19 -0.032977565996 0.023092357681 +2.2 -0.029772938118 0.020618609458 +2.21 -0.02676722353 0.018327015582 +2.22 -0.023957902849 0.016212045303 +2.23 -0.021342381035 0.014268136652 +2.24 -0.018915988096 0.01248852304 +2.25 -0.016675937928 0.010867451055 +2.26 -0.014617052811 0.0093979619195 +2.27 -0.012733401274 0.0080727169653 +2.28 -0.011021551686 0.0068855480404 +2.29 -0.0094727420429 0.0058278549281 +2.3 -0.0080800306285 0.004891808646 +2.31 -0.0068367888466 0.0040696905805 +2.32 -0.0057335326398 0.0033529863328 +2.33 -0.0047622051973 0.002733387986 +2.34 -0.0039142269282 0.0022027527767 +2.35 -0.003180287976 0.001753168643 +2.36 -0.0025514070016 0.0013760738931 +2.37 -0.002018701345 0.0010641189905 +2.38 -0.0015733073204 0.00081018747083 +2.39 -0.0012057416961 0.000605822272 +2.4 -0.00090807593031 0.00044534423372 +2.41 -0.00067211676854 0.00032260120986 +2.42 -0.00048836884014 0.00022964110509 +2.43 -0.00035064527094 0.00016295096584 +2.44 -0.00025146290618 0.00011730333122 +2.45 -0.00018232534191 8.6257107481e-05 +2.46 -0.00013859768358 6.798715492e-05 +2.47 -0.00011459867708 5.9010018177e-05 +2.48 -0.00010538902877 5.5962206051e-05 +2.49 -0.0001014873045 5.4745829591e-05 +2.5 -9.0886540089e-05 4.9286426798e-05 +2.51 -5.7587161462e-05 3.1411431787e-05 +2.52 -2.3406234748e-05 1.2820803441e-05 +2.53 2.0554012075e-06 -1.1129389439e-06 +2.54 6.3698356898e-06 -3.4490775717e-06 +2.55 4.1835875739e-06 -2.265288898e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 0.16768810909 -0.09197895777 +0.02 0.33490177013 -0.18377178817 +0.03 0.50116814078 -0.27519297668 +0.04 0.66601758415 -0.36605823207 +0.05 0.82898525636 -0.45618509197 +0.06 0.98961267609 -0.54539352126 +0.07 1.1474492702 -0.63350650089 +0.08 1.3020538895 -0.7203506047 +0.09 1.4529962891 -0.80575656223 +0.1 1.5998585678 -0.88955980496 +0.11 1.7422365611 -0.97160099425 +0.12 1.8797411823 -1.0517265285 +0.13 2.0119997077 -1.129789028 +0.14 2.138657 -1.2056477949 +0.15 2.2593766658 -1.2791692479 +0.16 2.3738421434 -1.3502273279 +0.17 2.4817577163 -1.4187038753 +0.18 2.5828494491 -1.4844889757 +0.19 2.6768660426 -1.5474812743 +0.2 2.7635796037 -1.6075882562 +0.21 2.8427863308 -1.6647264933 +0.22 2.9143071076 -1.7188218556 +0.23 2.9779880072 -1.7698096863 +0.24 3.0337007024 -1.8176349412 +0.25 3.0813427856 -1.862252292 +0.26 3.1208379864 -1.9036261898 +0.27 3.1521363034 -1.9417308947 +0.28 3.1752140319 -1.9765504665 +0.29 3.1900737018 -2.0080787184 +0.3 3.1967439186 -2.0363191351 +0.31 3.1952791115 -2.0612847542 +0.32 3.1857591886 -2.0829980127 +0.33 3.1682891025 -2.1014905583 +0.34 3.1429983262 -2.1168030277 +0.35 3.110040245 -2.128984793 +0.36 3.0695914617 -2.1380936747 +0.37 3.0218510224 -2.1441956261 +0.38 2.9670395749 -2.1473643925 +0.39 2.9053984349 -2.1476811361 +0.4 2.8371885957 -2.1452340407 +0.41 2.7626896752 -2.1401178945 +0.42 2.6821987951 -2.1324336474 +0.43 2.596029406 -2.1222879509 +0.44 2.5045100614 -2.1097926797 +0.45 2.4079831454 -2.0950644377 +0.46 2.3068035597 -2.0782240514 +0.47 2.2013373814 -2.0593960544 +0.48 2.0919605029 -2.0387081682 +0.49 1.979057148 -2.0162907368 +0.5 1.8630185217 -1.9922762184 +0.51 1.7442414153 -1.9667986605 +0.52 1.6231266566 -1.9399931205 +0.53 1.500077649 -1.9119951227 +0.54 1.3754991997 -1.88294023 +0.55 1.2497955863 -1.852963328 +0.56 1.123369708 -1.8221983257 +0.57 0.99662116191 -1.7907774522 +0.58 0.86994536282 -1.7588309508 +0.59 0.74373162879 -1.7264863891 +0.6 0.61836264621 -1.6938684873 +0.61 0.49421220571 -1.6610983091 +0.62 0.37164494723 -1.628293199 +0.63 0.25101461062 -1.5955661733 +0.64 0.1326628874 -1.5630255372 +0.65 0.016918843032 -1.5307747113 +0.66 -0.095902538203 -1.4989117459 +0.67 -0.20550134835 -1.4675290072 +0.68 -0.31159326886 -1.4367130384 +0.69 -0.4139103595 -1.4065443242 +0.7 -0.51220225666 -1.377096921 +0.71 -0.60623655903 -1.3484383751 +0.72 -0.6957993 -1.3206296129 +0.73 -0.78069554004 -1.2937247949 +0.74 -0.86074977919 -1.267771236 +0.75 -0.93580620366 -1.2428093839 +0.76 -1.0057293808 -1.218872661 +0.77 -1.0704039576 -1.1959876322 +0.78 -1.1297349296 -1.1741739926 +0.79 -1.1836476649 -1.1534446391 +0.8 -1.23208785 -1.1338057713 +0.81 -1.2750213562 -1.1152570226 +0.82 -1.3124340302 -1.0977916176 +0.83 -1.34433141 -1.0813965584 +0.84 -1.3707383679 -1.0660528348 +0.85 -1.3916986843 -1.0517356601 +0.86 -1.4072745528 -1.0384147296 +0.87 -1.4175460222 -1.0260545009 +0.88 -1.422610376 -1.0146144944 +0.89 -1.4225814539 -1.0040496119 +0.9 -1.4175889193 -0.99431047237 +0.91 -1.4077774754 -0.98534376271 +0.92 -1.393306035 -0.97709260167 +0.93 -1.3743468472 -0.96949691479 +0.94 -1.3510845875 -0.96249381864 +0.95 -1.3237154565 -0.95601798287 +0.96 -1.2924462012 -0.9500020467 +0.97 -1.2574929566 -0.94437710703 +0.98 -1.2190803772 -0.93907301456 +0.99 -1.1774405576 -0.9340188113 +1.0 -1.1328119871 -0.92914312957 +1.01 -1.0854385025 -0.92437458501 +1.02 -1.0355685243 -0.91964203242 +1.03 -0.98345355948 -0.9148751682 +1.04 -0.92934684605 -0.91000502329 +1.05 -0.87350300645 -0.90496400678 +1.06 -0.816176798 -0.89968636628 +1.07 -0.75762253462 -0.89410834934 +1.08 -0.6980927773 -0.88816867824 +1.09 -0.63783592919 -0.88180942862 +1.1 -0.57709733472 -0.87497547403 +1.11 -0.51611779897 -0.86761500666 +1.12 -0.45513452401 -0.85967911403 +1.13 -0.39437481773 -0.8511240992 +1.14 -0.33406039561 -0.84190970478 +1.15 -0.27440564306 -0.83199970915 +1.16 -0.2156182574 -0.82136164295 +1.17 -0.15789253305 -0.80996907161 +1.18 -0.10141514283 -0.79779940144 +1.19 -0.046364491037 -0.78483408344 +1.2 0.0070936861676 -0.77106002408 +1.21 0.058806100624 -0.75646931936 +1.22 0.10862841944 -0.74105803457 +1.23 0.15642703646 -0.72482675268 +1.24 0.20208507927 -0.70778222023 +1.25 0.24549451563 -0.68993480428 +1.26 0.28655709783 -0.67129881979 +1.27 0.32519346013 -0.65189485473 +1.28 0.3613324373 -0.63174663121 +1.29 0.39491300311 -0.61088161585 +1.3 0.42589280427 -0.58933283509 +1.31 0.45423664439 -0.56713591443 +1.32 0.47992110013 -0.54433022226 +1.33 0.50293883528 -0.52095932209 +1.34 0.52328858456 -0.49706895242 +1.35 0.54098326189 -0.47270841667 +1.36 0.55604726384 -0.44792957482 +1.37 0.56851053426 -0.42278632121 +1.38 0.5784190736 -0.3973351547 +1.39 0.58582409631 -0.37163397592 +1.4 0.59078619814 -0.34574254453 +1.41 0.59337646951 -0.3197213658 +1.42 0.59367015608 -0.29363258185 +1.43 0.59175335557 -0.26753813652 +1.44 0.58771624712 -0.24150074456 +1.45 0.58165595284 -0.21558336285 +1.46 0.57367486997 -0.18984748079 +1.47 0.56387989108 -0.16435632354 +1.48 0.55238206952 -0.13916913159 +1.49 0.53929613871 -0.11434699408 +1.5 0.52473924056 -0.089947318262 +1.51 0.50883077179 -0.066027001308 +1.52 0.49169241327 -0.042641156566 +1.53 0.47344482644 -0.019841140122 +1.54 0.45421261948 0.0023209806635 +1.55 0.43411462436 0.023800386214 +1.56 0.41327662113 0.044548868159 +1.57 0.39181369387 0.064528338728 +1.58 0.36984863239 0.083696839073 +1.59 0.34749294808 0.10202192594 +1.6 0.32486335081 0.11946916202 +1.61 0.30206634054 0.13601212183 +1.62 0.27921056742 0.15162430019 +1.63 0.25639594087 0.16628574014 +1.64 0.23372158101 0.17997799414 +1.65 0.21127927552 0.19268795322 +1.66 0.18915759431 0.20440513617 +1.67 0.16743876844 0.21512355823 +1.68 0.1462002202 0.22484043033 +1.69 0.12551333682 0.23355703571 +1.7 0.10544417695 0.24127785752 +1.71 0.08605219679 0.24801143198 +1.72 0.067392201978 0.25376898991 +1.73 0.049510885271 0.25856612142 +1.74 0.032452166492 0.26242022156 +1.75 0.016249456911 0.26535352678 +1.76 0.0009363277765 0.26738900334 +1.77 -0.01346722955 0.26855489619 +1.78 -0.026937386187 0.26887918915 +1.79 -0.039467262404 0.26839525362 +1.8 -0.051042203966 0.26713561709 +1.81 -0.061667545688 0.26513764513 +1.82 -0.071337628583 0.26243806272 +1.83 -0.08006745591 0.25907689464 +1.84 -0.087863054815 0.25509471266 +1.85 -0.094744724104 0.25053311675 +1.86 -0.10073011925 0.24543531389 +1.87 -0.10584404869 0.23984427835 +1.88 -0.11011403507 0.23380415827 +1.89 -0.11356907255 0.22735934704 +1.9 -0.11624417545 0.22055306109 +1.91 -0.11817265478 0.21343130087 +1.92 -0.11939406617 0.20603495539 +1.93 -0.11994678428 0.1984103598 +1.94 -0.1198720834 0.19059676551 +1.95 -0.11921240203 0.18263758346 +1.96 -0.11801032113 0.17457234326 +1.97 -0.1163093863 0.1664389658 +1.98 -0.11415448856 0.15827800515 +1.99 -0.11158739315 0.15012088135 +2.0 -0.10865483203 0.14200718854 +2.01 -0.10539681541 0.13396470695 +2.02 -0.10185745506 0.12602623509 +2.03 -0.098078174326 0.11822061558 +2.04 -0.094096518357 0.11057099092 +2.05 -0.089956724139 0.10310784468 +2.06 -0.085689851344 0.095846617351 +2.07 -0.081335739687 0.088811521611 +2.08 -0.07692635626 0.082018660843 +2.09 -0.072491428123 0.075481468962 +2.1 -0.068066323213 0.069218277483 +2.11 -0.063671603987 0.063233949849 +2.12 -0.059337660545 0.057541412683 +2.13 -0.055085519235 0.052145715315 +2.14 -0.050933564418 0.047049428304 +2.15 -0.046906930776 0.042260020905 +2.16 -0.043014644681 0.037772793136 +2.17 -0.039274785396 0.03358922043 +2.18 -0.035699058908 0.029706278232 +2.19 -0.032293597937 0.026116974923 +2.2 -0.029072509077 0.022818973327 +2.21 -0.026036085371 0.019801644234 +2.22 -0.023189532297 0.017056816878 +2.23 -0.020537674284 0.014575946706 +2.24 -0.018077146728 0.012346366913 +2.25 -0.015810412032 0.010357603026 +2.26 -0.01373392254 0.0085968309318 +2.27 -0.011842116593 0.0070502736984 +2.28 -0.010133845747 0.0057048888874 +2.29 -0.0085993951615 0.0045459798677 +2.3 -0.0072316775018 0.0035587606278 +2.31 -0.0060239032063 0.0027280843754 +2.32 -0.0049651302707 0.0020399454291 +2.33 -0.0040460304292 0.0014785514847 +2.34 -0.0032566099898 0.0010293505502 +2.35 -0.0025860125292 0.00067967964524 +2.36 -0.002022999236 0.00041341795836 +2.37 -0.0015572677849 0.00021897250312 +2.38 -0.0011785405003 8.5467448311e-05 +2.39 -0.00087447260941 -2.6406389151e-06 +2.4 -0.00063670211965 -5.2354589398e-05 +2.41 -0.00045599180494 -7.2310661646e-05 +2.42 -0.00031988217501 -7.6466931673e-05 +2.43 -0.00022336127653 -6.6408931605e-05 +2.44 -0.00015833856538 -4.8596801665e-05 +2.45 -0.00011438780266 -3.2809722084e-05 +2.46 -8.9293750204e-05 -1.6747173877e-05 +2.47 -7.7560959826e-05 -3.3623200012e-06 +2.48 -7.3416025735e-05 3.0092130857e-06 +2.49 -7.2127889582e-05 6.165594886e-06 +2.5 -6.5365569508e-05 6.6445226696e-06 +2.51 -4.1806181002e-05 4.9128311678e-06 +2.52 -1.7125758779e-05 2.2069704328e-06 +2.53 1.4939152703e-06 -1.4573388792e-07 +2.54 4.6480499279e-06 -4.5342490346e-07 +2.55 3.0647699647e-06 -2.98973343e-07 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 2 +0.0 0.0 0.0 +0.01 -0.0016302809637 0.00035794683369 +0.02 -0.0065163515418 0.0014296822768 +0.03 -0.014643908542 0.0032089002835 +0.04 -0.025989159095 0.005685121165 +0.05 -0.040518889195 0.0088437370407 +0.06 -0.058190559384 0.012666075205 +0.07 -0.07895242731 0.017129479149 +0.08 -0.10274369681 0.022207406906 +0.09 -0.12949469307 0.027869546304 +0.1 -0.15912706346 0.034081946665 +0.11 -0.1915540033 0.040807166393 +0.12 -0.22668050609 0.048004435845 +0.13 -0.26440363738 0.055629834819 +0.14 -0.30461283157 0.06363648392 +0.15 -0.34719021073 0.07197474901 +0.16 -0.39201092451 0.08059245791 +0.17 -0.43894351037 0.089435128454 +0.18 -0.48785027272 0.098446206939 +0.19 -0.53858768033 0.10756731604 +0.2 -0.59100678041 0.11673851104 +0.21 -0.64495362865 0.12589854357 +0.22 -0.70026973351 0.13498513144 +0.23 -0.75679251382 0.14393523369 +0.24 -0.81435576809 0.15268532967 +0.25 -0.87279015472 0.16117170107 +0.26 -0.93192368055 0.16933071533 +0.27 -0.99158219813 0.17709911015 +0.28 -1.0515899082 0.18441427678 +0.29 -1.1117698675 0.19121454188 +0.3 -1.1719445 0.19743944636 +0.31 -1.2319361091 0.20303002007 +0.32 -1.2915673908 0.20792905119 +0.33 -1.3506619462 0.21208134961 +0.34 -1.4090447891 0.21543400246 +0.35 -1.4665428517 0.21793662148 +0.36 -1.522985483 0.21954158083 +0.37 -1.5782049408 0.22020424436 +0.38 -1.632036874 0.21988318183 +0.39 -1.6843207957 0.21854037236 +0.4 -1.7349005444 0.21614139504 +0.41 -1.7836247317 0.21265560581 +0.42 -1.8303471759 0.20805629957 +0.43 -1.8749273204 0.20232085696 +0.44 -1.9172306348 0.19543087537 +0.45 -1.9571289972 0.18737228343 +0.46 -1.9945010581 0.17813543873 +0.47 -2.0292325838 0.16771520866 +0.48 -2.0612167791 0.15611103466 +0.49 -2.090354582 0.14332697147 +0.5 -2.1165549426 0.12937171885 +0.51 -2.1397350778 0.11425863393 +0.52 -2.1598206914 0.098005713701 +0.53 -2.176746171 0.080635565211 +0.54 -2.190454786 0.062175383691 +0.55 -2.2008987911 0.04265684497 +0.56 -2.2080395975 0.022116075258 +0.57 -2.211847815 0.00059350427024 +0.58 -2.2123033659 -0.02186620296 +0.59 -2.2093954581 -0.045214335243 +0.6 -2.203122684 -0.06939820269 +0.61 -2.1934928744 -0.09436140661 +0.62 -2.1805231747 -0.12004391211 +0.63 -2.1642398919 -0.14638230327 +0.64 -2.1446783791 -0.17330999434 +0.65 -2.1218829812 -0.20075738758 +0.66 -2.0959068035 -0.22865215777 +0.67 -2.066811508 -0.25691950291 +0.68 -2.0346671658 -0.28548234681 +0.69 -1.999552031 -0.31426159213 +0.7 -1.961552178 -0.34317645587 +0.71 -1.9207612656 -0.37214470648 +0.72 -1.8772802547 -0.40108292621 +0.73 -1.8312170584 -0.42990680852 +0.74 -1.7826861934 -0.45853144401 +0.75 -1.7318084432 -0.48687159166 +0.76 -1.6787103295 -0.51484205255 +0.77 -1.6235238527 -0.542357874 +0.78 -1.5663860216 -0.56933467063 +0.79 -1.5074384166 -0.59568891442 +0.8 -1.4468267401 -0.62133822183 +0.81 -1.3847003559 -0.64620163671 +0.82 -1.3212118192 -0.6701999082 +0.83 -1.2565163983 -0.69325576249 +0.84 -1.1907715897 -0.71529416746 +0.85 -1.1241366287 -0.73624258926 +0.86 -1.0567719967 -0.75603123994 +0.87 -0.98883892656 -0.77459331519 +0.88 -0.92049890866 -0.79186522139 +0.89 -0.85191319821 -0.80778679106 +0.9 -0.78324232642 -0.82230148608 +0.91 -0.71464561672 -0.83535658783 +0.92 -0.64628070784 -0.84690337375 +0.93 -0.57830308546 -0.85689727953 +0.94 -0.51086562408 -0.86529804664 +0.95 -0.4441181986 -0.8720698411 +0.96 -0.37820717285 -0.87718139491 +0.97 -0.31327485984 -0.88060612622 +0.98 -0.24945933646 -0.88232217166 +0.99 -0.18689397376 -0.88231247569 +1.0 -0.12570707039 -0.88056484165 +1.01 -0.066021505913 -0.87707196654 +1.02 -0.0079545268368 -0.871831461 +1.03 0.048382799934 -0.86484587151 +1.04 0.10288589917 -0.85612263048 +1.05 0.15545667375 -0.84567402721 +1.06 0.20600385579 -0.83351716716 +1.07 0.25444313475 -0.81967394231 +1.08 0.30069748155 -0.80417095395 +1.09 0.34469751688 -0.78703927055 +1.1 0.38638120612 -0.76831446513 +1.11 0.42569413044 -0.74803646205 +1.12 0.46258944515 -0.72624964296 +1.13 0.49702846809 -0.70300202094 +1.14 0.52898007647 -0.67834564529 +1.15 0.5584209087 -0.65233634601 +1.16 0.58533542208 -0.62503380899 +1.17 0.60971583811 -0.59650035482 +1.18 0.63156188005 -0.56680171908 +1.19 0.65088091394 -0.5360069337 +1.2 0.66768769801 -0.5041873814 +1.21 0.68200399062 -0.47141655935 +1.22 0.69385878215 -0.43777069314 +1.23 0.70328821897 -0.40332830942 +1.24 0.71033429279 -0.3681684753 +1.25 0.71504576268 -0.33237249494 +1.26 0.71747825821 -0.29602370728 +1.27 0.71769167806 -0.25920453432 +1.28 0.7157522691 -0.22199923787 +1.29 0.71173247372 -0.18449341844 +1.3 0.70570736789 -0.14677065047 +1.31 0.69775812741 -0.10891610311 +1.32 0.68797057633 -0.071014918121 +1.33 0.67643200525 -0.033149740959 +1.34 0.66323581364 0.0045952849381 +1.35 0.64847703464 0.042138299609 +1.36 0.63225221039 0.079399428431 +1.37 0.61466413529 0.1162974046 +1.38 0.59581231224 0.15275639672 +1.39 0.57580091001 0.18870011769 +1.4 0.55473560956 0.22405425572 +1.41 0.53271899406 0.25874913705 +1.42 0.50985999692 0.29271380992 +1.43 0.48626100505 0.32588396457 +1.44 0.462028045 0.35819554496 +1.45 0.43726595407 0.3895878196 +1.46 0.41207481124 0.42000499326 +1.47 0.38656057126 0.44939063297 +1.48 0.3608175272 0.47769672342 +1.49 0.33494686826 0.5048740466 +1.5 0.30904061714 0.53088029232 +1.51 0.28319158621 0.55567492895 +1.52 0.25748919177 0.57922141737 +1.53 0.2320165748 0.60148809531 +1.54 0.20685930393 0.62244468231 +1.55 0.18209023524 0.64206822004 +1.56 0.15778974576 0.66033516372 +1.57 0.13402102121 0.67723064065 +1.58 0.11085528354 0.69273914034 +1.59 0.088348538575 0.70685266968 +1.6 0.066561188568 0.71956401527 +1.61 0.045541575888 0.73087203775 +1.62 0.025339115934 0.74077747805 +1.63 0.0059938468033 0.74928603386 +1.64 -0.012455641987 0.75640594374 +1.65 -0.02997814 0.762149628 +1.66 -0.046545525399 0.76653231439 +1.67 -0.062135771965 0.76957291518 +1.68 -0.07673072274 0.77129311635 +1.69 -0.090317688661 0.77171796373 +1.7 -0.10288787787 0.77087507287 +1.71 -0.11443785977 0.76879513028 +1.72 -0.12496755668 0.76551117705 +1.73 -0.13448246389 0.76105890947 +1.74 -0.14299056152 0.75547633432 +1.75 -0.15050566682 0.74880338946 +1.76 -0.15704320029 0.74108264673 +1.77 -0.162624416 0.73235738719 +1.78 -0.167271634 0.72267441308 +1.79 -0.17101238302 0.71207933872 +1.8 -0.17387544024 0.70062292022 +1.81 -0.17589320453 0.68835205878 +1.82 -0.17710054196 0.67532110015 +1.83 -0.17753340761 0.66157787454 +1.84 -0.17723165939 0.64717812643 +1.85 -0.17623431712 0.63217172116 +1.86 -0.17458444823 0.61661402113 +1.87 -0.17232411255 0.60055706063 +1.88 -0.16949748901 0.58405433927 +1.89 -0.16614984949 0.56716024762 +1.9 -0.16232429048 0.54992499374 +1.91 -0.15806950473 0.53240543237 +1.92 -0.15342617165 0.51464805908 +1.93 -0.14844426834 0.49670968764 +1.94 -0.14316373324 0.47863572096 +1.95 -0.13763122805 0.46047840766 +1.96 -0.13188874746 0.44228507248 +1.97 -0.12597657543 0.42410101919 +1.98 -0.1199394249 0.40597590216 +1.99 -0.11381039894 0.38794793686 +2.0 -0.10763397714 0.37006616432 +2.01 -0.10144010235 0.35236584341 +2.02 -0.095265583264 0.33488859255 +2.03 -0.089142973563 0.31767206088 +2.04 -0.083098195882 0.30074770623 +2.05 -0.077166630832 0.28415568454 +2.06 -0.071365978984 0.26791978875 +2.07 -0.065724731008 0.25207376486 +2.08 -0.060262269553 0.23664279998 +2.09 -0.054995055424 0.22164931787 +2.1 -0.049945501387 0.2071213062 +2.11 -0.045120802665 0.19307206246 +2.12 -0.040537413162 0.17952387706 +2.13 -0.036203128407 0.16649020909 +2.14 -0.032122968947 0.15398166035 +2.15 -0.028307414412 0.1420144928 +2.16 -0.024753587768 0.13059030089 +2.17 -0.02146523645 0.11971812052 +2.18 -0.018441211133 0.10940089432 +2.19 -0.015675932017 0.099636041806 +2.2 -0.013168594079 0.090427492502 +2.21 -0.010909475495 0.081766015473 +2.22 -0.0088911546145 0.073646095504 +2.23 -0.00710559859 0.06606113076 +2.24 -0.0055403251186 0.058996127683 +2.25 -0.0041843627321 0.052439195792 +2.26 -0.00302493571 0.046375295857 +2.27 -0.002048258537 0.040787704768 +2.28 -0.0012396425611 0.03565823697 +2.29 -0.00058545857417 0.030970770784 +2.3 -7.0524374943e-05 0.026706982272 +2.31 0.00032128610825 0.022847262124 +2.32 0.00060267490947 0.019374509348 +2.33 0.00079004202243 0.016268357268 +2.34 0.00089784358158 0.013509078375 +2.35 0.00093745353246 0.011078294802 +2.36 0.00092553347338 0.0089539544753 +2.37 0.00087252062706 0.0071167327738 +2.38 0.00078791337712 0.0055471546788 +2.39 0.00068748885458 0.0042218314666 +2.4 0.00057611638219 0.0031221978739 +2.41 0.0004608042569 0.0022279088413 +2.42 0.00035518992628 0.0015146245011 +2.43 0.00025785924786 0.00096472501905 +2.44 0.00017329303283 0.00055694774566 +2.45 0.00010989139907 0.00026733139288 +2.46 6.1872491483e-05 7.829302357e-05 +2.47 2.953833815e-05 -3.334821152e-05 +2.48 1.4806544348e-05 -9.5806441567e-05 +2.49 9.6783992659e-06 -0.00011221658675 +2.5 8.5136479095e-06 -9.7376625643e-05 +2.51 4.2505513471e-06 -6.772287195e-05 +2.52 1.390094449e-06 -2.9382005001e-05 +2.53 -2.082365022e-07 2.20387025e-06 +2.54 -6.504514312e-07 6.8840503135e-06 +2.55 -4.3057462874e-07 4.5569849889e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -7.0231214206E+00 +0.01 -7.0453717502E+00 +0.02 -7.1143468216E+00 +0.03 -7.2090176842E+00 +0.04 -7.3083553876E+00 +0.05 -7.3952429292E+00 +0.06 -7.4597953536E+00 +0.07 -7.4991297585E+00 +0.08 -7.5152902525E+00 +0.09 -7.5128744992E+00 +0.1 -7.4971693665E+00 +0.11 -7.4730064339E+00 +0.12 -7.4442385526E+00 +0.13 -7.4136363893E+00 +0.14 -7.3830123504E+00 +0.15 -7.3534314963E+00 +0.16 -7.3254268122E+00 +0.17 -7.2991803384E+00 +0.18 -7.2746597190E+00 +0.19 -7.2517133907E+00 +0.2 -7.2301330081E+00 +0.21 -7.2096920714E+00 +0.22 -7.1901684172E+00 +0.23 -7.1713562496E+00 +0.24 -7.1530717169E+00 +0.25 -7.1351547228E+00 +0.26 -7.1174685568E+00 +0.27 -7.0998985249E+00 +0.28 -7.0823500273E+00 +0.29 -7.0647465397E+00 +0.3 -7.0470276188E+00 +0.31 -7.0291470239E+00 +0.32 -7.0110709744E+00 +0.33 -6.9927765879E+00 +0.34 -6.9742504049E+00 +0.35 -6.9554870639E+00 +0.36 -6.9364880834E+00 +0.37 -6.9172607312E+00 +0.38 -6.8978169707E+00 +0.39 -6.8781725207E+00 +0.4 -6.8583459681E+00 +0.41 -6.8383579698E+00 +0.42 -6.8182305369E+00 +0.43 -6.7979863986E+00 +0.44 -6.7776484445E+00 +0.45 -6.7572392483E+00 +0.46 -6.7367806674E+00 +0.47 -6.7162935188E+00 +0.48 -6.6957973262E+00 +0.49 -6.6753101150E+00 +0.5 -6.6548482902E+00 +0.51 -6.6344265604E+00 +0.52 -6.6140578820E+00 +0.53 -6.5937534575E+00 +0.54 -6.5735227996E+00 +0.55 -6.5533737172E+00 +0.56 -6.5333124572E+00 +0.57 -6.5133437238E+00 +0.58 -6.4934708208E+00 +0.59 -6.4736956904E+00 +0.6 -6.4540190739E+00 +0.61 -6.4344405251E+00 +0.62 -6.4149585733E+00 +0.63 -6.3955707664E+00 +0.64 -6.3762737537E+00 +0.65 -6.3570633874E+00 +0.66 -6.3379347681E+00 +0.67 -6.3188823126E+00 +0.68 -6.2998998303E+00 +0.69 -6.2809805806E+00 +0.7 -6.2621173131E+00 +0.71 -6.2433023331E+00 +0.72 -6.2245275562E+00 +0.73 -6.2057845580E+00 +0.74 -6.1870646259E+00 +0.75 -6.1683588117E+00 +0.76 -6.1496579790E+00 +0.77 -6.1309528635E+00 +0.78 -6.1122341237E+00 +0.79 -6.0934923948E+00 +0.8 -6.0747183433E+00 +0.81 -6.0559027212E+00 +0.82 -6.0370364200E+00 +0.83 -6.0181105239E+00 +0.84 -5.9991163614E+00 +0.85 -5.9800455551E+00 +0.86 -5.9608900694E+00 +0.87 -5.9416422543E+00 +0.88 -5.9222948867E+00 +0.89 -5.9028412073E+00 +0.9 -5.8832749527E+00 +0.91 -5.8635903839E+00 +0.92 -5.8437823081E+00 +0.93 -5.8238460972E+00 +0.94 -5.8037776995E+00 +0.95 -5.7835736400E+00 +0.96 -5.7632310315E+00 +0.97 -5.7427475878E+00 +0.98 -5.7221215886E+00 +0.99 -5.7013518781E+00 +1.0 -5.6804378489E+00 +1.01 -5.6593794226E+00 +1.02 -5.6381770133E+00 +1.03 -5.6168315332E+00 +1.04 -5.5953443695E+00 +1.05 -5.5737173207E+00 +1.06 -5.5519525829E+00 +1.07 -5.5300527090E+00 +1.08 -5.5080205953E+00 +1.09 -5.4858594768E+00 +1.1 -5.4635728450E+00 +1.11 -5.4411644391E+00 +1.12 -5.4186381993E+00 +1.13 -5.3959983139E+00 +1.14 -5.3732491142E+00 +1.15 -5.3503950752E+00 +1.16 -5.3274407901E+00 +1.17 -5.3043909888E+00 +1.18 -5.2812504690E+00 +1.19 -5.2580240944E+00 +1.2 -5.2347167970E+00 +1.21 -5.2113335516E+00 +1.22 -5.1878793578E+00 +1.23 -5.1643592404E+00 +1.24 -5.1407782392E+00 +1.25 -5.1171413949E+00 +1.26 -5.0934537514E+00 +1.27 -5.0697203419E+00 +1.28 -5.0459461891E+00 +1.29 -5.0221363068E+00 +1.3 -4.9982956801E+00 +1.31 -4.9744292806E+00 +1.32 -4.9505420602E+00 +1.33 -4.9266389339E+00 +1.34 -4.9027248049E+00 +1.35 -4.8788045416E+00 +1.36 -4.8548829774E+00 +1.37 -4.8309649370E+00 +1.38 -4.8070551869E+00 +1.39 -4.7831584769E+00 +1.4 -4.7592795265E+00 +1.41 -4.7354229970E+00 +1.42 -4.7115935473E+00 +1.43 -4.6877957587E+00 +1.44 -4.6640341935E+00 +1.45 -4.6403133701E+00 +1.46 -4.6166377312E+00 +1.47 -4.5930117259E+00 +1.48 -4.5694396733E+00 +1.49 -4.5459259073E+00 +1.5 -4.5224746470E+00 +1.51 -4.4990900739E+00 +1.52 -4.4757763045E+00 +1.53 -4.4525373437E+00 +1.54 -4.4293771944E+00 +1.55 -4.4062996642E+00 +1.56 -4.3833086284E+00 +1.57 -4.3604076923E+00 +1.58 -4.3376005463E+00 +1.59 -4.3148906194E+00 +1.6 -4.2922813822E+00 +1.61 -4.2697760768E+00 +1.62 -4.2473779371E+00 +1.63 -4.2250900034E+00 +1.64 -4.2029152612E+00 +1.65 -4.1808565335E+00 +1.66 -4.1589165503E+00 +1.67 -4.1370979017E+00 +1.68 -4.1154030601E+00 +1.69 -4.0938343674E+00 +1.7 -4.0723940409E+00 +1.71 -4.0510841603E+00 +1.72 -4.0299066951E+00 +1.73 -4.0088634503E+00 +1.74 -3.9879561588E+00 +1.75 -3.9671863413E+00 +1.76 -3.9465555084E+00 +1.77 -3.9260648902E+00 +1.78 -3.9057157915E+00 +1.79 -3.8855091504E+00 +1.8 -3.8654460818E+00 +1.81 -3.8455272395E+00 +1.82 -3.8257535560E+00 +1.83 -3.8061254478E+00 +1.84 -3.7866435899E+00 +1.85 -3.7673082659E+00 +1.86 -3.7481198716E+00 +1.87 -3.7290785831E+00 +1.88 -3.7101845281E+00 +1.89 -3.6914377938E+00 +1.9 -3.6728382611E+00 +1.91 -3.6543859374E+00 +1.92 -3.6360805050E+00 +1.93 -3.6179218406E+00 +1.94 -3.5999095315E+00 +1.95 -3.5820432604E+00 +1.96 -3.5643225918E+00 +1.97 -3.5467470213E+00 +1.98 -3.5293160912E+00 +1.99 -3.5120291557E+00 +2.0 -3.4948856938E+00 +2.01 -3.4778850017E+00 +2.02 -3.4610264332E+00 +2.03 -3.4443092916E+00 +2.04 -3.4277328079E+00 +2.05 -3.4112962860E+00 +2.06 -3.3949988785E+00 +2.07 -3.3788398185E+00 +2.08 -3.3628182624E+00 +2.09 -3.3469333416E+00 +2.1 -3.3311842302E+00 +2.11 -3.3155699746E+00 +2.12 -3.3000897024E+00 +2.13 -3.2847424519E+00 +2.14 -3.2695272359E+00 +2.15 -3.2544431327E+00 +2.16 -3.2394890382E+00 +2.17 -3.2246639597E+00 +2.18 -3.2099668049E+00 +2.19 -3.1953963961E+00 +2.2 -3.1809517062E+00 +2.21 -3.1666314047E+00 +2.22 -3.1524342799E+00 +2.23 -3.1383591039E+00 +2.24 -3.1244044055E+00 +2.25 -3.1105688087E+00 +2.26 -3.0968509426E+00 +2.27 -3.0832494382E+00 +2.28 -3.0697627913E+00 +2.29 -3.0563898779E+00 +2.3 -3.0431294386E+00 +2.31 -3.0299801887E+00 +2.32 -3.0169410939E+00 +2.33 -3.0040109831E+00 +2.34 -2.9911887500E+00 +2.35 -2.9784733778E+00 +2.36 -2.9658637789E+00 +2.37 -2.9533589232E+00 +2.38 -2.9409577917E+00 +2.39 -2.9286593571E+00 +2.4 -2.9164625797E+00 +2.41 -2.9043664216E+00 +2.42 -2.8923698868E+00 +2.43 -2.8804718654E+00 +2.44 -2.8686712935E+00 +2.45 -2.8569671495E+00 +2.46 -2.8453582499E+00 +2.47 -2.8338434784E+00 +2.48 -2.8224217322E+00 +2.49 -2.8110918278E+00 +2.5 -2.7998525918E+00 +2.51 -2.7887028310E+00 +2.52 -2.7776415409E+00 +2.53 -2.7666676118E+00 +2.54 -2.7557799085E+00 +2.55 -2.7449775402E+00 +2.56 -2.7342595084E+00 +2.57 -2.7236248015E+00 +2.58 -2.7130724883E+00 +2.59 -2.7026015965E+00 +2.6 -2.6922111078E+00 +2.61 -2.6819001800E+00 +2.62 -2.6716679021E+00 +2.63 -2.6615133021E+00 +2.64 -2.6514355718E+00 +2.65 -2.6414338420E+00 +2.66 -2.6315071900E+00 +2.67 -2.6216548251E+00 +2.68 -2.6118759290E+00 +2.69 -2.6021696307E+00 +2.7 -2.5925351489E+00 +2.71 -2.5829717156E+00 +2.72 -2.5734785124E+00 +2.73 -2.5640547609E+00 +2.74 -2.5546997426E+00 +2.75 -2.5454126928E+00 +2.76 -2.5361928300E+00 +2.77 -2.5270394846E+00 +2.78 -2.5179519390E+00 +2.79 -2.5089294248E+00 +2.8 -2.4999712991E+00 +2.81 -2.4910768846E+00 +2.82 -2.4822454645E+00 +2.83 -2.4734763795E+00 +2.84 -2.4647690035E+00 +2.85 -2.4561226740E+00 +2.86 -2.4475367100E+00 +2.87 -2.4390105350E+00 +2.88 -2.4305435276E+00 +2.89 -2.4221350247E+00 +2.9 -2.4137844549E+00 +2.91 -2.4054912392E+00 +2.92 -2.3972547682E+00 +2.93 -2.3890744373E+00 +2.94 -2.3809497189E+00 +2.95 -2.3728800481E+00 +2.96 -2.3648648185E+00 +2.97 -2.3569035173E+00 +2.98 -2.3489956165E+00 +2.99 -2.3411405637E+00 +3.0 -2.3333378034E+00 +3.01 -2.3255868594E+00 +3.02 -2.3178872183E+00 +3.03 -2.3102383329E+00 +3.04 -2.3026397232E+00 +3.05 -2.2950909160E+00 +3.06 -2.2875914196E+00 +3.07 -2.2801407030E+00 +3.08 -2.2727383444E+00 +3.09 -2.2653838770E+00 +3.1 -2.2580768156E+00 +3.11 -2.2508166903E+00 +3.12 -2.2436030858E+00 +3.13 -2.2364355573E+00 +3.14 -2.2293136286E+00 +3.15 -2.2222368833E+00 +3.16 -2.2152049132E+00 +3.17 -2.2082172941E+00 +3.18 -2.2012735614E+00 +3.19 -2.1943733441E+00 +3.2 -2.1875162423E+00 +3.21 -2.1807018469E+00 +3.22 -2.1739297205E+00 +3.23 -2.1671995163E+00 +3.24 -2.1605108486E+00 +3.25 -2.1538633207E+00 +3.26 -2.1472565296E+00 +3.27 -2.1406901400E+00 +3.28 -2.1341637841E+00 +3.29 -2.1276770788E+00 +3.3 -2.1212296497E+00 +3.31 -2.1148211738E+00 +3.32 -2.1084513002E+00 +3.33 -2.1021196610E+00 +3.34 -2.0958259044E+00 +3.35 -2.0895697208E+00 +3.36 -2.0833507757E+00 +3.37 -2.0771687174E+00 +3.38 -2.0710232112E+00 +3.39 -2.0649139620E+00 +3.4 -2.0588406508E+00 +3.41 -2.0528029436E+00 +3.42 -2.0468005172E+00 +3.43 -2.0408330924E+00 +3.44 -2.0349003651E+00 +3.45 -2.0290020196E+00 +3.46 -2.0231377399E+00 +3.47 -2.0173072634E+00 +3.48 -2.0115103000E+00 +3.49 -2.0057465539E+00 +3.5 -2.0000157114E+00 +3.51 -1.9943175274E+00 +3.52 -1.9886517255E+00 +3.53 -1.9830180291E+00 +3.54 -1.9774161269E+00 +3.55 -1.9718457881E+00 +3.56 -1.9663067504E+00 +3.57 -1.9607987500E+00 +3.58 -1.9553214969E+00 +3.59 -1.9498747526E+00 +3.6 -1.9444582756E+00 +3.61 -1.9390718144E+00 +3.62 -1.9337151017E+00 +3.63 -1.9283878872E+00 +3.64 -1.9230899509E+00 +3.65 -1.9178210530E+00 +3.66 -1.9125809500E+00 +3.67 -1.9073693767E+00 +3.68 -1.9021861346E+00 +3.69 -1.8970309951E+00 +3.7 -1.8919037296E+00 +3.71 -1.8868040854E+00 +3.72 -1.8817318554E+00 +3.73 -1.8766868316E+00 +3.74 -1.8716687958E+00 +3.75 -1.8666775210E+00 +3.76 -1.8617127770E+00 +3.77 -1.8567743797E+00 +3.78 -1.8518621212E+00 +3.79 -1.8469757937E+00 +3.8 -1.8421151646E+00 +3.81 -1.8372800505E+00 +3.82 -1.8324702607E+00 +3.83 -1.8276855969E+00 +3.84 -1.8229258539E+00 +3.85 -1.8181908181E+00 +3.86 -1.8134803248E+00 +3.87 -1.8087941851E+00 +3.88 -1.8041322098E+00 +3.89 -1.7994941909E+00 +3.9 -1.7948799518E+00 +3.91 -1.7902893242E+00 +3.92 -1.7857221279E+00 +3.93 -1.7811781826E+00 +3.94 -1.7766572800E+00 +3.95 -1.7721592746E+00 +3.96 -1.7676839959E+00 +3.97 -1.7632312719E+00 +3.98 -1.7588009238E+00 +3.99 -1.7543927664E+00 +4.0 -1.7500066580E+00 +4.01 -1.7456424349E+00 +4.02 -1.7412999332E+00 +4.03 -1.7369789755E+00 +4.04 -1.7326793994E+00 +4.05 -1.7284010643E+00 +4.06 -1.7241438138E+00 +4.07 -1.7199074918E+00 +4.08 -1.7156919241E+00 +4.09 -1.7114969666E+00 +4.1 -1.7073224806E+00 +4.11 -1.7031683172E+00 +4.12 -1.6990343275E+00 +4.13 -1.6949203421E+00 +4.14 -1.6908262304E+00 +4.15 -1.6867518576E+00 +4.16 -1.6826970815E+00 +4.17 -1.6786617602E+00 +4.18 -1.6746457304E+00 +4.19 -1.6706488710E+00 +4.2 -1.6666710522E+00 +4.21 -1.6627121385E+00 +4.22 -1.6587719945E+00 +4.23 -1.6548504641E+00 +4.24 -1.6509474321E+00 +4.25 -1.6470627748E+00 +4.26 -1.6431963631E+00 +4.27 -1.6393480680E+00 +4.28 -1.6355177416E+00 +4.29 -1.6317052709E+00 +4.3 -1.6279105396E+00 +4.31 -1.6241334248E+00 +4.32 -1.6203738033E+00 +4.33 -1.6166315370E+00 +4.34 -1.6129065112E+00 +4.35 -1.6091986186E+00 +4.36 -1.6055077418E+00 +4.37 -1.6018337635E+00 +4.38 -1.5981765561E+00 +4.39 -1.5945360003E+00 +4.4 -1.5909119986E+00 +4.41 -1.5873044391E+00 +4.42 -1.5837132098E+00 +4.43 -1.5801381951E+00 +4.44 -1.5765792679E+00 +4.45 -1.5730363413E+00 +4.46 -1.5695093087E+00 +4.47 -1.5659980634E+00 +4.48 -1.5625024987E+00 +4.49 -1.5590224884E+00 +4.5 -1.5555579458E+00 +4.51 -1.5521087730E+00 +4.52 -1.5486748682E+00 +4.53 -1.5452561298E+00 +4.54 -1.5418524452E+00 +4.55 -1.5384637135E+00 +4.56 -1.5350898499E+00 +4.57 -1.5317307575E+00 +4.58 -1.5283863393E+00 +4.59 -1.5250564978E+00 +4.6 -1.5217411153E+00 +4.61 -1.5184401207E+00 +4.62 -1.5151534216E+00 +4.63 -1.5118809255E+00 +4.64 -1.5086225400E+00 +4.65 -1.5053781613E+00 +4.66 -1.5021476999E+00 +4.67 -1.4989310782E+00 +4.68 -1.4957282082E+00 +4.69 -1.4925390016E+00 +4.7 -1.4893633704E+00 +4.71 -1.4862012066E+00 +4.72 -1.4830524446E+00 +4.73 -1.4799170014E+00 +4.74 -1.4767947930E+00 +4.75 -1.4736857355E+00 +4.76 -1.4705897384E+00 +4.77 -1.4675067108E+00 +4.78 -1.4644365871E+00 +4.79 -1.4613792870E+00 +4.8 -1.4583347306E+00 +4.81 -1.4553028376E+00 +4.82 -1.4522835159E+00 +4.83 -1.4492766914E+00 +4.84 -1.4462822960E+00 +4.85 -1.4433002532E+00 +4.86 -1.4403304868E+00 +4.87 -1.4373729203E+00 +4.88 -1.4344274613E+00 +4.89 -1.4314940487E+00 +4.9 -1.4285726134E+00 +4.91 -1.4256630825E+00 +4.92 -1.4227653832E+00 +4.93 -1.4198794427E+00 +4.94 -1.4170051697E+00 +4.95 -1.4141425126E+00 +4.96 -1.4112914024E+00 +4.97 -1.4084517699E+00 +4.98 -1.4056235455E+00 +4.99 -1.4028066588E+00 +5.0 -1.4000010240E+00 +5.01 -1.3972065925E+00 +5.02 -1.3944232980E+00 +5.03 -1.3916510745E+00 +5.04 -1.3888898557E+00 +5.05 -1.3861395741E+00 +5.06 -1.3834001481E+00 +5.07 -1.3806715315E+00 +5.08 -1.3779536612E+00 +5.09 -1.3752464741E+00 +5.1 -1.3725499072E+00 +5.11 -1.3698638970E+00 +5.12 -1.3671883633E+00 +5.13 -1.3645232633E+00 +5.14 -1.3618685368E+00 +5.15 -1.3592241238E+00 +5.16 -1.3565899640E+00 +5.17 -1.3539659973E+00 +5.18 -1.3513521482E+00 +5.19 -1.3487483721E+00 +5.2 -1.3461546139E+00 +5.21 -1.3435708162E+00 +5.22 -1.3409969216E+00 +5.23 -1.3384328728E+00 +5.24 -1.3358786008E+00 +5.25 -1.3333340559E+00 +5.26 -1.3307991892E+00 +5.27 -1.3282739460E+00 +5.28 -1.3257582715E+00 +5.29 -1.3232521111E+00 +5.3 -1.3207554034E+00 +5.31 -1.3182680912E+00 +5.32 -1.3157901326E+00 +5.33 -1.3133214753E+00 +5.34 -1.3108620674E+00 +5.35 -1.3084118565E+00 +5.36 -1.3059707901E+00 +5.37 -1.3035388010E+00 +5.38 -1.3011158551E+00 +5.39 -1.2987019027E+00 +5.4 -1.2962968941E+00 +5.41 -1.2939007796E+00 +5.42 -1.2915135095E+00 +5.43 -1.2891350244E+00 +5.44 -1.2867652793E+00 +5.45 -1.2844042331E+00 +5.46 -1.2820518385E+00 +5.47 -1.2797080480E+00 +5.48 -1.2773728144E+00 +5.49 -1.2750460886E+00 +5.5 -1.2727278110E+00 +5.51 -1.2704179504E+00 +5.52 -1.2681164617E+00 +5.53 -1.2658232996E+00 +5.54 -1.2635384189E+00 +5.55 -1.2612617746E+00 +5.56 -1.2589933135E+00 +5.57 -1.2567329919E+00 +5.58 -1.2544807741E+00 +5.59 -1.2522366172E+00 +5.6 -1.2500004780E+00 +5.61 -1.2477723135E+00 +5.62 -1.2455520805E+00 +5.63 -1.2433397235E+00 +5.64 -1.2411352113E+00 +5.65 -1.2389385051E+00 +5.66 -1.2367495638E+00 +5.67 -1.2345683465E+00 +5.68 -1.2323948119E+00 +5.69 -1.2302289177E+00 +5.7 -1.2280706113E+00 +5.71 -1.2259198665E+00 +5.72 -1.2237766443E+00 +5.73 -1.2216409055E+00 +5.74 -1.2195126109E+00 +5.75 -1.2173917213E+00 +5.76 -1.2152781938E+00 +5.77 -1.2131719831E+00 +5.78 -1.2110730624E+00 +5.79 -1.2089813942E+00 +5.8 -1.2068969412E+00 +5.81 -1.2048196662E+00 +5.82 -1.2027495316E+00 +5.83 -1.2006864953E+00 +5.84 -1.1986305166E+00 +5.85 -1.1965815689E+00 +5.86 -1.1945396164E+00 +5.87 -1.1925046237E+00 +5.88 -1.1904765552E+00 +5.89 -1.1884553752E+00 +5.9 -1.1864410430E+00 +5.91 -1.1844335204E+00 +5.92 -1.1824327817E+00 +5.93 -1.1804387931E+00 +5.94 -1.1784515207E+00 +5.95 -1.1764709306E+00 +5.96 -1.1744969887E+00 +5.97 -1.1725296572E+00 +5.98 -1.1705688974E+00 +5.99 -1.1686146860E+00 +6.0 -1.1666669908E+00 +6.01 -1.1647257794E+00 diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index 7092f3d5..2b835a73 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -32,6 +32,7 @@ USE_LIBROM="On" LIBROM_PATH=${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM cd libROM +git checkout 321d18f4d5adfa29f0a3de9be2699fee9732f2bf #./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake ./scripts/compile.sh cd ${BUILD_DIR} diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index a0ca470a..474d61e5 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra -Wno-cast-function-type" \ .. # call make install diff --git a/src/Control.cc b/src/Control.cc index 83fe2093..043e5640 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -31,23 +31,6 @@ float Control::total_spin_ = 0.; std::string Control::run_directory_("."); bool Control::with_spin_ = false; -static void finishRead(std::ifstream& tfile) -{ - // while( tfile.get()!='\n'); - // string str; - // getline(tfile,str); - char str[256]; - tfile.getline(str, 256); - - char cc = (char)tfile.peek(); - while (cc == ('#') || (cc == '\n') || cc == ' ') - { - while (tfile.get() != '\n') - ; - cc = (char)tfile.peek(); // look at next character - } -} - Control::Control() { assert(comm_global_ != MPI_COMM_NULL); @@ -1839,6 +1822,14 @@ int Control::checkOptions() return -1; } + if (DM_solver_ > 0 && lap_type == 0) + { + std::cerr << "DM_solver_ = " << DM_solver_ << std::endl; + std::cerr << "ERROR: Mehrstellen not compatible with MVP inner solvers!" + << std::endl; + return -1; + } + if (it_algo_type_ == 3 && lap_type == 0) { std::cerr diff --git a/src/Control.h b/src/Control.h index 162a68cc..a15bd24c 100644 --- a/src/Control.h +++ b/src/Control.h @@ -378,6 +378,8 @@ class Control // 10 or larger means CG, otherwise MG V-cycles bool MGPoissonSolver() { return (diel_flag_ / 10 == 0); } + bool LangevinThermostat() { return (thermostat_type == 1); } + // // data // diff --git a/src/DistributedIonicData.cc b/src/DistributedIonicData.cc index 6652c41a..05399cac 100644 --- a/src/DistributedIonicData.cc +++ b/src/DistributedIonicData.cc @@ -12,7 +12,6 @@ #include "tools.h" #include -using namespace std; DistributedIonicData::DistributedIonicData( const std::vector& local_names, @@ -31,16 +30,16 @@ DistributedIonicData::DistributedIonicData( int DistributedIonicData::pack(char* cbuff, double* dbuff) { - vector::iterator itf = data_.begin(); - double* dptr = dbuff; + std::vector::iterator itf = data_.begin(); + double* dptr = dbuff; int idx = 0; - for (vector::iterator it = ion_names_.begin(); + for (std::vector::iterator it = ion_names_.begin(); it != ion_names_.end(); ++it) { - string s(*it); + std::string s(*it); FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; @@ -58,7 +57,7 @@ void DistributedIonicData::unpack(char*& cptr, double*& dptr, const short ndata) for (short i = 0; i < ndata; i++) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_names_.push_back(name); cptr += IonData_MaxStrLength; diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index ef658955..0fac518e 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -428,7 +428,7 @@ void HDFrestart::addReleaseNumber2File(const char* release) // if( onpe0 ) { HDF_FixedLengthString t; - strncpy(t.mystring, release, MyHDFStrLength); + strncpy(t.mystring, release, MyHDFStrLength - 1); herr_t status = H5Awrite(attribute_id, strtype, &t); if (status < 0) { @@ -457,7 +457,8 @@ HDFrestart::HDFrestart(const std::string& filename, const pb::PEenv& pes, verbosity_ = 0; closed_ = false; - //(*MPIdata::sout)<<"HDFrestart::HDFrestart(), filename="<& data) +int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - { - (*MPIdata::sout) << "HDFrestart::readAtomicNumbers()..." << std::endl; - } + (*MPIdata::sout) << "HDFrestart::readAtomicData()..." << std::endl; if (active_) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_numbers", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return 0; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_numbers", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -1; } int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- No numbers!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("No " + datasetname); return -1; } data.resize(dim); @@ -2017,174 +1909,52 @@ int HDFrestart::readAtomicNumbers(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -1; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dclose failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -1; } } // if active_ // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); - return 0; -} - -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicIDs()..." << std::endl; - - if (active_) + if (useHdf5p()) { - assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- No IDs!!!" << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } + data.erase(std::remove(data.begin(), data.end(), -1), data.end()); } - if (gather_data_x_) gatherDataXdir(data); return 0; } -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicNLprojIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs()..." << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (!exists) return -1; - - hid_t dataset_id = H5Dopen2(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs() --- " - "H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- No IDs!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - return 0; -} - -int HDFrestart::readAtomicPositions(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { if (onpe0) - (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + (*MPIdata::sout) << "Read atomic data from hdf5 file" << std::endl; if (active_) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Ionic_positions", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return -1; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Ionic_positions", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -2; } int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- No positions!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("readAtomicData() --- No " + datasetname); return -2; } data.resize(dim); @@ -2193,19 +1963,21 @@ int HDFrestart::readAtomicPositions(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -2; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -2; } } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); + } if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2218,59 +1990,12 @@ int HDFrestart::readOldCenterOnMesh(std::vector& data, int i) "points from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); - - std::stringstream datasetstream; - datasetstream << "OldCenterOnMesh_" << i; - - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } + std::stringstream datasetstream; + datasetstream << "OldCenterOnMesh_" << i; - if (gather_data_x_) gatherDataXdir(data); + std::string datasetname = datasetstream.str(); - return 0; + return readAtomicData(datasetname, data); } int HDFrestart::readOldCenter(std::vector& data, int i) @@ -2279,256 +2004,16 @@ int HDFrestart::readOldCenter(std::vector& data, int i) (*MPIdata::sout) << "Read old localization centers from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); + std::stringstream datasetstream; + datasetstream << "OldCenter_" << i; - std::stringstream datasetstream; - datasetstream << "OldCenter_" << i; + std::string datasetname = datasetstream.str(); - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readGidsList(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read list of gids from hdf5 file" << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - std::string datasetname = "GidsList"; - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(int); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- No GidsList!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readAtomicVelocities(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read atomic velocities from hdf5 file" - << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (exists) - { - - // Open an existing dataset - hid_t dataset_id - = H5Dopen2(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (dataset_id < 0) - { - std::cerr << "HDFrestart::readAtomicVelocities(), " - "H5Dopen failed->no velocities read" - << std::endl; - data.clear(); - return -1; - } - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, &data[0]); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dread failed!!!"); - return -2; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dclose failed!!!"); - return -2; - } - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readLockedAtomNames(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames()..." << std::endl; - - std::vector buffer; - short name_length = 7; // default, value used before February 2016 - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (!exists) return 0; - - hid_t dataset_id = H5Dopen2(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames(), " - "H5Dopen failed->no locked atoms read" - << std::endl; - return -1; - } - - std::string attname("String_Length"); - htri_t existsA = H5Aexists(dataset_id, attname.c_str()); - if (existsA) - { - hid_t attribute_id = H5Aopen_name(dataset_id, attname.c_str()); - herr_t status = H5Aread(attribute_id, H5T_NATIVE_INT, &name_length); - // check validity of data just read - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Aread failed!!!"); - return -1; - } - } - - int dim = (int)H5Dget_storage_size(dataset_id) / name_length; - - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), dataset size=" << dim - << std::endl; - - if (dim == 0) return 0; - - buffer.resize(dim * name_length); - - // create type for std::strings of length IonData_MaxStrLength - hid_t strtype = H5Tcopy(H5T_C_S1); - H5Tset_size(strtype, name_length); - herr_t status = H5Dread( - dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buffer[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), H5Dread failed!!!" - << std::endl; - return -1; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - - if (gather_data_x_) gatherDataXdir(buffer); - - data.clear(); - for (unsigned short i = 0; i < buffer.size(); i += name_length) - { - std::string t(&buffer[i], name_length); - assert(t.size() > 0); - - stripLeadingAndTrailingBlanks(t); - - assert(t.size() > 0); - data.push_back(t); - } - - return 0; + return readAtomicData(datasetname, data); } -int HDFrestart::readAtomicNames(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -2541,11 +2026,12 @@ int HDFrestart::readAtomicNames(std::vector& data) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_names", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (exists) { // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_names", H5P_DEFAULT); + hid_t dataset_id + = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { MGMOL_HDFRESTART_FAIL("H5Dopen2 failed!!!"); @@ -2577,7 +2063,7 @@ int HDFrestart::readAtomicNames(std::vector& data) int dim = (int)H5Dget_storage_size(dataset_id) / name_length; if (dim == 0) { - if (onpe0) MGMOL_HDFRESTART_FAIL("No names!!!"); + MGMOL_HDFRESTART_FAIL("No names!!!"); return -1; } @@ -2618,6 +2104,11 @@ int HDFrestart::readAtomicNames(std::vector& data) data.push_back(t); } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), ""), data.end()); + } + return 0; } @@ -2652,7 +2143,6 @@ int HDFrestart::readRestartRandomStates(std::vector& data) dim = (int)H5Dget_storage_size(dataset_id) / sizeof(unsigned short); } - if (dim > 0) { data.resize(dim); @@ -2672,12 +2162,10 @@ int HDFrestart::readRestartRandomStates(std::vector& data) } if (!data.empty()) - if (data[0] != data[0]) + if (std::isnan(data[0])) { MGMOL_HDFRESTART_FAIL( - "ERROR: HDFrestart::readRestartRandomStates() " - "--- data[0]=" - << data[0]); + "readRestartRandomStates() is NaN"); return -2; } } diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 72c5c1f5..1ed2d080 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -260,19 +260,12 @@ class HDFrestart template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); - // int writeRandomState(unsigned short int rand_state[3]); - // int readRandomState(unsigned short* rand_state); - int readAtomicIDs(std::vector& data); - int readAtomicNLprojIDs(std::vector& data); - int readAtomicNumbers(std::vector& data); - int readAtomicNames(std::vector& data); - int readAtomicPositions(std::vector& data); - int readAtomicVelocities(std::vector& data); - int readLockedAtomNames(std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); int readRestartRandomStates(std::vector& data); int readOldCenter(std::vector& data, int i); int readOldCenterOnMesh(std::vector& data, int i); - int readGidsList(std::vector& data); void addDateToFilename(); void addMDTime2File(const float run_time); diff --git a/src/HamiltonianMVP_DMStrategy.cc b/src/HamiltonianMVP_DMStrategy.cc index 2b5f8913..be62b894 100644 --- a/src/HamiltonianMVP_DMStrategy.cc +++ b/src/HamiltonianMVP_DMStrategy.cc @@ -57,7 +57,7 @@ HamiltonianMVP_DMStrategy void HamiltonianMVP_DMStrategy::initialize(OrbitalsType& orbitals) + OrbitalsType>::initialize(OrbitalsType&) { } diff --git a/src/Ion.h b/src/Ion.h index 336dee72..ae870820 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -176,7 +176,7 @@ class Ion } double position(const short i) const { return position_[i]; } - double old_position(const short i) const { return old_position_[i]; } + double getPreviousPosition(const short i) const { return old_position_[i]; } void setPosition(const double x, const double y, const double z) { old_position_[0] = position_[0]; diff --git a/src/IonData.cc b/src/IonData.cc index 8f62ddd8..89c1bd0c 100644 --- a/src/IonData.cc +++ b/src/IonData.cc @@ -10,12 +10,11 @@ #include "IonData.h" #include "tools.h" #include -using namespace std; void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_name = name; cptr += IonData_MaxStrLength; @@ -52,16 +51,16 @@ void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) // pack Ions data for communication void IonData::packIonData( - char* cbuff, int* ibuff, double* dbuff, vector& data) + char* cbuff, int* ibuff, double* dbuff, std::vector& data) { // pack ion_names buffer int idx = 0; - for (vector::iterator idata = data.begin(); idata != data.end(); - ++idata) + for (std::vector::iterator idata = data.begin(); + idata != data.end(); ++idata) { - string s = (*idata).ion_name; + std::string s = (*idata).ion_name; FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; } @@ -69,8 +68,8 @@ void IonData::packIonData( // pack integer datatypes int* iptr = &ibuff[0]; // first pack local data size - *(iptr++) = data.size(); - vector::iterator idata = data.begin(); + *(iptr++) = data.size(); + std::vector::iterator idata = data.begin(); while (idata != data.end()) { // pack atomic_num diff --git a/src/IonicStepper.cc b/src/IonicStepper.cc index ca8d53cc..aa3a7f48 100644 --- a/src/IonicStepper.cc +++ b/src/IonicStepper.cc @@ -13,8 +13,6 @@ #include "MGmol_blas1.h" #include -using namespace std; - IonicStepper::IonicStepper(const double dt, const std::vector& atmove, std::vector& tau0, std::vector& taup) : atmove_(atmove), tau0_(tau0), taup_(taup) @@ -51,7 +49,8 @@ IonicStepper::IonicStepper(const double dt, const std::vector& atmove, } int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, - const vector& data, const string& name, const bool create) const + const std::vector& data, const std::string& name, + const bool create) const { hid_t file_id = h5f_file.file_id(); if (file_id < 0) return 0; @@ -62,7 +61,6 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (create) { - // Create the data space for new datasets hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; dataspace_id = H5Screate_simple(2, dims, nullptr); @@ -70,7 +68,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } @@ -81,7 +79,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dcreate2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -93,7 +91,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -103,7 +101,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeAtomicFields: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -111,14 +109,15 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeAtomicFields, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" << endl; + << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" + << std::endl; return -1; } @@ -129,7 +128,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -137,7 +136,8 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, return 0; } -int IonicStepper::writePositions(HDFrestart& h5f_file, const string& name) const +int IonicStepper::writePositions( + HDFrestart& h5f_file, const std::string& name) const { return writeAtomicFields(h5f_file, tau0_, name, false); } @@ -155,7 +155,8 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); if (dataspace_id < 0) { - (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" << endl; + (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" + << std::endl; return -1; } @@ -170,7 +171,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { std::cerr << "IonicStepper::writeVelocities, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -182,12 +183,12 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { (*MPIdata::serr) << "IonicStepper:: H5Dcreate2 /Ionic_velocities failed!!!" - << endl; + << std::endl; return -1; } } - vector data(taup_); + std::vector data(taup_); double minus = -1.; int n = (int)tau0_.size(), ione = 1; DAXPY(&n, &minus, &tau0_[0], &ione, &data[0], &ione); @@ -202,20 +203,20 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const if (status < 0) { (*MPIdata::serr) << "IonicStepper::H5Dwrite velocities failed!!!" - << endl; + << std::endl; return -1; } else { if (onpe0) (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } H5Sclose(dataspace_id); @@ -224,7 +225,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const } int IonicStepper::readAtomicFields( - HDFrestart& h5f_file, vector& data, const string& name) + HDFrestart& h5f_file, std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -239,7 +240,7 @@ int IonicStepper::readAtomicFields( if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper, H5Dopen2 failed for " << name - << " !!!" << endl; + << " !!!" << std::endl; return -1; } @@ -249,14 +250,15 @@ int IonicStepper::readAtomicFields( H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper, H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper, H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } @@ -265,13 +267,14 @@ int IonicStepper::readAtomicFields( return 0; } -int IonicStepper::readPositions_hdf5(HDFrestart& h5f_file, const string& name) +int IonicStepper::readPositions_hdf5( + HDFrestart& h5f_file, const std::string& name) { return readAtomicFields(h5f_file, tau0_, name); } int IonicStepper::writeRandomStates(HDFrestart& h5f_file, - vector& data, const string& name) const + std::vector& data, const std::string& name) const { hid_t file_id = h5f_file.file_id(); bool create = false; @@ -282,7 +285,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper:: H5Dopen2 " << name - << " failed!!! Creating new data file " << endl; + << " failed!!! Creating new data file " << std::endl; // Create the data space for the dataset hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; @@ -291,7 +294,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "Ions::writeRandomStates: H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } // Create the dataset @@ -300,7 +303,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) - << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << endl; + << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << std::endl; return -1; } create = true; @@ -312,7 +315,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeRandomStates: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -320,14 +323,15 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeRandomStates, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeRandomStates, H5Dclose failed!!!" << endl; + << "IonicStepper::writeRandomStates, H5Dclose failed!!!" + << std::endl; return -1; } @@ -338,7 +342,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -346,8 +350,8 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, return 0; } -int IonicStepper::readRandomStates( - HDFrestart& h5f_file, vector& data, const string& name) +int IonicStepper::readRandomStates(HDFrestart& h5f_file, + std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -360,10 +364,10 @@ int IonicStepper::readRandomStates( if (onpe0) { (*MPIdata::sout) - << "H5Dopen failed for /Ionic_RandomStates" << endl; + << "H5Dopen failed for /Ionic_RandomStates" << std::endl; (*MPIdata::sout) << "Set random states to default computed in Ion.cc" - << endl; + << std::endl; } } else @@ -373,19 +377,20 @@ int IonicStepper::readRandomStates( if (onpe0) (*MPIdata::sout) << "Read Ionic random states from " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; herr_t status = H5Dread(dataset_id, H5T_NATIVE_USHORT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper: H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper: H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } diff --git a/src/Ions.cc b/src/Ions.cc index 852151c7..e93d82ea 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -34,13 +34,13 @@ Timer ions_setup_tm("ions::setup"); const double ang2bohr = 1.8897269; // const double rmax = 8.0; -std::map Ions::map_species_ - = { { "H", 1 }, { "D", 1 }, { "Li", 3 }, { "Be", 4 }, { "B", 5 }, - { "C", 6 }, { "N", 7 }, { "O", 8 }, { "F", 9 }, { "Na", 11 }, - { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, { "P", 15 }, { "S", 16 }, - { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, { "Cr", 24 }, { "Mn", 25 }, - { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, { "Cu", 29 }, { "Zn", 30 }, - { "Ga", 31 }, { "Ge", 32 }, { "La", 57 }, { "Au", 79 } }; +std::map Ions::map_species_ = { { "H", 1 }, { "D", 1 }, + { "Li", 3 }, { "Be", 4 }, { "B", 5 }, { "C", 6 }, { "N", 7 }, { "O", 8 }, + { "F", 9 }, { "Na", 11 }, { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, + { "P", 15 }, { "S", 16 }, { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, + { "Cr", 24 }, { "Mn", 25 }, { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, + { "Cu", 29 }, { "Zn", 30 }, { "Ga", 31 }, { "Ge", 32 }, { "Br", 35 }, + { "La", 57 }, { "Au", 79 } }; int Ions::num_ions_ = -1; short Ions::max_num_proj_ = -1; @@ -700,13 +700,12 @@ void Ions::readLockedAtomNames(HDFrestart& h5f_file) if (dim == 0) return; std::vector data; - h5f_file.readLockedAtomNames(data); + std::string datasetname("/LockedAtomsNames"); + h5f_file.readAtomicData(datasetname, data); - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) + for (auto& i : data) { - lockAtom(*i); + lockAtom(i); } } @@ -874,13 +873,17 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) setupListIonsBoundaries(rmax); std::vector at_numbers; - h5_file.readAtomicNumbers(at_numbers); + std::string datasetname("/Atomic_numbers"); + h5_file.readAtomicData(datasetname, at_numbers); std::vector at_indexes; - int nidxs = h5_file.readAtomicIDs(at_indexes); + std::string datasetname_indexes("/Atomic_IDs"); + int nidxs = h5_file.readAtomicData(datasetname_indexes, at_indexes); std::vector at_nlprojIds; - int npids = h5_file.readAtomicNLprojIDs(at_nlprojIds); + std::string datasetname_nlprojIds("/AtomicNLproj_IDs"); + int npids = h5_file.readAtomicData(datasetname_nlprojIds, at_nlprojIds); std::vector at_names; - h5_file.readAtomicNames(at_names); + std::string datasetname_names("/Atomic_names"); + h5_file.readAtomicData(datasetname_names, at_names); if (onpe0 && ct.verbose > 2) { std::cout << "HDF file: at nb=" << at_numbers.size() << std::endl; @@ -908,8 +911,10 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); - mmpi.allreduce(&num_ions_, 1, MPI_SUM); - + if (!h5_file.useHdf5p()) + { + mmpi.allreduce(&num_ions_, 1, MPI_SUM); + } if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ @@ -944,9 +949,21 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) } readRestartPositions(h5_file); readRestartVelocities(h5_file); - readRestartRandomStates(h5_file); + if (ct.LangevinThermostat()) readRestartRandomStates(h5_file); readLockedAtomNames(h5_file); + // remove atoms from local list if not local + for (std::vector::iterator it = local_ions_.begin(); + it != local_ions_.end();) + { + double p[3]; + (*it)->getPosition(p); + if (!inLocalIons(p[0], p[1], p[2])) + it = local_ions_.erase(it); + else + ++it; + } + // rescale all velocities by factor specified in input rescaleVelocities(ct.VelocityScalingFactor()); @@ -967,7 +984,8 @@ void Ions::readRestartPositions(HDFrestart& h5_file) (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; std::vector data; - h5_file.readAtomicPositions(data); + std::string datasetname("/Ionic_positions"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) @@ -1142,7 +1160,8 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) << std::endl; std::vector data; - h5_file.readAtomicVelocities(data); + std::string datasetname("/Ionic_velocities"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) @@ -1608,7 +1627,6 @@ int Ions::readAtomsFromXYZ( const std::string& filename, const bool cell_relative) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); // set up list boundaries // get radius of projectors @@ -1698,9 +1716,6 @@ int Ions::readAtomsFromXYZ( int Ions::setAtoms( const std::vector& crds, const std::vector& spec) { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); - const int natoms = crds.size() / 3; double velocity[3] = { 0., 0., 0. }; @@ -1748,8 +1763,6 @@ int Ions::setAtoms( addIonToList(species_[isp], aname, &crds[3 * ia], velocity, locked); } - // std::cout< gids_map; for (unsigned int i = 0; i < gids.size(); i++) diff --git a/src/MD_IonicStepper.cc b/src/MD_IonicStepper.cc index 7e797602..a1b4acb1 100644 --- a/src/MD_IonicStepper.cc +++ b/src/MD_IonicStepper.cc @@ -114,7 +114,8 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) (*MPIdata::sout) << "MD_IonicStepper::init() --- use positions " "from restart file with dt=" << dt_ << endl; - // taum_: velocities -> displacements = -dt*vel + // taum_ was initialized with velocities + // set taum_ to displacements = -dt*vel double alpha = -1. * dt_; DSCAL(&size_tau, &alpha, &taum_[0], &ione); @@ -123,117 +124,13 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) taup_ = tau0_; DAXPY(&size_tau, &minus_one, &taum_[0], &ione, &taup_[0], &ione); - // taum_ -> previous positions: tau0_ - dt*vel + // Now set taum_ to previous positions: tau0_ - dt*vel DAXPY(&size_tau, &one, &tau0_[0], &ione, &taum_[0], &ione); } return 0; } -int MD_IonicStepper::writeForces(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)tau0_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_forces", H5T_NATIVE_DOUBLE, - dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_forces failed!!!" << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &fion_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite forces failed!!!" - << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) - << "Ionic forces written into " << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - -int MD_IonicStepper::writeTaum(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)taum_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_velocities", - H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_velocities failed!!!" - << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &taum_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite taum failed!!!" << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) { hid_t file_id = h5f_file.file_id(); @@ -251,11 +148,18 @@ int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) if (status < 0) return status; if (dt_ > 0.) + { writeVelocities(h5f_file); + std::string datasetname("/Ionic_previous_positions"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } else - writeTaum(h5f_file); - - writeForces(h5f_file); + { + std::string datasetname("/Ionic_velocities"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } + std::string datasetname("/Ionic_forces"); + writeAtomicFields(h5f_file, fion_, datasetname, true); } // @@ -469,7 +373,8 @@ void MD_IonicStepper::updateTau() tau0_ = taup_; if (dt_ > 0.) { - // update taup to be able to compute velocity... + // update taup_ to be able to compute velocity: + // taup_ <- tau0_-taum_ int size_tau = (int)tau0_.size(); int ione = 1; double alpha = 1.; diff --git a/src/MD_IonicStepper.h b/src/MD_IonicStepper.h index c62b57a5..24705a1a 100644 --- a/src/MD_IonicStepper.h +++ b/src/MD_IonicStepper.h @@ -7,15 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -//////////////////////////////////////////////////////////////////////////////// -// -// MD_IonicStepper.h: -// -//////////////////////////////////////////////////////////////////////////////// -// $Id$ - -#ifndef MD_IONICSTEPPER_H -#define MD_IONICSTEPPER_H +#ifndef MGMOL_MD_IONICSTEPPER_H +#define MGMOL_MD_IONICSTEPPER_H #include "IonicStepper.h" #include @@ -72,8 +65,6 @@ class MD_IonicStepper : public IonicStepper void updateTau(); double etol(void) const override; int write_hdf5(HDFrestart&) override; - int writeForces(HDFrestart& h5f_file); - int writeTaum(HDFrestart& h5f_file); int init(HDFrestart&) override; void printVelocities(std::ostream& os) const; diff --git a/src/MVP_DMStrategy.h b/src/MVP_DMStrategy.h index 4b7e3be0..2c16fbcc 100644 --- a/src/MVP_DMStrategy.h +++ b/src/MVP_DMStrategy.h @@ -48,7 +48,7 @@ class MVP_DMStrategy : public DMStrategy const std::vector>& overlappingGids, ProjectedMatricesInterface* proj_matrices, const bool use_old_dm); - void initialize(OrbitalsType& orbitals) override{}; + void initialize(OrbitalsType&) override{}; int update(OrbitalsType& orbitals) override; // H is updated with MVP loop, so no need to compute it outside diff --git a/src/Potentials.cc b/src/Potentials.cc index a54f0c47..9916c021 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -37,9 +37,8 @@ Potentials::~Potentials() #endif } -Potentials::Potentials(const bool vh_frozen) +Potentials::Potentials() { - //(*MPIdata::sout)<<"Potentials::setup()"<::MPaxpy(size_, alpha, &v_comp_[0], v); } +void Potentials::backupVh() +{ + memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); +} + void Potentials::initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp) { diff --git a/src/Potentials.h b/src/Potentials.h index 0ddd10fa..f0f762b3 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -76,6 +76,11 @@ class Potentials std::vector dv_; + /*! + * Backpup copy of Hartree potential to save previous state + */ + std::vector vh_rho_backup_; + int itindex_vxc_; int itindex_vh_; @@ -99,7 +104,7 @@ class Potentials const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc); public: - Potentials(const bool vh_frozen = false); + Potentials(); ~Potentials(); @@ -141,14 +146,15 @@ class Potentials double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } - POTDTYPE* vtot() { return &vtot_[0]; } - POTDTYPE* vh_rho() { return &vh_rho_[0]; } - RHODTYPE* rho_comp() { return &rho_comp_[0]; } + POTDTYPE* vtot() { return vtot_.data(); } + POTDTYPE* vh_rho() { return vh_rho_.data(); } + RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return &v_nuc_[0]; } - POTDTYPE* vext() { return &v_ext_[0]; } - POTDTYPE* vepsilon() { return &vepsilon_[0]; } + POTDTYPE* vnuc() { return v_nuc_.data(); } + POTDTYPE* vext() { return v_ext_.data(); } + POTDTYPE* vepsilon() { return vepsilon_.data(); } + POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -162,8 +168,6 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } - const double getBackgroundCharge() const { return background_charge_; } - /*! * initialize total potential as local pseudopotential */ @@ -201,6 +205,11 @@ class Potentials void initBackground(Ions& ions); void addBackgroundToRhoComp(); + /*! + * Save current Hartree potential into backup array + */ + void backupVh(); + void evalIonDensityOnSamplePts(Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc); #ifdef HAVE_TRICUBIC diff --git a/src/hdf_tools.cc b/src/hdf_tools.cc index e56c0478..d0308875 100644 --- a/src/hdf_tools.cc +++ b/src/hdf_tools.cc @@ -22,6 +22,18 @@ namespace mgmol_tools { +void string2fixedlength( + std::vector& data, std::vector& tc) +{ + tc.clear(); + for (auto& d : data) + { + FixedLengthString t; + strncpy(t.mystring, d.c_str(), IonData_MaxStrLength - 1); + tc.push_back(t); + } +} + void write1d(hid_t file_id, const std::string& datasetname, std::vector& data, size_t length) { @@ -235,14 +247,7 @@ void write2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; @@ -583,14 +588,7 @@ void parallelWrite2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { diff --git a/src/md.cc b/src/md.cc index 01e34a87..1d5ee937 100644 --- a/src/md.cc +++ b/src/md.cc @@ -54,6 +54,9 @@ void MGmol::moveVnuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // save Hartree potential internally + pot.backupVh(); + // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); initNuc(ions); diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index e746611c..a5de1ad5 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -91,7 +91,6 @@ int mgmol_check() Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (myPEenv.color() > 0) { diff --git a/src/restart.cc b/src/restart.cc index 2fa76ecb..30bdad73 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -117,7 +117,7 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicIDs(h5f_file); ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); - ions.writeRandomStates(h5f_file); + if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); @@ -141,6 +141,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, pot.vh_rho(), "Hartree", &ll[0], &origin[0]); if (ierr < 0) return ierr; + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + // Write if (ct.diel) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1855eb1e..6b5514d2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -514,6 +514,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/DMandEnergyAndForces/test.py b/tests/DMandEnergyAndForces/test.py index c36f0767..b21e6e72 100755 --- a/tests/DMandEnergyAndForces/test.py +++ b/tests/DMandEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test DMandEnergyAndForces...") @@ -33,6 +34,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: diff --git a/tests/HDF5single/h2o.xyz b/tests/HDF5single/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/HDF5single/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg new file mode 100644 index 00000000..1ff2adab --- /dev/null +++ b/tests/HDF5single/md.cfg @@ -0,0 +1,34 @@ +verbosity=3 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +input_type=single_file +output_level=4 +output_filename=WF_MD +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg new file mode 100644 index 00000000..4dba942a --- /dev/null +++ b/tests/HDF5single/mgmol.cfg @@ -0,0 +1,31 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py new file mode 100755 index 00000000..080ee0ba --- /dev/null +++ b/tests/HDF5single/test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +os.remove('WF') + +print("Check energy conservation...") +tol = 1.e-4 +energy = 0. +count = 0 +for line in lines: + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +if count<4: + print("ERROR needs 4 energy values for checking conservation!") + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index 349434e8..b62d39f8 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test RestartEnergyAndForces...") @@ -54,14 +55,14 @@ ref_energy=energy break -#sys.exit(0) - #run test command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + test_energy=1.e18 for line in lines: if line.count(b'%%'): diff --git a/tests/WFEnergyAndForces/test.py b/tests/WFEnergyAndForces/test.py index 45420ddf..6cac8c93 100755 --- a/tests/WFEnergyAndForces/test.py +++ b/tests/WFEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test WFEnergyAndForces...") @@ -37,6 +38,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: diff --git a/tests/testSetGhostValues.cc b/tests/testSetGhostValues.cc index f8520055..c3680087 100644 --- a/tests/testSetGhostValues.cc +++ b/tests/testSetGhostValues.cc @@ -18,8 +18,6 @@ TEST_CASE("Set ghost values", "[set ghosts") const double ll = 1.; const double lattice[3] = { ll, ll, ll }; - const int nfunc = 10; - MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -101,8 +99,6 @@ TEST_CASE("Set ghost values", "[set ghosts") gf.assign(inner_data.data(), 'd'); gf.set_updated_boundaries(false); - double norm_before = gf.norm2(); - // fill ghost values const bool direction[3] = { true, true, true }; gf.defaultTrade_boundaries(); From a50b67c9f8b88df038c197cd7ae82393d80e6fe4 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Mon, 24 Mar 2025 19:32:41 -0700 Subject: [PATCH 39/50] Merge release (#323) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- src/BlockVector.cc | 2 +- src/Control.cc | 18 +- src/Control.h | 2 +- src/DistMatrix/BlacsContext.cc | 10 +- src/DistMatrix/DistMatrix.h | 2 +- src/DistanceConstraint.cc | 2 +- src/Electrostatic.cc | 4 +- src/Electrostatic.h | 2 +- src/ExtendedGridOrbitals.cc | 2 +- src/Forces.cc | 3 +- src/HDFrestart.cc | 29 +- src/HDFrestart.h | 8 +- src/Ion.cc | 8 +- src/Ion.h | 16 +- src/IonicAlgorithm.cc | 2 +- src/Ions.cc | 131 ++++++++- src/Ions.h | 14 + src/KBPsiMatrixSparse.cc | 45 ++- src/KBPsiMatrixSparse.h | 15 +- src/LDAFunctional.cc | 3 +- src/LocalizationRegions.cc | 20 +- src/LocalizationRegions.h | 6 +- src/MDfiles.cc | 11 +- src/MGmol.cc | 59 ++-- src/MGmol.h | 11 +- src/MGmolInterface.h | 3 + src/NonOrthoDMStrategy.cc | 2 +- src/PBEFunctional.cc | 3 +- src/Poisson.h | 5 +- src/Potentials.cc | 277 +++++++++++------- src/Potentials.h | 42 ++- src/ProjectedMatricesSparse.cc | 2 +- src/main.cc | 2 +- src/md.cc | 37 ++- src/mgmol_memory.cc | 2 +- src/mgmol_run.cc | 2 +- src/pb/Lap.h | 2 +- src/pb/PEenv.cc | 12 +- src/pb/PEenv.h | 6 +- src/quench.cc | 1 - src/radial/RadialMeshFunction.cc | 4 +- src/restart.cc | 51 +--- src/rom_workflows.cc | 5 +- src/setup.cc | 4 +- src/sparse_linear_algebra/DataDistribution.cc | 6 +- src/tools.cc | 6 +- src/tools/MGmol_MPI.cc | 6 +- src/tools/MGmol_MPI.h | 2 +- src/tools/SymmetricMatrix.h | 1 + src/tools/Vector3D.cc | 2 +- tests/CMakeLists.txt | 15 + tests/DMandEnergyAndForces/mgmol.cfg | 18 +- tests/EnergyAndForces/mgmol.cfg | 18 +- tests/HDF5single/md.cfg | 4 +- tests/HDF5single/mgmol.cfg | 2 +- tests/HDF5single/test.py | 4 +- tests/MD_D72/test.py | 2 +- tests/MD_MVP/md.cfg | 2 +- tests/MD_MVP/test.py | 8 +- tests/RestartEnergyAndForces/h2o.xyz | 7 +- tests/RestartEnergyAndForces/mgmol.cfg | 18 +- tests/RestartEnergyAndForces/restart.cfg | 18 +- tests/RestartEnergyAndForces/test.py | 6 + .../testRestartEnergyAndForces.cc | 32 ++ tests/RhoVhRestart/h2o.xyz | 6 + tests/RhoVhRestart/md.cfg | 30 ++ tests/RhoVhRestart/mgmol.cfg | 28 ++ tests/RhoVhRestart/restart.cfg | 25 ++ tests/RhoVhRestart/test.py | 79 +++++ tests/RhoVhRestart/testRhoVhRestart.cc | 220 ++++++++++++++ tests/ShortSighted/test.py | 2 +- tests/testIons.cc | 115 +++++++- 72 files changed, 1128 insertions(+), 441 deletions(-) create mode 100644 tests/RhoVhRestart/h2o.xyz create mode 100644 tests/RhoVhRestart/md.cfg create mode 100644 tests/RhoVhRestart/mgmol.cfg create mode 100644 tests/RhoVhRestart/restart.cfg create mode 100755 tests/RhoVhRestart/test.py create mode 100644 tests/RhoVhRestart/testRhoVhRestart.cc diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 40271750..9a411ebe 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -209,7 +209,7 @@ void BlockVector::allocate_storage() std::cerr << "ERROR BlockVector: trying to use allocation " << size_storage_ << " bigger than initialy preallocated " << allocated_size_storage_ << "!!!" << std::endl; - ct.global_exit(0); + ct.global_exit(); } storage_ = class_storage_[my_allocation_]; assert(class_storage_.size() > 0); diff --git a/src/Control.cc b/src/Control.cc index 043e5640..d0b1a224 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -495,7 +495,7 @@ void Control::sync(void) if (mpirc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } }; @@ -1152,7 +1152,7 @@ void Control::setTolEigenvalueGram(const float tol) << threshold_eigenvalue_gram_ << std::endl; } -void Control::global_exit(int i) { MPI_Abort(comm_global_, i); } +void Control::global_exit() { MPI_Abort(comm_global_, EXIT_FAILURE); } void Control::setSpecies(Potentials& pot) { @@ -1342,7 +1342,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::serr) << "ERROR in Control::setOptions: Invalid restart dump type" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } (*MPIdata::sout) << "Output restart file: " << out_restart_file @@ -1549,14 +1549,14 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else { std::cerr << "ERROR: Spread Penalty needs a type" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (spread_penalty_target_ <= 0.) { (*MPIdata::sout) << "Invalid value for Spread Penalty target: " << spread_penalty_target_ << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } @@ -1599,7 +1599,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) { (*MPIdata::sout) << "Invalid value for Thermostat.type: " << thermostat_type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } tkel = vm["Thermostat.temperature"].as(); @@ -1608,7 +1608,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.temperature: " << tkel << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } thtime = vm["Thermostat.relax_time"].as(); if (thtime < 0.) @@ -1616,7 +1616,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.relax_time: " << thtime << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (str.compare("SCALING") == 0) @@ -1627,7 +1627,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.width: " << thwidth << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } } diff --git a/src/Control.h b/src/Control.h index a15bd24c..77932fb3 100644 --- a/src/Control.h +++ b/src/Control.h @@ -353,7 +353,7 @@ class Control return pair_mlwf_distance_threshold_; } - void global_exit(int i); + void global_exit(); bool Mehrstellen() const { return (lap_type == 0 || lap_type == 10); } diff --git a/src/DistMatrix/BlacsContext.cc b/src/DistMatrix/BlacsContext.cc index e87cb7dd..7610a3cb 100644 --- a/src/DistMatrix/BlacsContext.cc +++ b/src/DistMatrix/BlacsContext.cc @@ -191,7 +191,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: type = " << type << " is an incorrect parameter" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } size_ = nprow_ * npcol_; @@ -222,7 +222,7 @@ BlacsContext::BlacsContext( { std::cerr << " nprocs_=" << nprocs_ << std::endl; std::cerr << " BlacsContext nprow*npcol > nprocs_" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); @@ -252,7 +252,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << " in " << __FILE__ << ":" << __LINE__ << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -296,7 +296,7 @@ BlacsContext::BlacsContext(BlacsContext& bc, const int irow, const int icol, { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -350,7 +350,7 @@ BlacsContext::BlacsContext(const BlacsContext& bc, const char type) std::cerr << " BlacsContext::BlacsContext: row/col incorrect parameter: " << type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); diff --git a/src/DistMatrix/DistMatrix.h b/src/DistMatrix/DistMatrix.h index f56df2b4..175fbd9b 100644 --- a/src/DistMatrix/DistMatrix.h +++ b/src/DistMatrix/DistMatrix.h @@ -28,7 +28,7 @@ std::cerr << "ERROR in file " << __FILE__ << " at line " << __LINE__ \ << std::endl; \ std::cerr << "Error Message: " << X << std::endl; \ - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); #endif diff --git a/src/DistanceConstraint.cc b/src/DistanceConstraint.cc index d5175d7e..6d046f3b 100644 --- a/src/DistanceConstraint.cc +++ b/src/DistanceConstraint.cc @@ -99,7 +99,7 @@ bool DistanceConstraint::enforce(void) { cerr << "mype=" << mype << ", tau1p_[0]=" << tau1p_[0] << endl; cerr << "mype=" << mype << ", tau2p_[0]=" << tau2p_[0] << endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } if (locally_owned_) (*MPIdata::sout) << setprecision(8) << "DistanceConstraint, d=" << d diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 6ac4fef3..aba09d07 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -65,7 +65,7 @@ Electrostatic::~Electrostatic() if (grhoc_ != nullptr) delete grhoc_; } -void Electrostatic::setupInitialVh(const POTDTYPE* const vh_init) +void Electrostatic::setupInitialVh(const std::vector& vh_init) { poisson_solver_->set_vh(vh_init); @@ -153,7 +153,7 @@ void Electrostatic::setupPB( // initialize vh with last trial solution pb::GridFunc gf_vh(*pbGrid_, bc_[0], bc_[1], bc_[2]); - gf_vh.assign(pot.vh_rho()); + gf_vh.assign((pot.vh_rho()).data()); poisson_solver_->set_vh(gf_vh); } diff --git a/src/Electrostatic.h b/src/Electrostatic.h index a8eb4709..f49ad12f 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -63,7 +63,7 @@ class Electrostatic template void computeVh(const pb::GridFunc& vhinit, const Ions& ions, Rho& rho, Potentials& pot); - void setupInitialVh(const POTDTYPE* const); + void setupInitialVh(const std::vector&); void setupInitialVh(const pb::GridFunc&); template void computeVhRho(Rho& rho); diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 19c0d26b..07face7c 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -1239,7 +1239,7 @@ double ExtendedGridOrbitals::dotProduct( "dot product type" << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(2); + ct.global_exit(); } dot_product_tm_.stop(); diff --git a/src/Forces.cc b/src/Forces.cc index 9ab19f6a..04e6ffd7 100644 --- a/src/Forces.cc +++ b/src/Forces.cc @@ -272,9 +272,10 @@ void Forces::get_loc_proj(RHODTYPE* rho, const int numpt = mymesh->numpt(); Potentials& pot = hamiltonian_->potential(); + const std::vector& vh_rho(pot.vh_rho()); for (int idx = 0; idx < numpt; idx++) { - const double vhrho = pot.vh_rho(idx); + const double vhrho = vh_rho[idx]; for (short dir = 0; dir < 3; dir++) { double* lproj = &(loc_proj[dir * NPTS]); diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 0fac518e..1b2c7fb7 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1440,8 +1440,8 @@ int HDFrestart::read_1func_hdf5(T* vv, const std::string& datasetname) } template -int HDFrestart::write_1func_hdf5( - T* vv, const std::string& datasetname, double* ll, double* cell_origin) +int HDFrestart::write_1func_hdf5(const T* const vv, + const std::string& datasetname, double* ll, double* cell_origin) { assert(ll != nullptr); assert(cell_origin != nullptr); @@ -1652,7 +1652,7 @@ int HDFrestart::readData( } template -int HDFrestart::writeData(T* data, hid_t space_id, hid_t memspace, +int HDFrestart::writeData(const T* const data, hid_t space_id, hid_t memspace, hid_t dset_id, const short precision) { if (precision == 1) @@ -1922,10 +1922,12 @@ int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), -1), data.end()); } +#endif return 0; } @@ -1974,10 +1976,12 @@ int HDFrestart::readAtomicData( } } +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); } +#endif if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2017,7 +2021,8 @@ int HDFrestart::readAtomicData( { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNames()..." << std::endl; + (*MPIdata::sout) << "HDFrestart::readAtomicData(), dataset = " + << datasetname << std::endl; std::vector buffer; short name_length = 7; // default, value used before February 2016 @@ -2095,19 +2100,19 @@ int HDFrestart::readAtomicData( { std::string t(&buffer[i], name_length); assert(t.size() > 0); - // cout<<"name="< - int write_1func_hdf5( - T*, const std::string&, double* ll = nullptr, double* origin = nullptr); + int write_1func_hdf5(const T* const, const std::string&, + double* ll = nullptr, double* origin = nullptr); int read_att(const hid_t dset_id, const std::string& attname, std::vector& attr_data); // write data in file with precision "precision" template - int writeData(T* vv, hid_t filespace, hid_t memspace, hid_t dset_id, - const short precision); + int writeData(const T* const vv, hid_t filespace, hid_t memspace, + hid_t dset_id, const short precision); template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); diff --git a/src/Ion.cc b/src/Ion.cc index 33465635..90350477 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -294,12 +294,18 @@ void Ion::getIonData(IonData& idata) const } } +void Ion::resetPositionsToPrevious() +{ + for (int pos = 0; pos < 3; pos++) + position_[pos] = old_position_[pos]; +} + void Ion::setFromIonData(const IonData& data) { // random state setRandomState(data.rand_state[0], data.rand_state[1], data.rand_state[2]); // previous position - setOldPosition( + setPreviousPosition( data.old_position[0], data.old_position[1], data.old_position[2]); // velocity setVelocity(data.velocity[0], data.velocity[1], data.velocity[2]); diff --git a/src/Ion.h b/src/Ion.h index ae870820..871205a3 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -74,12 +74,6 @@ class Ion position_[i] += shift; } - void setOldPosition(const double x, const double y, const double z) - { - old_position_[0] = x; - old_position_[1] = y; - old_position_[2] = z; - } public: Ion(const Species& species, const std::string& name, const double crds[3], @@ -96,6 +90,13 @@ class Ion void init(const double crds[3], const double velocity[3], const bool lock); void setup(); + void setPreviousPosition(const double x, const double y, const double z) + { + old_position_[0] = x; + old_position_[1] = y; + old_position_[2] = z; + } + std::shared_ptr kbproj() { return kbproj_; } const std::shared_ptr kbproj() const { return kbproj_; } @@ -189,6 +190,9 @@ class Ion kbproj_->clear(); } + + void resetPositionsToPrevious(); + void shiftPositionXLBOMDTest(Vector3D shift) { for (short dir = 0; dir < 3; dir++) diff --git a/src/IonicAlgorithm.cc b/src/IonicAlgorithm.cc index 37a077e4..e804077a 100644 --- a/src/IonicAlgorithm.cc +++ b/src/IonicAlgorithm.cc @@ -57,7 +57,7 @@ void IonicAlgorithm::init(HDFrestart* h5f_file) if (ct.restart_info > 0) { int status = stepper_->init(*h5f_file); - if (status < 0) ct.global_exit(2); + if (status < 0) ct.global_exit(); // if restart data for lbfgs found if (status == 0) diff --git a/src/Ions.cc b/src/Ions.cc index e93d82ea..10baad10 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -853,6 +853,42 @@ void Ions::writePositions(HDFrestart& h5f_file) } } +void Ions::writePreviousPositions(HDFrestart& h5f_file) +{ + Control& ct(*(Control::instance())); + + if (onpe0 && ct.verbose > 1) + { + (*MPIdata::sout) << "Ions::writePositions" << std::endl; + } + + std::vector data; + if (h5f_file.gatherDataX()) + { + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MPI_Comm comm = myPEenv.comm_x(); + + gatherPreviousPositions(data, 0, comm); + } + else + { + for (auto& ion : local_ions_) + { + data.push_back(ion->getPreviousPosition(0)); + data.push_back(ion->getPreviousPosition(1)); + data.push_back(ion->getPreviousPosition(2)); + } + } + + hid_t file_id = h5f_file.file_id(); + if (file_id >= 0) + { + std::string datasetname("/Ionic_previous_positions"); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); + } +} + void Ions::initFromRestartFile(HDFrestart& h5_file) { assert(list_ions_.empty()); @@ -911,10 +947,12 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); +#ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } +#endif if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ @@ -995,14 +1033,39 @@ void Ions::readRestartPositions(HDFrestart& h5_file) } } +void Ions::readRestartPreviousPositions(HDFrestart& h5_file) +{ + Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 0) + (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + + std::vector data; + std::string datasetname("/Ionic_previous_positions"); + h5_file.readAtomicData(datasetname, data); + assert(data.size() == 3 * local_ions_.size()); + + int i = 0; + for (auto& ion : local_ions_) + { + ion->setPreviousPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); + i++; + } +} + +void Ions::resetPositionsToPrevious() +{ + for (auto& ion : local_ions_) + { + ion->resetPositionsToPrevious(); + } +} + void Ions::writeVelocities(HDFrestart& h5f_file) { Control& ct(*(Control::instance())); if (onpe0 && ct.verbose > 1) - { (*MPIdata::sout) << "Ions::writeVelocities" << std::endl; - } std::vector data; if (h5f_file.gatherDataX()) @@ -1225,6 +1288,29 @@ void Ions::writeForces(HDFrestart& h5f_file) } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& names) +{ + assert(forces.size() == 3 * names.size()); + + // loop over global list of forces and atom names + std::vector::const_iterator s = names.begin(); + for (auto it = forces.begin(); it != forces.end(); it += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + if (ion->compareName(*s)) + { + ion->set_force(0, *it); + ion->set_force(1, *(it + 1)); + ion->set_force(2, *(it + 2)); + } + } + s++; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { @@ -1900,7 +1986,6 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) double velocity[3] = { 0., 0., 0. }; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); short movable = 0; std::string query; @@ -1944,7 +2029,7 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) { std::cerr << "ERROR: Invalid name read in input file: " << name_read << std::endl; - ct.global_exit(2); + mmpi.abort(); } short dummy; ss >> dummy; // not used anymore (was species index) @@ -2125,6 +2210,22 @@ void Ions::getLocalPositions(std::vector& tau) const } } +void Ions::getLocalNames(std::vector& names) const +{ + for (auto& ion : local_ions_) + { + names.push_back(ion->name()); + } +} + +void Ions::getNames(std::vector& names) const +{ + for (auto& ion : list_ions_) + { + names.push_back(ion->name()); + } +} + void Ions::getPositions(std::vector& tau) { std::vector tau_local(3 * local_ions_.size()); @@ -2535,6 +2636,28 @@ void Ions::gatherPositions( if (mype == root) positions = data; } +void Ions::gatherPreviousPositions( + std::vector& positions, const int root, const MPI_Comm comm) const +{ + std::vector local_positions; + + for (auto& ion : local_ions_) + { + local_positions.push_back(ion->getPreviousPosition(0)); + local_positions.push_back(ion->getPreviousPosition(1)); + local_positions.push_back(ion->getPreviousPosition(2)); + } + + // gather data to PE root + std::vector data; + mgmol_tools::gatherV(local_positions, data, root, comm); + + int mype = 0; + MPI_Comm_rank(comm, &mype); + positions.clear(); + if (mype == root) positions = data; +} + void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { diff --git a/src/Ions.h b/src/Ions.h index ae4f6adb..33b8982b 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -147,6 +147,8 @@ class Ions const MPI_Comm comm) const; void gatherPositions(std::vector& positions, const int root, const MPI_Comm comm) const; + void gatherPreviousPositions(std::vector& positions, const int root, + const MPI_Comm comm) const; void gatherLockedNames(std::vector& names, const int root, const MPI_Comm comm) const; void gatherIndexes( @@ -198,6 +200,7 @@ class Ions ion++; } } + void resetPositionsToPrevious(); void removeMassCenterMotion(); bool hasNLprojectors() @@ -237,6 +240,7 @@ class Ions double kinetic_E(void) const; void writePositions(HDFrestart& h5f_file); + void writePreviousPositions(HDFrestart& h5f_file); void writeVelocities(HDFrestart& h5f_file); void writeRandomStates(HDFrestart& h5f_file); void writeForces(HDFrestart& h5f_file); @@ -281,11 +285,20 @@ class Ions const std::vector& tau, const std::vector& anumbers); void getLocalPositions(std::vector& tau) const; + void getLocalNames(std::vector& names) const; + void getNames(std::vector& names) const; void getPositions(std::vector& tau); void getAtomicNumbers(std::vector& atnumbers); void getForces(std::vector& forces); void getLocalForces(std::vector& tau) const; + + /*! + * set forces for ions in local_ions_ based on names matching + */ + void setLocalForces(const std::vector& forces, + const std::vector& names); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); @@ -350,6 +363,7 @@ class Ions void addIonToList(const Species& sp, const std::string& name, const double crds[3], const double velocity[3], const bool lock); + void readRestartPreviousPositions(HDFrestart& h5_file); // void checkUnicityLocalIons(); }; diff --git a/src/KBPsiMatrixSparse.cc b/src/KBPsiMatrixSparse.cc index 604e2f8e..8f5b3d49 100644 --- a/src/KBPsiMatrixSparse.cc +++ b/src/KBPsiMatrixSparse.cc @@ -11,16 +11,15 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" -#include "Ions.h" #include "LocGridOrbitals.h" #include "MGmol_MPI.h" #include "Mesh.h" #include "ProjectedMatrices.h" -#include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include + #define Ry2Ha 0.5; Timer KBPsiMatrixSparse::global_sum_tm_("KBPsiMatrixSparse::global_sum"); @@ -133,7 +132,7 @@ void KBPsiMatrixSparse::globalSumKBpsi() // Loop over the ions with projectors overlapping with local subdomain // and evaluate for some state. template -void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, T& orbitals, const int first_color, const int nb_colors, const bool flag) { assert(first_color >= 0); @@ -189,7 +188,7 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, // Loop over the ions if (gid != -1) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement( *ion, gid, iloc, ppsi + color * ldsize, flag); @@ -211,8 +210,8 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, compute_kbpsi_tm_.stop(); } -void KBPsiMatrixSparse::computeKBpsi( - Ions& ions, pb::GridFunc* phi, const int istate, const bool flag) +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, + pb::GridFunc* phi, const int istate, const bool flag) { assert(lapop_ != nullptr); compute_kbpsi_tm_.start(); @@ -238,7 +237,7 @@ void KBPsiMatrixSparse::computeKBpsi( for (int iloc = 0; iloc < subdivx; iloc++) { // Loop over the ions - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement(*ion, istate, iloc, ppsi, flag); } @@ -251,7 +250,7 @@ void KBPsiMatrixSparse::computeKBpsi( void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -266,8 +265,8 @@ void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) // loop over states to multiply kbpsi_[st][gid] and kbBpsi_[st][gid] // by coeff - (*kbpsimat_).scaleRow(gid, coeff); - if (lapop_) (*kbBpsimat_).scaleRow(gid, coeff); + kbpsimat_->scaleRow(gid, coeff); + if (lapop_) kbBpsimat_->scaleRow(gid, coeff); } } } @@ -454,7 +453,7 @@ SquareSubMatrix KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, Aij); } @@ -473,7 +472,7 @@ void KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, mat); } @@ -538,14 +537,14 @@ void KBPsiMatrixSparse::getPsiKBPsiSym( { // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { getPsiKBPsiSym(*ion, sm); } } template -void KBPsiMatrixSparse::computeAll(Ions& ions, T& orbitals) +void KBPsiMatrixSparse::computeAll(const Ions& ions, T& orbitals) { assert(count_proj_subdomain_ == ions.countProjectorsSubdomain()); @@ -603,7 +602,7 @@ double KBPsiMatrixSparse::getEvnl( double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -618,10 +617,9 @@ double KBPsiMatrixSparse::getEvnl( /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -637,7 +635,7 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -652,10 +650,9 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -736,12 +733,12 @@ double KBPsiMatrixSparse::getTraceDM( return trace; } -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, LocGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, LocGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, LocGridOrbitals&); -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, ExtendedGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, ExtendedGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, ExtendedGridOrbitals&); diff --git a/src/KBPsiMatrixSparse.h b/src/KBPsiMatrixSparse.h index 91f6c583..073fbddc 100644 --- a/src/KBPsiMatrixSparse.h +++ b/src/KBPsiMatrixSparse.h @@ -12,7 +12,9 @@ #include "DataDistribution.h" #include "DensityMatrixSparse.h" +#include "Ions.h" #include "KBPsiMatrixInterface.h" +#include "ProjectedMatricesSparse.h" #include "SquareSubMatrix.h" #include "VariableSizeMatrix.h" @@ -23,11 +25,6 @@ #include #include -class Ions; -class Ion; -class ProjectedMatricesInterface; -class ProjectedMatricesSparse; - class KBPsiMatrixSparse : public KBPsiMatrixInterface { static Timer global_sum_tm_; @@ -79,8 +76,8 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface void getPsiKBPsiSym(const Ions& ions, VariableSizeMatrix& sm); void getPsiKBPsiSym(const Ion& ion, VariableSizeMatrix& sm); template - void computeKBpsi(Ions& ions, OrbitalsType& orbitals, const int first_color, - const int nb_colors, const bool flag); + void computeKBpsi(const Ions& ions, OrbitalsType& orbitals, + const int first_color, const int nb_colors, const bool flag); void clearData(); public: @@ -105,7 +102,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface double getEvnl( const Ions& ions, ProjectedMatrices* proj_matrices); void computeKBpsi( - Ions& ions, pb::GridFunc*, const int, const bool flag); + const Ions& ions, pb::GridFunc*, const int, const bool flag); double getValIonState(const int gid, const int st) const { return (*kbpsimat_).get_value(gid, st); @@ -127,7 +124,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface const Ions& ions, ProjectedMatricesInterface* proj_matrices) const; template - void computeAll(Ions& ions, T& orbitals); + void computeAll(const Ions& ions, T& orbitals); void setup(const Ions& ions); double getTraceDM( diff --git a/src/LDAFunctional.cc b/src/LDAFunctional.cc index 4b60b51e..89f5ee5f 100644 --- a/src/LDAFunctional.cc +++ b/src/LDAFunctional.cc @@ -120,8 +120,7 @@ double LDAFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = (POTDTYPE)sum; return exc; diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 60f28301..f0f96412 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -492,7 +492,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'nglobal_'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (nglobal_ == 0) return; @@ -522,7 +522,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'centers'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } vector::iterator it = all_regions_.begin(); int i = 0; @@ -554,7 +554,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'radius'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } it = all_regions_.begin(); i = 0; @@ -642,7 +642,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'volume'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (ct.verbose > 0) printWithTimeStamp("LocalizationRegions::bcast() done...", cout); @@ -915,19 +915,9 @@ void LocalizationRegions::setupLocalRegionsFromOverlapRegions() cerr << "ERROR in distribution of localization centers: count=" << count << endl; cerr << "global number of regions=" << nglobal_ << endl; - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } -#if 0 - //if(onpe0) - if( count!=nglobal_ ) - { - cerr<<"ERROR in distribution of localization centers: count="<::initial() mmpi.barrier(); if (ct.verbose > 0) current_orbitals_->printChromaticNumber(os_); - pot.initBackground(*ions_); - // Random initialization of the wavefunctions if (ct.restart_info <= 2) { @@ -360,8 +358,8 @@ int MGmol::initial() } // Initialize the nuclear local potential and the compensating charges - if (ct.verbose > 0) printWithTimeStamp("initNuc()...", os_); - initNuc(*ions_); + if (ct.verbose > 0) printWithTimeStamp("setupPotentials()...", os_); + setupPotentials(*ions_); // initialize Rho if (ct.verbose > 0) printWithTimeStamp("Initialize Rho...", os_); @@ -402,7 +400,6 @@ int MGmol::initial() current_orbitals_->setDataWithGhosts(); current_orbitals_->trade_boundaries(); - // if(ct.restart_info <= 1)pot.initWithVnuc(); // initialize matrices S and invB if (ct.numst > 0) @@ -418,9 +415,6 @@ int MGmol::initial() current_orbitals_->checkCond(100000., ct.AtomsMove()); } - if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); - if (ct.restart_info == 0) { if (ct.verbose > 0) printWithTimeStamp("update_pot...", os_); @@ -730,12 +724,6 @@ void MGmol::write_header() if (ct.isLocMode() && ct.verbose > 3) lrs_->printAllRegions(os_); } -template -void MGmol::global_exit(int i) -{ - MPI_Abort(comm_, i); -} - template void MGmol::check_anisotropy() { @@ -750,7 +738,8 @@ void MGmol::check_anisotropy() << ", hmin=" << mygrid.hmin() << std::endl; (*MPIdata::serr) << "init: Anisotropy too large: " << mygrid.anisotropy() << std::endl; - global_exit(2); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.abort(); } } @@ -825,7 +814,7 @@ double get_trilinval(const double xc, const double yc, const double zc, #endif template -void MGmol::initNuc(Ions& ions) +void MGmol::setupPotentials(Ions& ions) { init_nuc_tm_.start(); @@ -834,26 +823,15 @@ void MGmol::initNuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // initialize poentials based on ionic positions and their species pot.initialize(ions); - // Check compensating charges - double comp_rho = getCharge(pot.rho_comp()); - - if (onpe0 && ct.verbose > 1) - { - os_ << std::setprecision(8) << std::fixed - << " Charge of rhoc: " << comp_rho << std::endl; - } - -#if 1 - pot.rescaleRhoComp(); -#endif - - pot.addBackgroundToRhoComp(); + if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); + g_kbpsi_->setup(*ions_); electrostat_->setupRhoc(pot.rho_comp()); - if (onpe0 && ct.verbose > 3) os_ << " initNuc done" << std::endl; + if (onpe0 && ct.verbose > 3) os_ << " setupPotentials done" << std::endl; init_nuc_tm_.stop(); } @@ -1062,7 +1040,8 @@ void MGmol::setup() total_tm_.start(); setup_tm_.start(); - Control& ct = *(Control::instance()); + Control& ct = *(Control::instance()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (ct.verbose > 0) printWithTimeStamp("MGmol::setup()...", os_); @@ -1080,7 +1059,8 @@ void MGmol::setup() #else int ierr = initial(); #endif - if (ierr < 0) global_exit(0); + + if (ierr < 0) mmpi.abort(); // Write header to stdout write_header(); @@ -1112,7 +1092,6 @@ void MGmol::dumpRestart() // create restart file std::string filename(std::string(ct.out_restart_file)); - if (ct.out_restart_file_naming_strategy) filename += "0"; HDFrestart h5restartfile( filename, myPEenv, gdim, ct.out_restart_file_type); @@ -1411,14 +1390,14 @@ void MGmol::update_pot(const Ions& ions) const bool flag_mixing = (fabs(ct.mix_pot - 1.) > 1.e-3); - // evaluate potential correction + // update total potential if (flag_mixing) { - pot.delta_v(rho_->rho_); - pot.update(ct.mix_pot); + pot.computeDeltaV(rho_->rho_); + pot.updateVtot(ct.mix_pot); } else - pot.update(rho_->rho_); + pot.updateVtot(rho_->rho_); } template @@ -1461,7 +1440,7 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); @@ -1483,7 +1462,7 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); // initialize electronic density rho_->update(*dorbitals); diff --git a/src/MGmol.h b/src/MGmol.h index 315a0ff9..43df67f2 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -153,7 +153,6 @@ class MGmol : public MGmolInterface void initialMasks(); int setupLRsFromInput(const std::string filename); - void setup(); int setupLRs(const std::string input_file) override; int setupFromInput(const std::string input_file) override; int setupConstraintsFromInput(const std::string input_file) override; @@ -180,6 +179,8 @@ class MGmol : public MGmolInterface ~MGmol() override; + void setup(); + /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } std::shared_ptr> getHamiltonian() @@ -223,10 +224,9 @@ class MGmol : public MGmolInterface void getAtomicNumbers(std::vector& an); - void initNuc(Ions& ions); + void setupPotentials(Ions& ions); void initKBR(); - void global_exit(int i); void printEigAndOcc(); int readCoordinates(std::ifstream* tfile, const bool cell_relative); @@ -345,6 +345,11 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + void setPositions(const std::vector& positions, + const std::vector& atnumbers) + { + ions_->setPositions(positions, atnumbers); + } /* * simply dump current state diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9a9bf8a6..dc54a8e7 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -40,6 +40,9 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual void setPositions(const std::vector& positions, + const std::vector& atnumbers) + = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/src/NonOrthoDMStrategy.cc b/src/NonOrthoDMStrategy.cc index 5452592f..3835b039 100644 --- a/src/NonOrthoDMStrategy.cc +++ b/src/NonOrthoDMStrategy.cc @@ -46,7 +46,7 @@ int NonOrthoDMStrategy::update(OrbitalsType& orbitals) { std::cerr << "NonOrthoDMStrategy, Invalid mixing value: " << mix_ << std::endl; - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); } if (mmpi.PE0() && ct.verbose > 2) diff --git a/src/PBEFunctional.cc b/src/PBEFunctional.cc index 4a878c3c..29151a14 100644 --- a/src/PBEFunctional.cc +++ b/src/PBEFunctional.cc @@ -167,8 +167,7 @@ double PBEFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << std::endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = sum; return exc; diff --git a/src/Poisson.h b/src/Poisson.h index 14716cda..4bdfeeab 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -71,7 +71,10 @@ class Poisson : public PoissonInterface virtual void set_rhod(pb::GridFunc* /*rhod*/){}; void set_vh(const pb::GridFunc& vh) { (*vh_) = vh; }; - void set_vh(const POTDTYPE* const vh) { vh_->assign(vh, 'd'); }; + void set_vh(const std::vector& vh) + { + vh_->assign(vh.data(), 'd'); + }; void resetVh() { vh_->resetData(); } void set_vepsilon(const POTDTYPE* const vepsilon) { diff --git a/src/Potentials.cc b/src/Potentials.cc index 9916c021..a620cdeb 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -25,7 +25,6 @@ #include "mputils.h" #include -using namespace std; // unit conversion factor Ha -> Ry const double ha2ry = 2.; @@ -88,41 +87,24 @@ Potentials::Potentials() #endif } -void Potentials::initWithVnuc() -{ - assert(size_ > 0); - if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::initWithVnuc()" << endl; - itindex_vxc_ = 0; - itindex_vh_ = 0; - int ione = 1; - Tcopy(&size_, &v_nuc_[0], &ione, &vtot_[0], &ione); - double one = 1.; - LinearAlgebraUtils::MPaxpy( - size_, one, &v_ext_[0], &vtot_[0]); - // factor ha2ry to get total potential in [Ry] for calculations - LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); -} - double Potentials::max() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmax = (*max_element(vtot_.begin(), vtot_.end())); - vmax = myPEenv.double_max_all(vmax); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmax = (*max_element(vtot_.begin(), vtot_.end())); + vmax = mmpi.allreduce(&vmax, 1, MPI_MAX); return vmax; } double Potentials::min() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmin = -(*min_element(vtot_.begin(), vtot_.end())); - vmin = -myPEenv.double_max_all(vmin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmin = -(*min_element(vtot_.begin(), vtot_.end())); + vmin = -mmpi.allreduce(&vmin, 1, MPI_MAX); return vmin; } -void Potentials::evalNormDeltaVtotRho(const vector>& rho) +void Potentials::evalNormDeltaVtotRho( + const std::vector>& rho) { Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); @@ -141,17 +123,16 @@ void Potentials::evalNormDeltaVtotRho(const vector>& rho) mmpi.allreduce(&scf_dvrho_, 1, MPI_SUM); } -double Potentials::update(const vector>& rho) +double Potentials::updateVtot(const std::vector>& rho) { assert(itindex_vxc_ >= 0); assert(itindex_vh_ >= 0); assert(itindex_vxc_ == itindex_vh_); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::update(rho)" << endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + (*MPIdata::sout) << "Potentials::update(rho)" << std::endl; + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -181,28 +162,27 @@ double Potentials::update(const vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; } -void Potentials::update(const double mix) +void Potentials::updateVtot(const double mix) { assert(itindex_vxc_ == itindex_vh_); #ifdef DEBUG - if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << endl; + if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << std::endl; #endif // int ione=1; double potmix = mix; @@ -210,17 +190,16 @@ void Potentials::update(const double mix) size_, potmix, &dv_[0], &vtot_[0]); } -double Potentials::delta_v(const vector>& rho) +double Potentials::computeDeltaV(const std::vector>& rho) { assert(itindex_vxc_ == itindex_vh_); assert(size_ > 0); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::delta_v()" << endl; + (*MPIdata::sout) << "Potentials::delta_v()" << std::endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -249,24 +228,23 @@ double Potentials::delta_v(const vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; } // in Ry -void Potentials::getVofRho(vector& vrho) const +void Potentials::getVofRho(std::vector& vrho) const { vrho.resize(size_); int ione = 1; @@ -283,7 +261,7 @@ void Potentials::getVofRho(vector& vrho) const // type: // 2->text // 3->binary -void Potentials::readExternalPot(const string filename, const char type) +void Potentials::readExternalPot(const std::string filename, const char type) { assert(type == 2 || type == 3); @@ -304,27 +282,27 @@ void Potentials::readExternalPot(const string filename, const char type) { from->seekg(0, ios::end); const int length = from->tellg(); - (*MPIdata::sout) << "Length file = " << length << endl; + (*MPIdata::sout) << "Length file = " << length << std::endl; from->seekg(0, ios::beg); if (length <= 0) { (*MPIdata::serr) - << "ERROR Potential: file length <=0!!!!" << endl; + << "ERROR Potential: file length <=0!!!!" << std::endl; mmpi.abort(); } } } if (!from) { - (*MPIdata::serr) << " Cannot open file " << filename << endl; + (*MPIdata::serr) << " Cannot open file " << filename << std::endl; mmpi.abort(); } if (onpe0) { (*MPIdata::sout) << "Potentials::read_ExternalPot(), filename=" - << filename << endl; - if (type == 2) (*MPIdata::sout) << "text file..." << endl; - if (type == 3) (*MPIdata::sout) << "binary file..." << endl; + << filename << std::endl; + if (type == 2) (*MPIdata::sout) << "text file..." << std::endl; + if (type == 3) (*MPIdata::sout) << "binary file..." << std::endl; } // read origin and end of cell (to check compatibility) @@ -348,29 +326,32 @@ void Potentials::readExternalPot(const string filename, const char type) if (onpe0) for (short d = 0; d < 3; d++) { - (*MPIdata::sout) << setprecision(8); + (*MPIdata::sout) << std::setprecision(8); if (fabs(origin[d] - mygrid.origin(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell origin in direction " - << d << endl; - (*MPIdata::serr) << "Potential origin=" << origin[d] << endl; - (*MPIdata::serr) << "MGmol origin=" << mygrid.origin(d) << endl; + << d << std::endl; + (*MPIdata::serr) + << "Potential origin=" << origin[d] << std::endl; + (*MPIdata::serr) + << "MGmol origin=" << mygrid.origin(d) << std::endl; (*MPIdata::serr) << "Difference=" << fabs(origin[d] - mygrid.origin(d)) - << endl; + << std::endl; mmpi.abort(); } if (fabs(ll[d] - mygrid.ll(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell " "dimension in direction " - << d << endl; - (*MPIdata::serr) << "Potential cell end=" << end[d] << endl; + << d << std::endl; (*MPIdata::serr) - << "Potential cell dimension=" << ll[d] << endl; + << "Potential cell end=" << end[d] << std::endl; (*MPIdata::serr) - << "MGmol cell dimension=" << mygrid.ll(d) << endl; + << "Potential cell dimension=" << ll[d] << std::endl; + (*MPIdata::serr) + << "MGmol cell dimension=" << mygrid.ll(d) << std::endl; mmpi.abort(); } } @@ -392,9 +373,9 @@ void Potentials::readExternalPot(const string filename, const char type) if (nxyz[i] != gdim_[i]) { (*MPIdata::serr) << "Potentials::read_ExternalPot(): dimension " - << i << " incompatible with Grid!!!" << endl; + << i << " incompatible with Grid!!!" << std::endl; (*MPIdata::serr) - << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << endl; + << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << std::endl; mmpi.abort(); } @@ -434,7 +415,6 @@ void Potentials::readExternalPot(const string filename, const char type) { assert(index < size_); (*from) >> v_ext_[index]; - //(*MPIdata::sout)<grid(); double hmin = mygrid.hmin(); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "hmin= " << hmin << endl; + (*MPIdata::sout) << "hmin= " << hmin << std::endl; - vector::const_iterator it_filename = pot_filenames_.begin(); - int isp = 0; + std::vector::const_iterator it_filename + = pot_filenames_.begin(); + int isp = 0; while (it_filename != pot_filenames_.end()) { if (pot_types_[isp] == 'n' || pot_types_[isp] == 's' @@ -556,13 +538,14 @@ void Potentials::readAll(vector& sp) #else (*MPIdata::sout) << "ERROR: cannot read external potential " - << " -> need to compile with Tricubic library" << endl; + << " -> need to compile with Tricubic library" << std::endl; #endif } it_filename++; isp++; } } + template void Potentials::setVxc(const T* const vxc, const int iterativeIndex) { @@ -571,13 +554,6 @@ void Potentials::setVxc(const T* const vxc, const int iterativeIndex) itindex_vxc_ = iterativeIndex; MPcpy(&vxc_rho_[0], vxc, size_); } -void Potentials::setVh(const POTDTYPE* const vh, const int iterativeIndex) -{ - assert(iterativeIndex >= 0); - int ione = 1; - itindex_vh_ = iterativeIndex; - Tcopy(&size_, vh, &ione, &vh_rho_[0], &ione); -} void Potentials::setVh( const pb::GridFunc& vh, const int iterativeIndex) @@ -594,11 +570,6 @@ void Potentials::axpVcompToVh(const double alpha) size_, alpha, &v_comp_[0], &vh_rho_[0]); } -void Potentials::axpVcomp(POTDTYPE* v, const double alpha) -{ - LinearAlgebraUtils::MPaxpy(size_, alpha, &v_comp_[0], v); -} - void Potentials::backupVh() { memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); @@ -792,9 +763,12 @@ void Potentials::initialize(Ions& ions) const pb::Grid& mygrid = mymesh->grid(); const int numpt = mygrid.size(); - memset(&v_comp_[0], 0, numpt * sizeof(POTDTYPE)); - memset(&rho_comp_[0], 0, numpt * sizeof(RHODTYPE)); - memset(&v_nuc_[0], 0, numpt * sizeof(RHODTYPE)); + memset(v_comp_.data(), 0, numpt * sizeof(POTDTYPE)); + memset(rho_comp_.data(), 0, numpt * sizeof(RHODTYPE)); + memset(v_nuc_.data(), 0, numpt * sizeof(RHODTYPE)); + + // Count up the total ionic charge + ionic_charge_ = ions.computeIonicCharge(); char flag_filter = pot_type(0); @@ -805,6 +779,7 @@ void Potentials::initialize(Ions& ions) Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + // initialize rho_comp_, v_comp_, v_nuc_ if (flag_filter == 's') { const int sampleRate = 3; @@ -820,6 +795,13 @@ void Potentials::initialize(Ions& ions) initializeRadialDataOnMesh(position, sp); } } + + // rescale rho_comp_ due to finite mesh effects + rescaleRhoComp(); + + initBackground(); + + addBackgroundToRhoComp(); } void Potentials::rescaleRhoComp() @@ -831,24 +813,33 @@ void Potentials::rescaleRhoComp() const pb::Grid& mygrid = mymesh->grid(); // Check compensating charges - double comp_rho = getCharge(&rho_comp_[0]); + double comp_rho = getCharge(rho_comp_.data()); + if (onpe0 && ct.verbose > 1) + { + std::cout << std::setprecision(8) << std::fixed + << " Charge of rhoc: " << comp_rho << std::endl; + } if (onpe0 && ct.verbose > 1) { - cout << " Rescaling rhoc" << endl; + std::cout << " Rescaling rhoc" << std::endl; } + + // rescale rho_comp_ (initialized by sampling on mesh) + // so that its integral exactly matches ionic_charge_ if (ionic_charge_ > 0.) { const int numpt = mygrid.size(); double t = ionic_charge_ / comp_rho; - LinearAlgebraUtils::MPscal(numpt, t, &rho_comp_[0]); + LinearAlgebraUtils::MPscal( + numpt, t, rho_comp_.data()); // Check new compensating charges - comp_rho = getCharge(&rho_comp_[0]); + comp_rho = getCharge(rho_comp_.data()); } if (onpe0 && ct.verbose > 1) - cout << " Rescaled compensating charges: " << setprecision(8) << fixed - << comp_rho << endl; + std::cout << " Rescaled compensating charges: " << std::setprecision(8) + << std::fixed << comp_rho << std::endl; if (comp_rho < 0.) mmpi.abort(); } @@ -868,26 +859,23 @@ void Potentials::addBackgroundToRhoComp() { if (onpe0) { - cout << setprecision(12) << scientific - << "Add background charge " << background << " to rhoc " - << endl; + std::cout << std::setprecision(12) << std::scientific + << "Add background charge " << background + << " to rhoc " << std::endl; } for (int i = 0; i < numpt; i++) rho_comp_[i] += background; // Check new compensating charges - getCharge(&rho_comp_[0]); + getCharge(rho_comp_.data()); } } } -void Potentials::initBackground(Ions& ions) +void Potentials::initBackground() { Control& ct = *(Control::instance()); - // Count up the total ionic charge - ionic_charge_ = ions.computeIonicCharge(); - // calculation the compensating background charge // for charged supercell calculations background_charge_ = 0.; @@ -898,9 +886,9 @@ void Potentials::initBackground(Ions& ions) } if (onpe0 && ct.verbose > 0) { - cout << "N electrons= " << ct.getNel() << endl; - cout << "ionic charge= " << ionic_charge_ << endl; - cout << "background charge=" << background_charge_ << endl; + std::cout << "N electrons= " << ct.getNel() << std::endl; + std::cout << "ionic charge= " << ionic_charge_ << std::endl; + std::cout << "background charge=" << background_charge_ << std::endl; } if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; @@ -918,8 +906,8 @@ void Potentials::evalIonDensityOnSamplePts( { if (onpe0) { - cout << "Potentials::evalIonDensityOnSamplePts - flag_filter s is not supported" - << endl; + std::cout << "Potentials::evalIonDensityOnSamplePts - flag_filter s is not supported" + << std::endl; } mmpi.abort(); } @@ -997,6 +985,75 @@ void Potentials::initializeRadialDataOnSampledPts( return; } +int Potentials::read(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + // Read total potential + h5f_file.read_1func_hdf5(vtot_.data(), "Vtotal"); + + // Read the hartree potential + h5f_file.read_1func_hdf5(vh_rho_.data(), "Hartree"); + itindex_vh_ = 0; + + // Read dielectric potential + if (ct.diel) + { + h5f_file.read_1func_hdf5(vepsilon_.data(), "VDielectric"); + } + + std::string datasetname("Preceding_Hartree"); + if (h5f_file.checkDataExists(datasetname)) + { + h5f_file.read_1func_hdf5(vh_rho_backup_.data(), datasetname); + } + + return 0; +} + +int Potentials::write(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double ll[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + double origin[3] = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + + // Write total potential + int ierr + = h5f_file.write_1func_hdf5(vtot_.data(), "Vtotal", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + // Write the hartree potential + ierr = h5f_file.write_1func_hdf5( + vh_rho_.data(), "Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + vh_rho_backup_.data(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + + // Write + if (ct.diel) + { + ierr = h5f_file.write_1func_hdf5( + vepsilon_.data(), "VDielectric", &ll[0], &origin[0]); + } + if (ierr < 0) return ierr; + + // Write external potential + ierr = h5f_file.write_1func_hdf5(v_ext_.data(), "Vext", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + return ierr; +} + template void Potentials::setVxc( const double* const vxc, const int iterativeIndex); template void Potentials::setVxc( diff --git a/src/Potentials.h b/src/Potentials.h index f0f762b3..6803edc3 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -10,6 +10,7 @@ #ifndef MGMOL_POTENTIALS_H #define MGMOL_POTENTIALS_H +#include "HDFrestart.h" #include "Rho.h" #include "TriCubic.h" @@ -25,7 +26,6 @@ class GridFunc; class Potentials { - int size_; int gdim_[3]; int dim_[3]; @@ -103,6 +103,12 @@ class Potentials void initializeRadialDataOnSampledPts( const Vector3D& position, const Species& sp, const std::vector &local_idx, std::vector &sampled_rhoc); + void rescaleRhoComp(); + + void addBackgroundToRhoComp(); + + void initBackground(); + public: Potentials(); @@ -142,54 +148,46 @@ class Potentials void turnOnDiel() { diel_ = true; } + int write(HDFrestart& h5f_file); + int read(HDFrestart& h5f_file); + int size() const { return size_; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return vtot_.data(); } - POTDTYPE* vh_rho() { return vh_rho_.data(); } RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return v_nuc_.data(); } - POTDTYPE* vext() { return v_ext_.data(); } + const std::vector& vh_rho() const { return vh_rho_; } + POTDTYPE* vepsilon() { return vepsilon_.data(); } - POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); - void axpVcomp(POTDTYPE* v, const double alpha); - - POTDTYPE vtot(const int i) { return vtot_[i]; } - POTDTYPE vh_rho(const int i) { return vh_rho_[i]; } - POTDTYPE vxc_rho(const int i) { return vxc_rho_[i]; } - POTDTYPE vepsilon(const int i) { return vepsilon_[i]; } bool diel() const { return diel_; } double getChargeInCell() const { return charge_in_cell_; } - /*! - * initialize total potential as local pseudopotential - */ - void initWithVnuc(); + const double getBackgroundCharge() const { return background_charge_; } void getVofRho(std::vector& vrho) const; /*! * evaluate potential correction associated with a new rho */ - double delta_v(const std::vector>& rho); + double computeDeltaV(const std::vector>& rho); /*! - * update potentials based on argument rho + * update total potential with updated components */ - double update(const std::vector>& rho); + double updateVtot(const std::vector>& rho); /*! * update potentials based on potential correction delta v and mixing * parameter */ - void update(const double mix); + void updateVtot(const double mix); double max() const; double min() const; @@ -197,19 +195,17 @@ class Potentials template void setVxc(const T* const vxc, const int iterativeIndex); - void setVh(const POTDTYPE* const vh, const int iterativeIndex); void setVh(const pb::GridFunc& vh, const int iterativeIndex); void initialize(Ions& ions); - void rescaleRhoComp(); - void initBackground(Ions& ions); - void addBackgroundToRhoComp(); /*! * Save current Hartree potential into backup array */ void backupVh(); + void resetVhRho2Backup() { vh_rho_ = vh_rho_backup_; } + void evalIonDensityOnSamplePts(Ions& ions, const std::vector &local_idx, std::vector &sampled_rhoc); #ifdef HAVE_TRICUBIC diff --git a/src/ProjectedMatricesSparse.cc b/src/ProjectedMatricesSparse.cc index 8b328b1c..15ad6d6c 100644 --- a/src/ProjectedMatricesSparse.cc +++ b/src/ProjectedMatricesSparse.cc @@ -354,7 +354,7 @@ double ProjectedMatricesSparse::dotProductSimple( << std::endl; MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); return -1.; } diff --git a/src/main.cc b/src/main.cc index 3fcbf791..3c6830a0 100644 --- a/src/main.cc +++ b/src/main.cc @@ -45,7 +45,7 @@ int main(int argc, char** argv) if (mpirc != MPI_SUCCESS) { std::cerr << "MPI Initialization failed!!!" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } MPI_Comm comm = MPI_COMM_WORLD; diff --git a/src/md.cc b/src/md.cc index 1d5ee937..f21fe542 100644 --- a/src/md.cc +++ b/src/md.cc @@ -59,7 +59,7 @@ void MGmol::moveVnuc(Ions& ions) // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); - initNuc(ions); + setupPotentials(ions); pot.axpVcompToVh(-1.); proj_matrices_->setHiterativeIndex(-1, -1); @@ -242,9 +242,12 @@ int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, std::string filename(std::string(ct.out_restart_file)); // add an integer corresponding to attempt number/count // to allow several attempts at creating and writing file - std::stringstream s; - s << count; - filename += s.str(); + if (count > 0) + { + std::stringstream s; + s << count; + filename += s.str(); + } HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); @@ -737,11 +740,29 @@ void MGmol::loadRestartFile(const std::string filename) if (ierr < 0) { if (onpe0) - (*MPIdata::serr) - << "loadRestartFile: failed to read the restart file." - << std::endl; + std::cerr << "loadRestartFile: failed to read the restart file." + << std::endl; - global_exit(0); + mmpi.abort(); + } + if (!ct.fullyOccupied()) + { + // overwrite DM with restart data in dataset Density_Matrix_WF + if (h5file.checkDataExists("Density_Matrix_WF")) + ierr = proj_matrices_->readWFDM(h5file); + } + + if (h5file.checkDataExists("Preceding_Hartree")) + { + ions_->readRestartPreviousPositions(h5file); + ions_->resetPositionsToPrevious(); + ions_->setup(); + + Potentials& pot = hamiltonian_->potential(); + pot.initialize(*ions_); + if (onpe0) std::cout << "Reset VhRho to backup..." << std::endl; + pot.resetVhRho2Backup(); + electrostat_->setupRhoc(pot.rho_comp()); } if (!ct.fullyOccupied()) { diff --git a/src/mgmol_memory.cc b/src/mgmol_memory.cc index a1360c99..b1be12d6 100644 --- a/src/mgmol_memory.cc +++ b/src/mgmol_memory.cc @@ -62,7 +62,7 @@ void addTrack(long addr, long asize) if (nPos > MAXNUMALLOCATIONS) { printf("ERROR: Not enough memory slots!!!"); - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } // if(onpe0)printf("nPos=%d, addr=%ld, size=%ld\n",nPos,addr,asize); } diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index a5de1ad5..468d61bd 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -96,7 +96,7 @@ int mgmol_check() { std::cerr << "Code should be called with " << myPEenv.n_mpi_tasks() << " MPI tasks only" << std::endl; - ct.global_exit(2); + ct.global_exit(); } assert(ct.getMGlevels() >= -1); diff --git a/src/pb/Lap.h b/src/pb/Lap.h index 33d90706..460e36c1 100644 --- a/src/pb/Lap.h +++ b/src/pb/Lap.h @@ -35,7 +35,7 @@ class Lap : public FDoper virtual void applyWithPot(GridFunc&, const double* const, T*) { std::cerr << "ERROR: Lap::applyWithPot() not implemented" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } std::string name() const { return name_; } diff --git a/src/pb/PEenv.cc b/src/pb/PEenv.cc index d5ff78f3..3e2fb777 100644 --- a/src/pb/PEenv.cc +++ b/src/pb/PEenv.cc @@ -197,7 +197,7 @@ PEenv::~PEenv() if (mpirc != MPI_SUCCESS) { std::cerr << "MPI_Comm_free failed!" << std::endl; - MPI_Abort(comm_, 2); + MPI_Abort(comm_, EXIT_FAILURE); } } if (cart_comm_ != MPI_COMM_NULL) MPI_Comm_free(&cart_comm_); @@ -277,7 +277,7 @@ void PEenv::task2xyz() if (rc != MPI_SUCCESS) { std::cerr << " error in MPI_Cart_coords()!!!" << std::endl; - MPI_Abort(comm_, 1); + MPI_Abort(comm_, EXIT_FAILURE); } #else mytask_dir_[2] = mytask_ % n_mpi_tasks_dir_[2]; @@ -650,7 +650,7 @@ void PEenv::split_comm(const int nx, const int ny, const int nz, const int bias) { std::cerr << "MPI_Comm_split failed!, my color_=" << color_ << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } MPI_Comm_size(comm_active_, &n_mpi_tasks_); #ifndef NDEBUG @@ -696,7 +696,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Recv() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } else if (ip == mytask_) @@ -708,7 +708,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Send() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } if (mytask_ == 0) @@ -717,7 +717,7 @@ void PEenv::printPEnames(std::ostream& os) const if (mytask_ == 0) os << std::endl; } -void PEenv::globalExit() const { MPI_Abort(comm_, 2); } +void PEenv::globalExit() const { MPI_Abort(comm_, EXIT_FAILURE); } void PEenv::bcast(int* val, const int n) const { diff --git a/src/pb/PEenv.h b/src/pb/PEenv.h index bc4751a5..9782ddb9 100644 --- a/src/pb/PEenv.h +++ b/src/pb/PEenv.h @@ -166,7 +166,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxXdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -181,7 +181,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxYdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -196,7 +196,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxZdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } diff --git a/src/quench.cc b/src/quench.cc index 59a16d50..073798f6 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -545,7 +545,6 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, // get actual indexes of stored functions const std::vector>& gids(orbitals.getOverlappingGids()); - g_kbpsi_->setup(*ions_); electrostat_->setup(ct.vh_its); rho_->setup(ct.getOrthoType(), gids); diff --git a/src/radial/RadialMeshFunction.cc b/src/radial/RadialMeshFunction.cc index 499e482a..79b32518 100644 --- a/src/radial/RadialMeshFunction.cc +++ b/src/radial/RadialMeshFunction.cc @@ -98,7 +98,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int i = 0; i < nn[1]; i++) @@ -110,7 +110,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } } } diff --git a/src/restart.cc b/src/restart.cc index 30bdad73..532043af 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -43,27 +43,11 @@ int MGmol::read_rho_and_pot_hdf5( os_ << "Try to read density and potentials" << std::endl; Potentials& pot = hamiltonian_->potential(); + pot.read(file); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - POTDTYPE* tmp = new POTDTYPE[mygrid.size()]; - - // Read total potential - file.read_1func_hdf5(pot.vtot(), "Vtotal"); - - // Read the hartree potential - file.read_1func_hdf5(tmp, "Hartree"); - pot.setVh(tmp, 0); - - // Read dielectric potential - if (ct.diel) - { - file.read_1func_hdf5(pot.vepsilon(), "VDielectric"); - } // Read the Density rho.readRestart(file); - delete[] tmp; return 0; } @@ -118,6 +102,8 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + ions.writePreviousPositions(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); @@ -131,41 +117,12 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, if (ct.out_restart_info > 1) { - // Write total potential - int ierr = h5f_file.write_1func_hdf5( - pot.vtot(), "Vtotal", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - // Write the hartree potential - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho(), "Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - if (ct.AtomsDynamic() == AtomsDynamicType::MD) - { - // Write hartree potential before extrapolation - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - } - - // Write - if (ct.diel) - { - ierr = h5f_file.write_1func_hdf5( - pot.vepsilon(), "VDielectric", &ll[0], &origin[0]); - } - if (ierr < 0) return ierr; + int ierr = pot.write(h5f_file); // Write the Density ierr = h5f_file.write_1func_hdf5( &rho[0][0], "Density", &ll[0], &origin[0]); if (ierr < 0) return ierr; - - // Write external potential - ierr - = h5f_file.write_1func_hdf5(pot.vext(), "Vext", &ll[0], &origin[0]); - if (ierr < 0) return ierr; } // Write wavefunctions and old centers. diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 52ad6e47..8fbd4cb3 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -103,7 +103,7 @@ void readRestartFiles(MGmolInterface *mgmol_) break; default: - ct.global_exit(-1); + ct.global_exit(); break; } @@ -129,7 +129,8 @@ void readRestartFiles(MGmolInterface *mgmol_) case ROMVariable::POTENTIAL: basis_prefix += "_potential"; /* we save hartree potential */ - basis_generator.takeSample(pot.vh_rho()); + // TODO: consider create overloaded takeSample with const + basis_generator.takeSample(const_cast(pot.vh_rho().data())); break; } } diff --git a/src/setup.cc b/src/setup.cc index 9963bfc5..2ff6d79e 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -140,7 +140,7 @@ int MGmol::setupLRsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { @@ -173,7 +173,7 @@ int MGmol::setupConstraintsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { diff --git a/src/sparse_linear_algebra/DataDistribution.cc b/src/sparse_linear_algebra/DataDistribution.cc index d4b26d93..67bb1319 100644 --- a/src/sparse_linear_algebra/DataDistribution.cc +++ b/src/sparse_linear_algebra/DataDistribution.cc @@ -288,7 +288,7 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, std::cout << "ERROR: " << name_ << ", dir=" << dir << ", remote_size=" << remote_size << ", bsiz=" << bsiz << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } // string stamp="DataDistribution ("+name_+"), buffer size checked..."; // printWithTimeStamp(stamp,cout); @@ -302,14 +302,14 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, if (mpircv != MPI_SUCCESS) { std::cout << "ERROR in MPI_Irecv, code=" << mpircv << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } int mpisnd = MPI_Isend(packed_buffer.sendBuffer(), siz, MPI_CHAR, dest, 0, cart_comm_, &request[1]); if (mpisnd != MPI_SUCCESS) { std::cout << "ERROR in MPI_Isend, code=" << mpisnd << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } /* wait to complete communication */ MPI_Waitall(2, request, MPI_STATUSES_IGNORE); diff --git a/src/tools.cc b/src/tools.cc index 95790c59..aff34933 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -29,7 +29,7 @@ void noMoreMemory() std::cerr << "Unable to satisfy request for memory for MPI task " << mype << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(3); + ct.global_exit(); } // an atom name should start with a capital letter and end with a number @@ -229,12 +229,12 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) if( mpierr!=MPI_SUCCESS ) { cerr << " Error in MPI!!!" << std::endl; - MPI_Abort(mmpi.commGlobal(),1); + MPI_Abort(mmpi.commGlobal(),EXIT_FAILURE); } if( r!=mmpi.size()*s && onpe0 ) { cerr << " Error in barrier: "< #include #include +#include #include // full lower triangular part of symmetric matrix with compact storage diff --git a/src/tools/Vector3D.cc b/src/tools/Vector3D.cc index 445f8787..2fe9a0e4 100644 --- a/src/tools/Vector3D.cc +++ b/src/tools/Vector3D.cc @@ -80,7 +80,7 @@ void bcastvv3d(std::vector& vv, MPI_Comm comm) std::cerr << "ERROR!!!! bcastvv3d(), Failure in MPI_Bcast of 'radii_'!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int j = 0; j < n; j++) for (short i = 0; i < 3; i++) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6b5514d2..5141a887 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,6 +250,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +add_executable(testRhoVhRestart + ${CMAKE_SOURCE_DIR}/tests/RhoVhRestart/testRhoVhRestart.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) add_executable(testWFEnergyAndForces @@ -392,6 +394,17 @@ add_test(NAME testPinnedH2O_3DOF ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRhoVhRestart + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRhoVhRestart + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) + if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -587,6 +600,7 @@ target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) +target_include_directories(testRhoVhRestart PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -600,6 +614,7 @@ target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testPinnedH2O_3DOF PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) +target_link_libraries(testRhoVhRestart mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/DMandEnergyAndForces/mgmol.cfg b/tests/DMandEnergyAndForces/mgmol.cfg index 5278f40b..e7712706 100644 --- a/tests/DMandEnergyAndForces/mgmol.cfg +++ b/tests/DMandEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=LDA FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/EnergyAndForces/mgmol.cfg b/tests/EnergyAndForces/mgmol.cfg index cf23889e..d1299db9 100644 --- a/tests/EnergyAndForces/mgmol.cfg +++ b/tests/EnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=1 xcFunctional=LDA FDtype=Mehrstellen [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg index 1ff2adab..8d6b21e2 100644 --- a/tests/HDF5single/md.cfg +++ b/tests/HDF5single/md.cfg @@ -25,10 +25,10 @@ max_steps=24 atol=1.e-8 [Restart] input_level=4 -input_filename=WF +input_filename=wf.h5 input_type=single_file output_level=4 -output_filename=WF_MD +output_filename=wf_md.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg index 4dba942a..e8f01a29 100644 --- a/tests/HDF5single/mgmol.cfg +++ b/tests/HDF5single/mgmol.cfg @@ -25,7 +25,7 @@ initial_type=Random initial_width=1.5 [Restart] output_level=4 -output_filename=WF +output_filename=wf.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py index 080ee0ba..e2f0596a 100755 --- a/tests/HDF5single/test.py +++ b/tests/HDF5single/test.py @@ -47,7 +47,7 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') -os.remove('WF') +os.remove('wf.h5') print("Check energy conservation...") tol = 1.e-4 @@ -71,5 +71,7 @@ print("ERROR needs 4 energy values for checking conservation!") sys.exit(1) +os.remove('wf_md.h5') + print("Test SUCCESSFUL!") sys.exit(0) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index 17b70de2..3f98b2bd 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg index 83d5aa4c..e30a8807 100644 --- a/tests/MD_MVP/md.cfg +++ b/tests/MD_MVP/md.cfg @@ -34,6 +34,6 @@ solver=MVP nb_inner_it=1 mixing=1. [Restart] -input_filename=wave.out +input_filename=snapshotMVP input_level=3 output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py index 489c9c8c..47b5462d 100755 --- a/tests/MD_MVP/test.py +++ b/tests/MD_MVP/test.py @@ -42,20 +42,20 @@ #run MD for i in range(2): - command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) - os.rename(restart_file, 'wave.out') + os.rename(restart_file, 'snapshotMVP') #run MGmol command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) #remove used restart files - shutil.rmtree('wave.out') + shutil.rmtree('snapshotMVP') #analyse mgmol standard output lines=output2.split(b'\n') @@ -81,7 +81,7 @@ sys.exit(1) #remove last restart files -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') shutil.rmtree(restart_file) diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz index cdc906f6..1616f79c 100644 --- a/tests/RestartEnergyAndForces/h2o.xyz +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -1,6 +1,5 @@ 3 -https://pubchem.ncbi.nlm.nih.gov/compound/Water -O 2.5369 -0.1550 0.0 -H 3.0739 0.1550 0.0 -H 2.0000 0.1550 0.0 +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg index e590f810..b96f3d9c 100644 --- a/tests/RestartEnergyAndForces/mgmol.cfg +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg index 99bc77d8..280778e9 100644 --- a/tests/RestartEnergyAndForces/restart.cfg +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index b62d39f8..585722f0 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -64,7 +64,13 @@ shutil.rmtree('WF') test_energy=1.e18 +l=-1 for line in lines: + if line.count(b'Positions'): + l=0 + if l>=0 and l<4: + print(line) + l=l+1 if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index e323afbc..3027ab55 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,6 +124,8 @@ int main(int argc, char** argv) } } + mgmol->setPositions(positions, anumbers); + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -165,6 +167,36 @@ int main(int argc, char** argv) projmatrices->setDMuniform(ct.getNelSpin(), 0); projmatrices->printDM(std::cout); + // swap H and O to make sure order of atoms in list does not matter + double x = positions[0]; + double y = positions[1]; + double z = positions[2]; + positions[0] = positions[3]; + positions[1] = positions[4]; + positions[2] = positions[5]; + positions[3] = x; + positions[4] = y; + positions[5] = z; + short tmp = anumbers[0]; + anumbers[0] = anumbers[1]; + anumbers[1] = tmp; + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + mgmol->setPositions(positions, anumbers); + // // evaluate energy and forces with wavefunctions just read // diff --git a/tests/RhoVhRestart/h2o.xyz b/tests/RhoVhRestart/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/RhoVhRestart/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/RhoVhRestart/md.cfg b/tests/RhoVhRestart/md.cfg new file mode 100644 index 00000000..2b8a378b --- /dev/null +++ b/tests/RhoVhRestart/md.cfg @@ -0,0 +1,30 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +output_level=4 +output_filename=WF_MD diff --git a/tests/RhoVhRestart/mgmol.cfg b/tests/RhoVhRestart/mgmol.cfg new file mode 100644 index 00000000..eee7f11c --- /dev/null +++ b/tests/RhoVhRestart/mgmol.cfg @@ -0,0 +1,28 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF diff --git a/tests/RhoVhRestart/restart.cfg b/tests/RhoVhRestart/restart.cfg new file mode 100644 index 00000000..20f0293a --- /dev/null +++ b/tests/RhoVhRestart/restart.cfg @@ -0,0 +1,25 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF_MD diff --git a/tests/RhoVhRestart/test.py b/tests/RhoVhRestart/test.py new file mode 100755 index 00000000..a34b962f --- /dev/null +++ b/tests/RhoVhRestart/test.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-7] +test_exe = sys.argv[nargs-6] +input1 = sys.argv[nargs-5] +input2 = sys.argv[nargs-4] +input3 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("Initial quench failed to complete!") + sys.exit(1) + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("MD failed to complete!") + sys.exit(1) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input3,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') +for line in lines: + print(line) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc new file mode 100644 index 00000000..09da3320 --- /dev/null +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -0,0 +1,220 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "Electrostatic.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Poisson.h" +#include "Potentials.h" +#include "mgmol_run.h" + +#include +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +template +int testRhoRestart(MGmolInterface* mgmol_) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + std::shared_ptr> rho = mgmol->getRho(); + + /* save density from the restart file to elsewhere */ + std::vector rho0(rho->rho_[0].size()); + rho0 = rho->rho_[0]; + + /* recompute rho from the orbital */ + rho->update(*mgmol->getOrbitals()); + + /* check if the recomputed density is the same */ + for (int d = 0; d < (int)rho0.size(); d++) + { + double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); + if (error > 1e-10) + { + printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, + rho->rho_[0][d], d, rho0[d]); + std::cerr << "Density is inconsistent!!!" << std::endl; + return -1; + } + } + if (rank == 0) std::cout << "Density is consistent..." << std::endl; + + return 0; +} + +template +int testPotRestart(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + Potentials& pot = mgmol->getHamiltonian()->potential(); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + std::shared_ptr> rho = mgmol->getRho(); + + /* GridFunc initialization inputs */ + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = ct.bcPoisson[d]; + + /* save potential from the restart file to elsewhere */ + pb::GridFunc vh0_gf(mygrid, bc[0], bc[1], bc[2]); + vh0_gf.assign((pot.vh_rho()).data(), 'd'); + double n = vh0_gf.norm2(); + std::cout << "Norm2 of vh = " << n << std::endl; + + std::vector vh0(pot.size()); + const std::vector& d_vhrho(pot.vh_rho()); + for (int d = 0; d < (int)vh0.size(); d++) + vh0[d] = d_vhrho[d]; + + /* recompute potential */ + pb::GridFunc grho(mygrid, bc[0], bc[1], bc[2]); + grho.assign(&rho->rho_[0][0]); + pb::GridFunc* grhoc = mgmol->electrostat_->getRhoc(); + + poisson->solve(grho, *grhoc); + const pb::GridFunc& vh(poisson->vh()); + + pb::GridFunc error_gf(vh0_gf); + error_gf -= vh; + + double rel_error = error_gf.norm2() / vh0_gf.norm2(); + if (rank == 0) + { + printf("FOM potential relative error: %.3e\n", rel_error); + } + if (rel_error > 1e-9) + { + if (rank == 0) + { + std::cerr << "Potential is inconsistent!!!" << std::endl; + } + return -1; + } + if (rank == 0) std::cout << "Potential is consistent..." << std::endl; + + return 0; +} + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + int status = 0; + + // Enter main scope + { + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + mgmol->setup(); + + /* load a restart file */ + MGmol* mgmol_ext + = dynamic_cast*>(mgmol); + mgmol_ext->loadRestartFile(ct.restart_file); + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testRhoRestart..." << std::endl; + status = testRhoRestart(mgmol); + if (status < 0) return status; + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testPotRestart..." << std::endl; + status = testPotRestart(mgmol); + if (status < 0) return status; + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 27da7f7d..68a617aa 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') diff --git a/tests/testIons.cc b/tests/testIons.cc index 30b2ad0c..e23e0e24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -8,6 +8,8 @@ int main(int argc, char** argv) { + int status = 0; + int mpirc = MPI_Init(&argc, &argv); MPI_Comm comm = MPI_COMM_WORLD; @@ -41,7 +43,7 @@ int main(int argc, char** argv) // read species info from pseudopotential file std::string file_path = argv[1]; std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); - std::cout << "Potential = " << filename << std::endl; + if (myrank == 0) std::cout << "Potential = " << filename << std::endl; sp.read_1species(filename); sp.set_dim_nl(h[0]); @@ -80,25 +82,110 @@ int main(int argc, char** argv) ions.setup(); - std::vector& new_local_ions(ions.local_ions()); + // verify sum of local ions adds up to total number of ions + { + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + status = 1; + } + } + MPI_Barrier(MPI_COMM_WORLD); - int nlocal = new_local_ions.size(); - std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + // verify some functionalities of class Ions + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + ions.getAtomicNumbers(anumbers); + if (myrank == 0) + { + int i = 0; + for (auto& position : positions) + { + std::cout << position; + if (i % 3 == 2) + std::cout << std::endl; + else + std::cout << " "; + i++; + } + } + MPI_Barrier(MPI_COMM_WORLD); + + // swap x and z + for (size_t i = 0; i < positions.size() - 2; i++) + { + double x = positions[i]; + double z = positions[i + 2]; + positions[i] = z; + positions[i + 2] = x; + } + + ions.setPositions(positions, anumbers); + } - int ntotal = 0; - MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); - mpirc = MPI_Finalize(); - if (mpirc != MPI_SUCCESS) + MPI_Barrier(MPI_COMM_WORLD); { - std::cerr << "MPI Finalize failed!!!" << std::endl; - return 1; + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cerr << "ntotal = " << ntotal << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } } - if (ntotal != na) + // get the names of all the ions + std::vector names; + ions.getNames(names); + if (myrank == 0) + for (auto& name : names) + std::cout << "name = " << name << std::endl; + if (names.size() != na) { - std::cout << "ntotal = " << ntotal << std::endl; - return 1; + std::cerr << "Incorrect count of names..." << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + MPI_Barrier(MPI_COMM_WORLD); + + std::vector forces(3 * na); + // arbitrary value + const double fval = 1.12; + for (auto& f : forces) + f = fval; + ions.setLocalForces(forces, names); + + int nlocal = ions.getNumLocIons(); + std::vector lforces(3 * nlocal); + ions.getLocalForces(lforces); + for (auto& f : lforces) + { + if (std::abs(f - fval) > 1.e-14) + { + std::cerr << "f = " << f << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + status = 1; } - return 0; + return status; } From 893c637d26fd3190cdfea7a84ecac94581301be2 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 27 Mar 2025 13:33:32 -0700 Subject: [PATCH 40/50] Merge release (#327) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- CMakeLists.txt | 15 +- scripts/build_ubuntu22_mixedp.sh | 32 ++ src/BlockVector.cc | 4 +- src/ExtendedGridOrbitals.cc | 29 +- src/Ion.cc | 2 + src/Ion.h | 2 + src/Ions.cc | 109 ++---- src/Ions.h | 31 +- src/LocGridOrbitals.cc | 14 +- src/MGmol.cc | 2 + src/Rho.cc | 2 +- src/global.h | 2 +- src/linear_algebra/CMakeLists.txt | 2 +- src/linear_algebra/MGmol_blas1.h | 14 +- src/linear_algebra/gemm_impl.cc | 251 ++++++++++++ src/linear_algebra/gemm_impl.h | 14 + src/linear_algebra/mputils.cc | 550 +++++++-------------------- src/linear_algebra/mputils.h | 23 +- src/linear_algebra/syrk_impl.cc | 227 +++++++++++ src/linear_algebra/syrk_impl.h | 13 + src/md.cc | 6 - src/mgmol_run.cc | 8 + tests/CMakeLists.txt | 30 ++ tests/RestartEnergyAndForces/h2o.xyz | 1 + tests/testIons.cc | 4 + 25 files changed, 835 insertions(+), 552 deletions(-) create mode 100755 scripts/build_ubuntu22_mixedp.sh create mode 100644 src/linear_algebra/gemm_impl.cc create mode 100644 src/linear_algebra/gemm_impl.h create mode 100644 src/linear_algebra/syrk_impl.cc create mode 100644 src/linear_algebra/syrk_impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c48a2e9..fb9f80fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,12 @@ else() find_package(OpenMP) endif() +# Use mixed precision +set(MGMOL_WITH_MP FALSE CACHE BOOL "Compile with mixed precision") +if(MGMOL_WITH_MP) + add_definitions(-DMGMOL_USE_MIXEDP) +endif() + # HDF5 (required) set(MGMOL_USE_HDF5P TRUE CACHE BOOL "Use HDF5 parallel capability") if(HDF5_LIBRARIES) #user sets libraries explicitly @@ -100,9 +106,16 @@ if(NOT BLAS_LIBRARIES) message(FATAL_ERROR "Required blas library not found.") endif(NOT ${BLAS_FOUND}) else(NOT BLAS_LIBRARIES) - message(STATUS "BLAS_LIBARIES: ${BLAS_LIBRARIES}") + message(STATUS "BLAS_LIBARIES set to ${BLAS_LIBRARIES}") endif(NOT BLAS_LIBRARIES) +set(MGMOL_WITH_BLIS FALSE CACHE BOOL "Use BLIS library") +if(MGMOL_WITH_BLIS) + add_definitions(-DMGMOL_USE_BLIS) + message(STATUS "BLIS_LIBRARIES include: ${BLIS_INCLUDE_DIRS}") + include_directories("${BLIS_INCLUDE_DIRS}") +endif() + if(NOT LAPACK_LIBRARIES) find_package(LAPACK REQUIRED) message(STATUS "LAPACK_LIBARIES: ${LAPACK_LIBRARIES}") diff --git a/scripts/build_ubuntu22_mixedp.sh b/scripts/build_ubuntu22_mixedp.sh new file mode 100755 index 00000000..d85ab9a5 --- /dev/null +++ b/scripts/build_ubuntu22_mixedp.sh @@ -0,0 +1,32 @@ +#/bin/bash +MGMOL_ROOT=`pwd` + +INSTALL_DIR=${MGMOL_ROOT}/mgmol_install +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_COMPILER=mpiCC.openmpi \ + -DMGMOL_WITH_BLIS=ON \ + -DBLAS_LIBRARIES=/home/q8j/blis/lib/libblis.so \ + -DBLIS_INCLUDE_DIRS="/home/q8j/blis//include" \ + -DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/openblas-pthread/liblapack.so \ + -DCMAKE_Fortran_COMPILER=mpif77.openmpi \ + -DSCALAPACK_LIBRARY=/usr/lib/x86_64-linux-gnu/libscalapack-openmpi.so.2.1 \ + -DMPIEXEC_EXECUTABLE=/usr/bin/mpirun \ + -DMPIEXEC_NUMPROC_FLAG="-np" \ + -DMPIEXEC_PREFLAGS="--oversubscribe" \ + -DMGMOL_WITH_CLANG_FORMAT=ON \ + -DMGMOL_WITH_MP=ON \ + -DCMAKE_PREFIX_PATH=${HOME}/bin \ + -D CMAKE_CXX_FLAGS="-Wall" \ + .. + +# call make install +make -j 4 +make install diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 9a411ebe..671ba90d 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -556,7 +556,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); @@ -586,7 +586,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 07face7c..3e887dad 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -926,10 +926,10 @@ void ExtendedGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(numst_, nf, loc_numpt_, 1., - orbitals_psi_host_view + iloc * loc_numpt_, lda_, - work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * numst_, - numst_); + LinearAlgebraUtils::MPgemmTN(numst_, nf, + loc_numpt_, 1., orbitals_psi_host_view + iloc * loc_numpt_, + lda_, work + iloc * loc_numpt_, lda_, 0., + ssiloc + icolor * numst_, numst_); } } @@ -970,7 +970,7 @@ void ExtendedGridOrbitals::getLocalOverlap( if (numst_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else ORBDTYPE* psi = block_vector_.vect(0); @@ -1041,21 +1041,12 @@ void ExtendedGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, const int lda = transpose ? ld : lda_; const int ldb = transpose ? lda_ : ld; -#ifdef USE_MP - // use temporary float data for matrix ss - LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); -#else - LocalMatrices& ssf(ss); -#endif for (short iloc = 0; iloc < subdivx_; iloc++) { - LinearAlgebraUtils::MPgemm('T', 'N', numst_, numst_, + LinearAlgebraUtils::MPgemmTN(numst_, numst_, loc_numpt_, 1., a + iloc * loc_numpt_, lda, b + +iloc * loc_numpt_, - ldb, 0., ssf.getRawPtr(iloc), ssf.m()); + ldb, 0., ss.getRawPtr(iloc), ss.m()); } -#ifdef USE_MP - ss.copy(ssf); -#endif ss.scal(grid_.vel()); } @@ -1709,9 +1700,9 @@ void ExtendedGridOrbitals::addDotWithNcol2Matrix( Apsi.getPsi(0, iloc), phi_size, phi_host_view); // TODO this can be done on the GPU - MPgemmTN(numst_, numst_, loc_numpt_, vel, - block_vector_host_view + iloc * loc_numpt_, lda_, phi_host_view, - lda_, 1., work.data(), numst_); + LinearAlgebraUtils::MPgemmTN(numst_, numst_, + loc_numpt_, vel, block_vector_host_view + iloc * loc_numpt_, lda_, + phi_host_view, lda_, 1., work.data(), numst_); MemorySpace::Memory::free_host_view( phi_host_view); diff --git a/src/Ion.cc b/src/Ion.cc index 90350477..896a1961 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -24,6 +24,8 @@ unsigned short isqrt(unsigned value) static unsigned int _nlproj_gid = 0; static unsigned int _index = 0; +void Ion::resetIndexCount() { _index = 0; } + Ion::Ion(const Species& species, const std::string& name, const double crds[3], const double velocity[3], const bool lock) : name_(name), species_(species), index_(_index), nlproj_gid_(_nlproj_gid) diff --git a/src/Ion.h b/src/Ion.h index 871205a3..cf52db97 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -314,6 +314,8 @@ class Ion void getKBcoeffs(std::vector& coeffs); double energyDiff( Ion& ion, const double lattice[3], const short bc[3]) const; + + static void resetIndexCount(); }; #endif diff --git a/src/Ions.cc b/src/Ions.cc index 10baad10..f10fb834 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -9,7 +9,6 @@ #include "Ions.h" #include "Control.h" -#include "HDFrestart.h" #include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "MPIdata.h" @@ -20,13 +19,10 @@ #include "mgmol_mpi_tools.h" #include "tools.h" -#include - #include #include #include -#include -#include +#include Timer ions_setupInteractingIons_tm("ions_setupInteractingIons"); Timer ions_setup_tm("ions::setup"); @@ -187,6 +183,8 @@ void Ions::setup() ions_setup_tm.start(); + Ion::resetIndexCount(); + updateListIons(); //#ifndef NDEBUG @@ -394,7 +392,7 @@ void Ions::iiforce(const short bc[3]) std::vector::const_iterator ion1 = local_ions_.begin(); int ion1_index = 0; - ; + while (ion1 != local_ions_.end()) { const double z1 = (*ion1)->getZion(); @@ -1800,7 +1798,7 @@ int Ions::readAtomsFromXYZ( } int Ions::setAtoms( - const std::vector& crds, const std::vector& spec) + const std::vector& crds, const std::vector& anum) { const int natoms = crds.size() / 3; @@ -1813,7 +1811,7 @@ int Ions::setAtoms( while (it != species_.end()) { ++isp; - if (it->getAtomicNumber() == spec[ia]) + if (it->getAtomicNumber() == anum[ia]) { break; } @@ -1823,7 +1821,7 @@ int Ions::setAtoms( for (std::map::iterator itr = map_species_.begin(); itr != map_species_.end(); ++itr) { - if (itr->second == spec[ia]) + if (itr->second == anum[ia]) { spname = itr->first; break; @@ -1833,7 +1831,7 @@ int Ions::setAtoms( { (*MPIdata::serr) << "Ions::setAtoms() --- ERROR: unknown " "species for atomic number " - << spec[ia] << std::endl; + << anum[ia] << std::endl; return -1; } @@ -2212,6 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { + names.clear(); for (auto& ion : local_ions_) { names.push_back(ion->name()); @@ -2220,6 +2219,7 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { + names.clear(); for (auto& ion : list_ions_) { names.push_back(ion->name()); @@ -2513,7 +2513,6 @@ void Ions::gatherLockedNames( std::vector& names, const int root, const MPI_Comm comm) const { std::vector local_names; - for (auto& ion : local_ions_) { if (ion->locked()) local_names.push_back(ion->name()); @@ -2533,7 +2532,6 @@ void Ions::gatherIndexes( std::vector& indexes, const int root, const MPI_Comm comm) const { std::vector local_indexes; - for (auto& ion : local_ions_) { local_indexes.push_back(ion->index()); @@ -2553,7 +2551,6 @@ void Ions::gatherNLprojIds( std::vector& nlprojids, const int root, const MPI_Comm comm) const { std::vector local_nlprojids; - for (auto& ion : local_ions_) { local_nlprojids.push_back(ion->nlprojid()); @@ -2573,7 +2570,6 @@ void Ions::gatherAtomicNumbers( std::vector& atnumbers, const int root, const MPI_Comm comm) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2593,7 +2589,6 @@ void Ions::gatherRandStates(std::vector& rstates, const int root, const MPI_Comm comm) const { std::vector local_rstates; - for (auto& ion : local_ions_) { local_rstates.push_back(ion->randomState(0)); @@ -2615,10 +2610,8 @@ void Ions::gatherPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { - // get position of local ion double position[3]; ion->getPosition(&position[0]); local_positions.push_back(position[0]); @@ -2640,7 +2633,6 @@ void Ions::gatherPreviousPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { local_positions.push_back(ion->getPreviousPosition(0)); @@ -2662,17 +2654,14 @@ void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { std::vector local_forces; - for (auto& ion : local_ions_) { - // get position of local ion double force[3]; ion->getForce(&force[0]); local_forces.push_back(force[0]); local_forces.push_back(force[1]); local_forces.push_back(force[2]); } - // gather data to PE root std::vector data; mgmol_tools::gatherV(local_forces, data, root, comm); @@ -2687,7 +2676,6 @@ void Ions::gatherVelocities( std::vector& velocities, const int root, const MPI_Comm comm) const { std::vector local_velocities; - for (auto& ion : local_ions_) { local_velocities.push_back(ion->velocity(0)); @@ -2711,13 +2699,10 @@ void Ions::gatherPositions(std::vector& positions, const int root) const positions.resize(3 * num_ions_, 0.); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local positions - const int index = (*ion)->index(); - (*ion)->getPosition(&positions[3 * index]); - ++ion; + const int index = ion->index(); + ion->getPosition(&positions[3 * index]); } // gather data to PE root @@ -2726,17 +2711,17 @@ void Ions::gatherPositions(std::vector& positions, const int root) const void Ions::gatherForces(std::vector& forces, const int root) const { + assert(num_ions_ > 0); + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); forces.resize(3 * num_ions_, 0.); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local forces - const int index = (*ion)->index(); - (*ion)->getForce(&forces[3 * index]); - ion++; + const int index = ion->index(); + assert(forces.size() >= 3 * index); + assert(index < num_ions_); + ion->getForce(&forces[3 * index]); } // gather data to PE root @@ -2749,15 +2734,10 @@ void Ions::gatherLockedData(std::vector& locked_data, const int root) const MGmol_MPI& mmpi(*(MGmol_MPI::instance())); locked_data.resize(num_ions_, 0); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local ion index - const int index = (*ion)->index(); - if ((*ion)->locked()) locked_data[index] = 1; - - ++ion; + const int index = ion->index(); + if (ion->locked()) locked_data[index] = 1; } // gather data to PE root @@ -3214,11 +3194,9 @@ void Ions::updateTaupInteractingIons() void Ions::clearLists() { local_ions_.clear(); - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - delete *ion; - ion++; + delete ion; } list_ions_.clear(); } @@ -3335,24 +3313,21 @@ void Ions::initStepperData() { clearStepperData(); - std::vector::iterator lion = local_ions_.begin(); - while (lion != local_ions_.end()) + for (auto& lion : local_ions_) { - local_names_.push_back((*lion)->name()); - atmove_.push_back(!(*lion)->locked()); - pmass_.push_back((*lion)->getMass()); - gids_.push_back((*lion)->index()); + local_names_.push_back(lion->name()); + atmove_.push_back(!lion->locked()); + pmass_.push_back(lion->getMass()); + gids_.push_back(lion->index()); for (short i = 0; i < 3; i++) { - taum_.push_back((*lion)->getPreviousPosition(i)); - tau0_.push_back((*lion)->position(i)); - fion_.push_back((*lion)->force(i)); - velocity_.push_back((*lion)->velocity(i)); - rand_states_.push_back((*lion)->randomState(i)); + taum_.push_back(lion->getPreviousPosition(i)); + tau0_.push_back(lion->position(i)); + fion_.push_back(lion->force(i)); + velocity_.push_back(lion->velocity(i)); + rand_states_.push_back(lion->randomState(i)); } - - lion++; } // initialize taup to enable computing velocities int size_tau = (int)tau0_.size(); @@ -3399,11 +3374,9 @@ void Ions::updateIons() void Ions::shiftIons(const Vector3D& shift) { // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->shiftPositionXLBOMDTest(shift); - ion++; + ion->shiftPositionXLBOMDTest(shift); } // update various list of ions @@ -3421,12 +3394,10 @@ void Ions::rescaleVelocities(const double factor) std::cout << "Ions::rescaleVelocities() with factor " << factor << std::endl; } - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - (*ion)->rescaleVelocity(factor); - ion++; + for (auto& ion : local_ions_) + { + ion->rescaleVelocity(factor); } } diff --git a/src/Ions.h b/src/Ions.h index 33b8982b..3420d725 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -10,20 +10,20 @@ #ifndef MGMOL_IONS_H #define MGMOL_IONS_H -#include -#include -#include -#include -#include - #include "DistributedIonicData.h" +#include "HDFrestart.h" #include "Ion.h" -#include "hdf5.h" -class HDFrestart; +#include +#include +#include class Ions { +private: + /*! + * map species to atomic numbers + */ static std::map map_species_; static int num_ions_; @@ -37,7 +37,7 @@ class Ions std::vector list_ions_; - /* + /*! * ions located in local sub-domain */ std::vector local_ions_; @@ -61,6 +61,13 @@ class Ions bool has_locked_atoms_; + /*! + * Prevent usage of copy constructor by making it private and + * non-implemented + */ + Ions(const Ions&); + void operator=(const Ions&); + void readRestartVelocities(HDFrestart& h5_file); void readRestartRandomStates(HDFrestart& h5_file); void readRestartPositions(HDFrestart& h5_file); @@ -193,11 +200,9 @@ class Ions std::vector& getGids() { return gids_; } void resetForces() { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->resetForce(); - ion++; + ion->resetForce(); } } void resetPositionsToPrevious(); diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index 4a1b9442..3d14e051 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -1448,8 +1448,8 @@ void LocGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(chromatic_number_, nf, loc_numpt_, 1., - orbitals_psi + iloc * loc_numpt_, lda_, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + nf, loc_numpt_, 1., orbitals_psi + iloc * loc_numpt_, lda_, work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * chromatic_number_, chromatic_number_); } @@ -1490,7 +1490,7 @@ void LocGridOrbitals::getLocalOverlap( if (chromatic_number_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else const ORBDTYPE* const psi = block_vector_.vect(0); @@ -1560,7 +1560,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, MemorySpace::Memory::copy_view_to_host( const_cast(b), b_size, b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP // use temporary float data for matrix ss LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); #else @@ -1575,7 +1575,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, a_host_view); MemorySpace::Memory::free_host_view( b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP ss.copy(ssf); #endif @@ -2495,8 +2495,8 @@ void LocGridOrbitals::addDotWithNcol2Matrix( { MATDTYPE* ssiloc = ss.getRawPtr(iloc); - // TODO - MPgemmTN(chromatic_number_, chromatic_number_, loc_numpt_, vel, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + chromatic_number_, loc_numpt_, vel, block_vector_.vect(0) + iloc * loc_numpt_, lda_, Apsi.getPsi(0, iloc), lda_, 0., ssiloc, chromatic_number_); } diff --git a/src/MGmol.cc b/src/MGmol.cc index 25b4f34f..9f9692e8 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -87,6 +87,7 @@ extern Timer sgemm_tm; extern Timer dgemm_tm; extern Timer mpgemm_tm; extern Timer tttgemm_tm; +extern Timer bligemm_tm; extern Timer dsyrk_tm; extern Timer ssyrk_tm; extern Timer mpsyrk_tm; @@ -864,6 +865,7 @@ void MGmol::printTimers() dgemm_tm.print(os_); mpgemm_tm.print(os_); tttgemm_tm.print(os_); + bligemm_tm.print(os_); ssyrk_tm.print(os_); dsyrk_tm.print(os_); diff --git a/src/Rho.cc b/src/Rho.cc index 45a41f3e..e80e74fe 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -588,7 +588,7 @@ Rho::computeRho>( ExtendedGridOrbitals&, const dist_matrix::DistMatrix&); template void Rho::computeRho>( LocGridOrbitals&, const dist_matrix::DistMatrix&); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template double Rho::dotWithRho( const float* const func) const; #endif diff --git a/src/global.h b/src/global.h index de6c8559..a23e4e3b 100644 --- a/src/global.h +++ b/src/global.h @@ -15,7 +15,7 @@ //#include "mgmol_memory.h" -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP typedef float ORBDTYPE; #else typedef double ORBDTYPE; diff --git a/src/linear_algebra/CMakeLists.txt b/src/linear_algebra/CMakeLists.txt index 8bd5ef2f..0178f539 100644 --- a/src/linear_algebra/CMakeLists.txt +++ b/src/linear_algebra/CMakeLists.txt @@ -1,4 +1,4 @@ -set(SOURCES mputils.cc) +set(SOURCES mputils.cc gemm_impl.cc syrk_impl.cc) add_library(mgmol_linear_algebra ${SOURCES}) target_link_libraries(mgmol_linear_algebra PUBLIC MPI::MPI_CXX) if(${MGMOL_WITH_MAGMA}) diff --git a/src/linear_algebra/MGmol_blas1.h b/src/linear_algebra/MGmol_blas1.h index 41204226..32431468 100644 --- a/src/linear_algebra/MGmol_blas1.h +++ b/src/linear_algebra/MGmol_blas1.h @@ -11,6 +11,9 @@ #define MGMOL_MYBLAS1_H #include "fc_mangle.h" +#ifdef MGMOL_USE_BLIS +#include +#endif #include #include @@ -18,6 +21,8 @@ #define MY_VERSION 0 #define EPSILON 1.e-12 +#ifndef MGMOL_USE_BLIS + #ifdef __cplusplus extern "C" { @@ -42,12 +47,17 @@ extern "C" float SNRM2(const int* const, const float* const, const int* const); int IDAMAX(const int* const, const double* const, const int* const); int ISAMAX(const int* const, const float* const, const int* const); - void DROT(int*, double*, int*, double*, int*, double*, double*); - void SROT(int*, float*, int*, float*, int*, float*, float*); + void DROT(const int*, double*, const int*, double*, const int*, + const double*, const double*); + void SROT(const int*, float*, const int*, float*, const int*, const float*, + const float*); + #ifdef __cplusplus } #endif +#endif // MGMOL_USE_BLIS + inline void my_daxpy( const int n, const double alpha, const double* const a, double* b) { diff --git a/src/linear_algebra/gemm_impl.cc b/src/linear_algebra/gemm_impl.cc new file mode 100644 index 00000000..3500410c --- /dev/null +++ b/src/linear_algebra/gemm_impl.cc @@ -0,0 +1,251 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "gemm_impl.h" +#include "Timer.h" +#include "mputils.h" + +#include +#include + +Timer mpgemm_tm("mpgemm"); +Timer tttgemm_tm("tttgemm"); + +using LAU_H = LinearAlgebraUtils; + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) +{ + tttgemm_tm.start(); + // std::cout<<"template MPgemm..."< buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const T2* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = static_cast(c[pos]) * beta; + const T1* __restrict__ ai = a + lda * i; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const T1* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (T3)sum; + } + } + } + } + + tttgemm_tm.stop(); +} + +// input/output in float, computation in double +template <> +void gemm_impl(const char transa, const char transb, + const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const float* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + mpgemm_tm.start(); + + if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; + + /* case transb == 'N' and transa == 'N' */ + if (transb == 'N' || transb == 'n') + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = (double)c[pos] * beta; + const float* __restrict__ ai = a + lda * i; + c[pos] = (float)(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const float* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (float)sum; + } + } + } + } + + mpgemm_tm.stop(); +} + +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, + const int ldc); diff --git a/src/linear_algebra/gemm_impl.h b/src/linear_algebra/gemm_impl.h new file mode 100644 index 00000000..5140b143 --- /dev/null +++ b/src/linear_algebra/gemm_impl.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc); diff --git a/src/linear_algebra/mputils.cc b/src/linear_algebra/mputils.cc index 77b08689..487d1367 100644 --- a/src/linear_algebra/mputils.cc +++ b/src/linear_algebra/mputils.cc @@ -16,19 +16,27 @@ #include #endif +#include "MGmol_blas1.h" +#include "gemm_impl.h" +#include "syrk_impl.h" + +#ifdef MGMOL_USE_BLIS +#include +#else +#include "blas2_c.h" +#include "blas3_c.h" +#endif + #include #include #include Timer dgemm_tm("dgemm"); Timer sgemm_tm("sgemm"); -Timer mpgemm_tm("mpgemm"); -Timer tttgemm_tm("tttgemm"); +Timer bligemm_tm("bligemm"); Timer dsyrk_tm("dsyrk"); Timer ssyrk_tm("ssyrk"); -Timer mpsyrk_tm("mpsyrk"); -Timer tttsyrk_tm("tttsyrk"); Timer mpdot_tm("mpdot"); Timer ttdot_tm("ttdot"); @@ -304,106 +312,6 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, dsyrk_tm.stop(); } -template <> -void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, - const double alpha, const float* const a, const int lda, const double beta, - float* c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(c); - - mpsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - mpsyrk_tm.stop(); -} - template <> template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, @@ -413,97 +321,7 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, MemorySpace::assert_is_host_ptr(a); MemorySpace::assert_is_host_ptr(c); - tttsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - - tttsyrk_tm.stop(); + syrk_impl(uplo, trans, n, k, alpha, a, lda, beta, c, ldc); } // MemorySpace::Device @@ -725,103 +543,7 @@ void LAU_H::MPgemm(const char transa, const char transb, const int m, MemorySpace::assert_is_host_ptr(b); MemorySpace::assert_is_host_ptr(c); - tttgemm_tm.start(); - // if(onpe0)cout<<"template MPgemm..."< buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const T2* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = static_cast(c[pos]) * beta; - const T1* __restrict__ ai = a + lda * i; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const T1* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (T3)sum; - } - } - } - } - - tttgemm_tm.stop(); + gemm_impl(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } // input/output in double, computation in double @@ -842,111 +564,6 @@ void LAU_H::MPgemm(const char transa, const char transb, dgemm_tm.stop(); } -// input/output in float, computation in double -template <> -template <> -void LAU_H::MPgemm(const char transa, const char transb, - const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(b); - MemorySpace::assert_is_host_ptr(c); - - mpgemm_tm.start(); - - if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; - - /* case transb == 'N' and transa == 'N' */ - if (transb == 'N' || transb == 'n') - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = (double)c[pos] * beta; - const float* __restrict__ ai = a + lda * i; - c[pos] = (float)(alpha * MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const float* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (float)sum; - } - } - } - } - - mpgemm_tm.stop(); -} - // MemorySpace::Device #ifdef HAVE_MAGMA template <> @@ -1014,10 +631,6 @@ void LAU_D::MPgemm(const char transa, const char transb, const int m, } #endif -/////////////////////////////// -// MPgemmNN // -/////////////////////////////// - template template void LinearAlgebraUtils::MPgemmNN(const int m, const int n, @@ -1031,19 +644,122 @@ void LinearAlgebraUtils::MPgemmNN(const int m, const int n, transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } +template <> +template <> +void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const double* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, k, const_cast(a), 1, lda, &A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, k, n, const_cast(b), 1, ldb, &B); + + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else + char transa = 'n'; + char transb = 'n'; + + LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif +} + // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc) { + // std::cout << "LAU_H::MPgemmTN" << std::endl; + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, m, const_cast(a), 1, lda, &A); + bli_obj_toggle_trans(&A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, n, const_cast(b), 1, ldb, &B); + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + // dafault: precision of C + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else char transa = 't'; char transb = 'n'; LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif } // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, float* const c, const int ldc) { @@ -1053,12 +769,12 @@ void MPgemmTN(const int m, const int n, const int k, const double alpha, LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } -/////// additional calls ... may be removed later if unused - +template template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc) +void LinearAlgebraUtils::MPgemmTN(const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) { // if(onpe0)cout<<"template MPgemmNN..."<(const char transa, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc); -template void LAU_H::MPgemmNN(const int m, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double* const b, const int ldb, const double beta, float* const c, +template void LAU_H::MPgemm(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, const int ldc); + +// template void LAU_H::MPgemmNN(const int m, const int n, +// const int k, const double alpha, const float* const a, const int lda, +// const double* const b, const int ldb, const double beta, float* const c, +// const int ldc); template void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, @@ -1127,14 +849,18 @@ template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, double* __restrict__ yptr); template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, float* __restrict__ yptr); + template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, float* c, const int ldc); template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const float* const a, const int lda, const double beta, double* c, const int ldc); +template void LAU_H::MPsyrk(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, float* c, const int ldc); -template void MPgemmTN(const int m, const int n, +template void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, const int ldc); diff --git a/src/linear_algebra/mputils.h b/src/linear_algebra/mputils.h index 17bd0b4b..fa838f17 100644 --- a/src/linear_algebra/mputils.h +++ b/src/linear_algebra/mputils.h @@ -11,8 +11,6 @@ #define MGMOL_MPUTILS_H #include "MGmol_blas1.h" -#include "blas2_c.h" -#include "blas3_c.h" #include "memory_space.h" /* scal */ @@ -100,6 +98,11 @@ struct LinearAlgebraUtils const double alpha, const T1* const a, const int lda, const T2* const b, const int ldb, const double beta, T3* const c, const int ldc); + template + static void MPgemmTN(const int m, const int n, const int k, + const double alpha, const T1* const a, const int lda, const T2* const b, + const int ldb, const double beta, T3* const c, const int ldc); + /* mixed-precision scalar times vector. Accumulates results * in double precision and stores as single precision. */ @@ -125,28 +128,12 @@ struct LinearAlgebraUtils static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, double* c, const int ldc); - static void MPsyrk(const char uplo, const char trans, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double beta, float* c, const int ldc); template static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const T1* const a, const int lda, const double beta, T2* c, const int ldc); }; -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, double* const c, const int ldc); - -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc); - -template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc); - /* trsm */ void Ttrsm(const char, const char, const char, const char, const int, const int, const double, const double* const, const int, double* const, const int); diff --git a/src/linear_algebra/syrk_impl.cc b/src/linear_algebra/syrk_impl.cc new file mode 100644 index 00000000..e1a4833f --- /dev/null +++ b/src/linear_algebra/syrk_impl.cc @@ -0,0 +1,227 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Timer.h" +#include "mputils.h" + +#include "MGmol_blas1.h" +#include "syrk_impl.h" + +#include "blas3_c.h" + +#include +#include +#include + +Timer mpsyrk_tm("mpsyrk"); +Timer tttsyrk_tm("tttsyrk"); + +using LAU_H = LinearAlgebraUtils; + +template <> +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const float* const a, const int lda, const double beta, + float* c, const int ldc) +{ + mpsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + mpsyrk_tm.stop(); +} + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc) +{ + tttsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + + tttsyrk_tm.stop(); +} + +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const double* const a, + const int lda, const double beta, float* c, const int ldc); +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, double* c, const int ldc); diff --git a/src/linear_algebra/syrk_impl.h b/src/linear_algebra/syrk_impl.h new file mode 100644 index 00000000..ab3786f4 --- /dev/null +++ b/src/linear_algebra/syrk_impl.h @@ -0,0 +1,13 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc); diff --git a/src/md.cc b/src/md.cc index f21fe542..8ad3d7b8 100644 --- a/src/md.cc +++ b/src/md.cc @@ -764,12 +764,6 @@ void MGmol::loadRestartFile(const std::string filename) pot.resetVhRho2Backup(); electrostat_->setupRhoc(pot.rho_comp()); } - if (!ct.fullyOccupied()) - { - // overwrite DM with restart data in dataset Density_Matrix_WF - if (h5file.checkDataExists("Density_Matrix_WF")) - ierr = proj_matrices_->readWFDM(h5file); - } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index 468d61bd..92390e83 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -20,6 +20,10 @@ #include #include +#ifdef MGMOL_USE_BLIS +#include +#endif + #ifdef _OPENMP #include #endif @@ -39,6 +43,10 @@ int mgmol_init(MPI_Comm comm) assert(mype > -1); MPIdata::onpe0 = (MPIdata::mype == 0); +#ifdef MGMOL_USE_BLIS + bli_init(); +#endif + #ifdef HAVE_MAGMA magma_int_t magmalog = magma_init(); if (magmalog == MAGMA_SUCCESS) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5141a887..0b4dfc53 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,8 @@ add_executable(testDistVector ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -32,6 +34,8 @@ add_executable(testReplicated2DistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -43,6 +47,8 @@ add_executable(testDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -54,6 +60,8 @@ add_executable(testConditionDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -68,6 +76,8 @@ add_executable(testConditionDistMatrixPower ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -79,6 +89,8 @@ add_executable(testPower ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -93,6 +105,8 @@ add_executable(testPowerDistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -123,6 +137,8 @@ add_executable(testVariableSizeMatrix ${CMAKE_SOURCE_DIR}/src/sparse_linear_algebra/Table.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -138,6 +154,8 @@ add_executable(testTradeGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -153,6 +171,8 @@ add_executable(testSetGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -172,6 +192,8 @@ add_executable(testLaph4 ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -189,6 +211,8 @@ add_executable(testBatchLaph4 ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -204,6 +228,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -225,6 +251,8 @@ add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -244,6 +272,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz index 1616f79c..d5171c8b 100644 --- a/tests/RestartEnergyAndForces/h2o.xyz +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -3,3 +3,4 @@ O 0.00 0.00 0.00 H -0.76 0.59 0.00 H 0.76 0.59 0.00 + diff --git a/tests/testIons.cc b/tests/testIons.cc index e23e0e24..39c96f4a 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -166,8 +166,12 @@ int main(int argc, char** argv) const double fval = 1.12; for (auto& f : forces) f = fval; + + ions.getNames(names); ions.setLocalForces(forces, names); + ions.printForcesGlobal(std::cout); + int nlocal = ions.getNumLocIons(); std::vector lforces(3 * nlocal); ions.getLocalForces(lforces); From 58990e5e562c88a2de43f56154dae3a58c28eb7e Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 27 Mar 2025 14:14:30 -0700 Subject: [PATCH 41/50] Merge release (#329) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- src/Electrostatic.h | 1 - src/Ions.cc | 32 +++++++++++--------------- src/Ions.h | 6 ++--- tests/CMakeLists.txt | 18 ++++++++------- tests/MVP/mvp.cfg | 1 - tests/RhoVhRestart/testRhoVhRestart.cc | 4 ++-- tests/testIons.cc | 30 ++++++++++++++++++++---- 7 files changed, 53 insertions(+), 39 deletions(-) diff --git a/src/Electrostatic.h b/src/Electrostatic.h index f49ad12f..619da18a 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -49,7 +49,6 @@ class Electrostatic const bool isDielectric() { return diel_flag_; } pb::GridFunc* getRhoc() { return grhoc_; } - Poisson* getPoissonSolver() { return poisson_solver_; } void setup(const short max_sweeps); diff --git a/src/Ions.cc b/src/Ions.cc index f10fb834..3b128b0e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -109,14 +109,12 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { - std::vector::const_iterator ion = ions.list_ions_.begin(); - while (ion != ions.list_ions_.end()) + for (const auto& ion : ions.list_ions_) { - Ion* newion = new Ion(**ion); + Ion* newion = new Ion(*ion); newion->shiftPosition(shift); newion->setup(); list_ions_.push_back(newion); - ion++; } for (short i = 0; i < 3; ++i) lattice_[i] = ions.lattice_[i]; @@ -947,10 +945,11 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) num_ions_ = at_names.size(); #ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) +#endif { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } -#endif + if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ @@ -2211,7 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { names.clear(); - for (auto& ion : local_ions_) + for (const auto& ion : local_ions_) { names.push_back(ion->name()); } @@ -2219,14 +2218,14 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { - names.clear(); - for (auto& ion : list_ions_) - { - names.push_back(ion->name()); - } + std::vector local_names; + getLocalNames(local_names); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(local_names, names); } -void Ions::getPositions(std::vector& tau) +void Ions::getPositions(std::vector& tau) const { std::vector tau_local(3 * local_ions_.size()); @@ -2236,10 +2235,9 @@ void Ions::getPositions(std::vector& tau) mmpi.allGatherV(tau_local, tau); } -void Ions::getAtomicNumbers(std::vector& atnumbers) +void Ions::getAtomicNumbers(std::vector& atnumbers) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2249,15 +2247,11 @@ void Ions::getAtomicNumbers(std::vector& atnumbers) mmpi.allGatherV(local_atnumbers, atnumbers); } -void Ions::getForces(std::vector& forces) +void Ions::getForces(std::vector& forces) const { std::vector forces_local(3 * local_ions_.size()); - getLocalForces(forces_local); - int n = getNumIons(); - forces.resize(3 * n); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allGatherV(forces_local, forces); } diff --git a/src/Ions.h b/src/Ions.h index 3420d725..3d382eb7 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -292,10 +292,10 @@ class Ions void getLocalPositions(std::vector& tau) const; void getLocalNames(std::vector& names) const; void getNames(std::vector& names) const; - void getPositions(std::vector& tau); - void getAtomicNumbers(std::vector& atnumbers); + void getPositions(std::vector& tau) const; + void getAtomicNumbers(std::vector& atnumbers) const; - void getForces(std::vector& forces); + void getForces(std::vector& forces) const; void getLocalForces(std::vector& tau) const; /*! diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0b4dfc53..02edbe8b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -557,14 +557,16 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -add_test(NAME testHDF5single - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz - ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(MGMOL_USE_HDF5P) + add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +endif() add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/mvp.cfg b/tests/MVP/mvp.cfg index bb3ab52a..868e5703 100644 --- a/tests/MVP/mvp.cfg +++ b/tests/MVP/mvp.cfg @@ -32,4 +32,3 @@ solver=MVP nb_inner_it=2 [Restart] output_level=2 -output_type=single_file diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc index 09da3320..ce40272d 100644 --- a/tests/RhoVhRestart/testRhoVhRestart.cc +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -46,8 +46,8 @@ int testRhoRestart(MGmolInterface* mgmol_) /* check if the recomputed density is the same */ for (int d = 0; d < (int)rho0.size(); d++) { - double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); - if (error > 1e-10) + double error = abs(rho0[d] - rho->rho_[0][d]); + if (error > 1e-10 * abs(rho0[d])) { printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, rho->rho_[0][d], d, rho0[d]); diff --git a/tests/testIons.cc b/tests/testIons.cc index 39c96f4a..1741e0f7 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -107,6 +107,7 @@ int main(int argc, char** argv) ions.getAtomicNumbers(anumbers); if (myrank == 0) { + std::cout << "Positions:" << std::endl; int i = 0; for (auto& position : positions) { @@ -121,7 +122,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); // swap x and z - for (size_t i = 0; i < positions.size() - 2; i++) + for (size_t i = 0; i < positions.size() - 2; i += 3) { double x = positions[i]; double z = positions[i + 2]; @@ -162,10 +163,13 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); std::vector forces(3 * na); - // arbitrary value - const double fval = 1.12; + // set forces to a different arbitrary value for each component + int i = 0; for (auto& f : forces) - f = fval; + { + f = (double)i; + i++; + } ions.getNames(names); ions.setLocalForces(forces, names); @@ -177,13 +181,29 @@ int main(int argc, char** argv) ions.getLocalForces(lforces); for (auto& f : lforces) { - if (std::abs(f - fval) > 1.e-14) + if (std::fmod(f, 1.) > 1.e-14) { std::cerr << "f = " << f << std::endl; MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + ions.getForces(forces); + if (myrank == 0) + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < 1.e-14) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 + << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + } mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From c554b54b180905cecedb0b5a7dffae4c5b762820 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Fri, 28 Mar 2025 07:15:18 -0700 Subject: [PATCH 42/50] Merge release (#332) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- src/ABPG.cc | 6 +- src/DFTsolver.cc | 4 +- src/DavidsonSolver.cc | 8 +- src/Energy.cc | 29 ++- src/Energy.h | 12 +- src/GrassmanLineMinimization.cc | 4 +- src/HamiltonianMVPSolver.cc | 14 +- src/Ions.cc | 191 +++++++++--------- src/Ions.h | 19 +- src/MGmol.cc | 55 +++-- src/MGmol.h | 3 +- src/MVPSolver.cc | 4 +- src/PolakRibiereSolver.cc | 8 +- src/lbfgsrlx.cc | 1 - src/quench.cc | 4 +- .../testRestartEnergyAndForces.cc | 4 - tests/testIons.cc | 1 - 17 files changed, 184 insertions(+), 183 deletions(-) diff --git a/src/ABPG.cc b/src/ABPG.cc index 62cb6ba1..107edc63 100644 --- a/src/ABPG.cc +++ b/src/ABPG.cc @@ -38,7 +38,7 @@ void ABPG::setup(T& orbitals) // // orthof=true: wants orthonormalized updated wave functions template -int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, +int ABPG::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool /*orthof*/, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -51,8 +51,8 @@ int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, T res("Residual", orbitals, false); const bool check_res = (atol > 0.); - double normRes = mgmol_strategy_->computeResidual( - orbitals, work_orbitals, res, (print_res || check_res), check_res); + double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, + ions, res, (print_res || check_res), check_res); if (normRes < atol && check_res) { abpg_nl_update_tm_.stop(); diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index 91c4a0f6..206fba91 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -196,8 +196,8 @@ double DFTsolver::evaluateEnergy( // Get the new total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - eks_history_[0] - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + eks_history_[0] = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); sum_eig_[1] = sum_eig_[0]; sum_eig_[0] = 2. * proj_matrices_->getEigSum(); // 2.*sum in [Ry] diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index 0dc88ef4..a049fde7 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -500,7 +500,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(projmatrices, (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, projmatrices, orbitals, printE, os_); + ts0, projmatrices, ions_, orbitals, printE, os_); retval = checkConvergence(e0, outer_it, ct.conv_tol); if (retval == 0 || (outer_it == ct.max_electronic_steps)) @@ -549,7 +549,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, proj_mat2N_.get(), orbitals, printE, os_); + ts0, proj_mat2N_.get(), ions_, orbitals, printE, os_); } // 2N x 2N target... @@ -621,8 +621,8 @@ int DavidsonSolver::solve( const double ts1 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat2N_.get(), orbitals, ct.verbose - 1, os_); + const double e1 = energy_->evaluateTotal(ts1, proj_mat2N_.get(), + ions_, orbitals, ct.verbose - 1, os_); // line minimization beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); diff --git a/src/Energy.cc b/src/Energy.cc index 1cee6d67..f53dbc93 100644 --- a/src/Energy.cc +++ b/src/Energy.cc @@ -25,11 +25,10 @@ #define RY2HA 0.5 template -Energy::Energy(const pb::Grid& mygrid, const Ions& ions, - const Potentials& pot, const Electrostatic& es, const Rho& rho, - const XConGrid& xc, SpreadPenaltyInterface* spread_penalty) +Energy::Energy(const pb::Grid& mygrid, const Potentials& pot, + const Electrostatic& es, const Rho& rho, const XConGrid& xc, + SpreadPenaltyInterface* spread_penalty) : mygrid_(mygrid), - ions_(ions), pot_(pot), es_(es), rho_(rho), @@ -59,7 +58,7 @@ double Energy::getEVrhoRho() const } template -double Energy::evaluateEnergyIonsInVext() +double Energy::evaluateEnergyIonsInVext(Ions& ions) { double energy = 0.; @@ -69,12 +68,12 @@ double Energy::evaluateEnergyIonsInVext() //(*MPIdata::sout)<<"Energy::evaluateEnergyIonsInVext()"< positions; - positions.reserve(3 * ions_.local_ions().size()); + positions.reserve(3 * ions.local_ions().size()); // loop over ions int nions = 0; - std::vector::const_iterator ion = ions_.local_ions().begin(); - while (ion != ions_.local_ions().end()) + std::vector::const_iterator ion = ions.local_ions().begin(); + while (ion != ions.local_ions().end()) { (*ion)->getPosition(position); positions.push_back(position[0]); @@ -88,9 +87,9 @@ double Energy::evaluateEnergyIonsInVext() pot_.getValVext(positions, val); // loop over ions again - ion = ions_.local_ions().begin(); + ion = ions.local_ions().begin(); int ion_index = 0; - while (ion != ions_.local_ions().end()) + while (ion != ions.local_ions().end()) { const double z = (*ion)->getZion(); // int ion_index=(*ion)->index(); @@ -112,16 +111,16 @@ double Energy::evaluateEnergyIonsInVext() template double Energy::evaluateTotal(const double ts, // in [Ha] - ProjectedMatricesInterface* projmatrices, const T& phi, const int verbosity, - std::ostream& os) + ProjectedMatricesInterface* projmatrices, Ions& ions, const T& phi, + const int verbosity, std::ostream& os) { eval_te_tm_.start(); Control& ct = *(Control::instance()); - const double eself = ions_.energySelf(); - const double ediff = ions_.energyDiff(ct.bcPoisson); - const double eipot = evaluateEnergyIonsInVext(); + const double eself = ions.energySelf(); + const double ediff = ions.energyDiff(ct.bcPoisson); + const double eipot = evaluateEnergyIonsInVext(ions); const double eigsum = 0.5 * projmatrices->getExpectationH(); diff --git a/src/Energy.h b/src/Energy.h index e6cafd98..d55d4040 100644 --- a/src/Energy.h +++ b/src/Energy.h @@ -11,6 +11,7 @@ #define MGMOL_ENERGY_H #include "Grid.h" +#include "Ions.h" #include "Rho.h" #include "SpreadPenaltyInterface.h" #include "Timer.h" @@ -20,7 +21,6 @@ #include class Potentials; -class Ions; class Electrostatic; class ProjectedMatricesInterface; class XConGrid; @@ -29,7 +29,6 @@ template class Energy { const pb::Grid& mygrid_; - const Ions& ions_; const Potentials& pot_; const Electrostatic& es_; const Rho& rho_; @@ -45,16 +44,15 @@ class Energy double getEVrhoRho() const; public: - Energy(const pb::Grid&, const Ions&, const Potentials&, - const Electrostatic&, const Rho&, const XConGrid&, - SpreadPenaltyInterface*); + Energy(const pb::Grid&, const Potentials&, const Electrostatic&, + const Rho&, const XConGrid&, SpreadPenaltyInterface*); static Timer eval_te_tm() { return eval_te_tm_; } - double evaluateTotal(const double ts, ProjectedMatricesInterface*, + double evaluateTotal(const double ts, ProjectedMatricesInterface*, Ions&, const T& phi, const int, std::ostream&); - double evaluateEnergyIonsInVext(); + double evaluateEnergyIonsInVext(Ions&); void saveVofRho(); }; diff --git a/src/GrassmanLineMinimization.cc b/src/GrassmanLineMinimization.cc index a83d6b08..f642e0f4 100644 --- a/src/GrassmanLineMinimization.cc +++ b/src/GrassmanLineMinimization.cc @@ -29,7 +29,7 @@ Timer GrassmanLineMinimization::update_states_tm_("Grassman_update_states"); // // orthof=true: wants orthonormalized updated wave functions template -int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, +int GrassmanLineMinimization::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool orthof, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -61,7 +61,7 @@ int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, // Update wavefunctions const bool check_res = (atol > 0.); double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, - *new_grad_, (print_res || check_res), check_res); + ions, *new_grad_, (print_res || check_res), check_res); if (normRes < atol && check_res) { nl_update_tm_.stop(); diff --git a/src/HamiltonianMVPSolver.cc b/src/HamiltonianMVPSolver.cc index 4bc017dc..f90dd228 100644 --- a/src/HamiltonianMVPSolver.cc +++ b/src/HamiltonianMVPSolver.cc @@ -160,8 +160,8 @@ int HamiltonianMVPSolver::solve( // compute energy at origin const int printE = (ct.verbose > 1) ? 1 : 0; - double e0 - = energy_->evaluateTotal(ts0, projmatrices, orbitals, printE, os_); + double e0 = energy_->evaluateTotal( + ts0, projmatrices, ions_, orbitals, printE, os_); // // compute energy at end for new H @@ -189,8 +189,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at end (beta=1.) - double e1 - = energy_->evaluateTotal(ts1, projmatrices, orbitals, printE, os_); + double e1 = energy_->evaluateTotal( + ts1, projmatrices, ions_, orbitals, printE, os_); // // evaluate energy at mid-point @@ -226,8 +226,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at midpoint - double ei - = energy_->evaluateTotal(tsi, projmatrices, orbitals, printE, os_); + double ei = energy_->evaluateTotal( + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization double beta @@ -285,7 +285,7 @@ int HamiltonianMVPSolver::solve( // compute energy at end (beta=1.) ei = energy_->evaluateTotal( - tsi, projmatrices, orbitals, printE, os_); + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization beta = minQuadPolynomialFrom3values( diff --git a/src/Ions.cc b/src/Ions.cc index 3b128b0e..a372593d 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -70,8 +70,18 @@ void writeData2d(HDFrestart& h5f_file, std::string datasetname, } } -Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) +Ions::Ions(const double lat[3], const std::vector& sp) + : species_(sp), setup_(false), has_locked_atoms_(false) { + setupSubdomains(lat); +} + +void Ions::setupSubdomains(const double lat[3]) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + for (short i = 0; i < 3; i++) { assert(lat[i] > 0.); @@ -79,15 +89,10 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) { (*MPIdata::serr) << "Ions constructor: lattice[" << i << "]=" << lat[i] << "!!!" << std::endl; - exit(2); + mmpi.abort(); } lattice_[i] = lat[i]; } - setup_ = false; - has_locked_atoms_ = false; - - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); for (short i = 0; i < 3; i++) div_lattice_[i] = lattice_[i] / (double)(myPEenv.n_mpi_task(i)); @@ -107,6 +112,20 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) MPI_Cart_shift(cart_comm_, dir, disp, &source_[dir], &dest_[dir]); } +Ions::Ions(const std::vector& p, const std::vector& anum, + const double lat[3], const std::vector& sp) + : species_(sp) +{ + setupSubdomains(lat); + + double rmax = getMaxListRadius(); + setupListIonsBoundaries(rmax); + + num_ions_ = setAtoms(p, anum); + + setup(); +} + Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { for (const auto& ion : ions.list_ions_) @@ -197,7 +216,7 @@ void Ions::setup() has_locked_atoms_ = hasLockedAtoms(); // initialize data for constraints - setupContraintsData(interacting_ions_); + setupContraintsData(); computeMaxNumProjs(); @@ -297,13 +316,13 @@ void Ions::setupInteractingIons() // setup arrays to be used in constraints enforcement // using references to local_ions and extra "dummy" data -void Ions::setupContraintsData(std::vector& ions_for_constraints) +void Ions::setupContraintsData() { Control& ct = *(Control::instance()); if (ct.verbose > 0) printWithTimeStamp("Ions::setupContraintsData()...", std::cout); - const int nnloc = ions_for_constraints.size() - local_ions_.size(); + const int nnloc = interacting_ions_.size() - local_ions_.size(); // std::cout<<"interacting_ions_.size()="<::finalEnergy() // Get the total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] total_energy_ = energy_->evaluateTotal( - ts, proj_matrices_.get(), *current_orbitals_, 2, os_); + ts, proj_matrices_.get(), *ions_, *current_orbitals_, 2, os_); } template @@ -828,7 +827,7 @@ void MGmol::setupPotentials(Ions& ions) pot.initialize(ions); if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); + g_kbpsi_->setup(ions); electrostat_->setupRhoc(pot.rho_comp()); @@ -1182,8 +1181,8 @@ void MGmol::precond_mg(OrbitalsType& phi) template double MGmol::computeResidual(OrbitalsType& orbitals, - OrbitalsType& work_orbitals, OrbitalsType& res, const bool print_residual, - const bool norm_res) + OrbitalsType& work_orbitals, Ions& ions, OrbitalsType& res, + const bool print_residual, const bool norm_res) { assert(orbitals.getIterativeIndex() >= 0); @@ -1202,7 +1201,7 @@ double MGmol::computeResidual(OrbitalsType& orbitals, // get H*psi stored in work_orbitals.psi // and psi^T H psi in Hij - getHpsiAndTheta(*ions_, orbitals, work_orbitals); + getHpsiAndTheta(ions, orbitals, work_orbitals); double norm2Res = computeConstraintResidual( orbitals, work_orbitals, res, print_residual, norm_res); @@ -1440,17 +1439,22 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, Control& ct = *(Control::instance()); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); - quench(*dorbitals, *ions_, ct.max_electronic_steps, 20, eks); + quench(*dorbitals, ions, ct.max_electronic_steps, 20, eks); - force(*dorbitals, *ions_); + force(*dorbitals, ions); - ions_->getForces(forces); + ions.getForces(forces); return eks; } @@ -1462,35 +1466,42 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, { OrbitalsType* dorbitals = dynamic_cast(orbitals); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); // initialize electronic density rho_->update(*dorbitals); // initialize potential - update_pot(*ions_); + update_pot(ions); // initialize projected matrices - updateHmatrix(*dorbitals, *ions_); + updateHmatrix(*dorbitals, ions); proj_matrices_->updateThetaAndHB(); // compute DM std::shared_ptr> dm_strategy( DMStrategyFactory>::create(comm_, os_, *ions_, + dist_matrix::DistMatrix>::create(comm_, os_, ions, rho_.get(), energy_.get(), electrostat_.get(), this, proj_matrices_.get(), dorbitals)); dm_strategy->update(*dorbitals); // evaluate energy and forces - double ts = 0.; - double eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), *dorbitals, 2, os_); + double ts = 0.; + double eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, *dorbitals, 2, os_); + + force(*dorbitals, ions); - force(*dorbitals, *ions_); + ions.getForces(forces); ions_->getForces(forces); diff --git a/src/MGmol.h b/src/MGmol.h index 43df67f2..799a7edf 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -339,7 +339,8 @@ class MGmol : public MGmolInterface void precond_mg(OrbitalsType& orbitals); void setGamma(const pb::Lap& lapOper, const Potentials& pot); double computeResidual(OrbitalsType& orbitals, OrbitalsType& work_orbitals, - OrbitalsType& res, const bool print_residual, const bool norm_res); + Ions& ions, OrbitalsType& res, const bool print_residual, + const bool norm_res); void applyAOMMprojection(OrbitalsType&); void force(OrbitalsType& orbitals, Ions& ions) { diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 7ab9f665..6990e29d 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -256,7 +256,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts0 = evalEntropyMVP(current_proj_mat, (ct.verbose > 1), os_); const double e0 = energy_->evaluateTotal( - ts0, current_proj_mat, orbitals, printE, os_); + ts0, current_proj_mat, ions_, orbitals, printE, os_); MatrixType target("target", numst_, numst_); @@ -309,7 +309,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts1 = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, orbitals, ct.verbose - 1, os_); + ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); // line minimization const double beta diff --git a/src/PolakRibiereSolver.cc b/src/PolakRibiereSolver.cc index 6b5f88de..3f46a00e 100644 --- a/src/PolakRibiereSolver.cc +++ b/src/PolakRibiereSolver.cc @@ -243,9 +243,9 @@ double PolakRibiereSolver::evaluateEnergy( const OrbitalsType& orbitals, const bool print_flag) { // Get the new total energy - const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - const double energy - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] + const double energy = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); // Control& ct(*(Control::instance())); // if( ct.verbose>2 && onpe0 )os_<<"energy="<::solve(OrbitalsType& orbitals, // evaluate residuals, preconditioned residuals for current orbitals double normRes = mgmol_strategy_->computeResidual(orbitals, - work_orbitals, *r_k_, (print_res || ct.checkResidual()), + work_orbitals, ions_, *r_k_, (print_res || ct.checkResidual()), ct.checkResidual()); if (normRes < ct.conv_tol && ct.checkResidual()) { diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index 6698eef0..ce05c1a3 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -14,7 +14,6 @@ #include "Energy.h" #include "Ions.h" #include "LBFGS.h" -#include "LBFGS_IonicStepper.h" #include "LocalizationRegions.h" #include "MGmol.h" #include "MGmol_blas1.h" diff --git a/src/quench.cc b/src/quench.cc index 073798f6..6eed74bb 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -600,8 +600,8 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, << " TS [Ha] = " << ts << std::endl; } } - last_eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), orbitals, 2, os_); + last_eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, orbitals, 2, os_); if (ct.computeCondGramMD()) { diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index 3027ab55..b1fb0094 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,8 +124,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -195,8 +193,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - // // evaluate energy and forces with wavefunctions just read // diff --git a/tests/testIons.cc b/tests/testIons.cc index 1741e0f7..6bea5d64 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -170,7 +170,6 @@ int main(int argc, char** argv) f = (double)i; i++; } - ions.getNames(names); ions.setLocalForces(forces, names); From 2f1c16feb8603c62d6ffbb239831d46b4e8a5ff9 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 1 Apr 2025 11:32:57 -0700 Subject: [PATCH 43/50] Merge release (#334) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ * Add cleanup and MGmolInterface (#331) * Add function to set local forces (#333) * based on matching coordinates to local ions --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- src/Ions.cc | 33 +++++++++++ src/Ions.h | 6 ++ src/MGmol.cc | 2 - src/MGmolInterface.h | 1 + .../testWFEnergyAndForces.cc | 1 + tests/testIons.cc | 58 ++++++++++++++----- 6 files changed, 86 insertions(+), 15 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index a372593d..cb6dddf5 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1324,6 +1324,39 @@ void Ions::setLocalForces( } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& coords) +{ + assert(forces.size() == coords.size()); + + // tolerance can be pretty loose, as long as it does not allow mix up + // with coordinates of other atoms + const double tol = 1.e-2; + + // loop over global list of forces and coordinates + std::vector::const_iterator cit = coords.begin(); + for (auto fit = forces.begin(); fit != forces.end(); fit += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + double p[3]; + ion->getPosition(&p[0]); + double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) + + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + double d = std::sqrt(d2); + if (d < tol) + { + ion->set_force(0, *fit); + ion->set_force(1, *(fit + 1)); + ion->set_force(2, *(fit + 2)); + } + } + cit += 3; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { diff --git a/src/Ions.h b/src/Ions.h index f39e1fed..aaf48e79 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -308,6 +308,12 @@ class Ions void setLocalForces(const std::vector& forces, const std::vector& names); + /*! + * set forces for ions in local_ions_ based on coordinates matching + */ + void setLocalForces( + const std::vector& forces, const std::vector& coords); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/src/MGmol.cc b/src/MGmol.cc index 97e1ee6f..4ccb8e90 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1503,8 +1503,6 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, ions.getForces(forces); - ions_->getForces(forces); - return eks; } diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index dc54a8e7..3c2384fd 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -25,6 +25,7 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual void cleanup() = 0; virtual double evaluateEnergyAndForces(const std::vector& tau, const std::vector& atnumbers, std::vector& forces) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index c9e28d48..64765c7a 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,7 @@ int main(int argc, char** argv) } } + mgmol->cleanup(); delete mgmol; } // close main scope diff --git a/tests/testIons.cc b/tests/testIons.cc index 6bea5d64..33c3cd24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -6,6 +6,29 @@ #include +// check that all forces components have integer values larger than 0 +// and differ from each other +int checkForces(std::vector& forces) +{ + const double tol = 1.e-14; + + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < tol || *f1 < tol || *f0 < tol) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 << std::endl; + return 1; + } + } + } + + return 0; +} + int main(int argc, char** argv) { int status = 0; @@ -164,7 +187,7 @@ int main(int argc, char** argv) std::vector forces(3 * na); // set forces to a different arbitrary value for each component - int i = 0; + int i = 1; for (auto& f : forces) { f = (double)i; @@ -189,20 +212,29 @@ int main(int argc, char** argv) ions.getForces(forces); if (myrank == 0) - for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + + // test Ions::setLocalForces based on coordinates matching + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + + ions.setLocalForces(forces, positions); + + ions.printForcesGlobal(std::cout); + + ions.getForces(forces); + if (myrank == 0) { - std::cout << "f0 = " << *f0 << std::endl; - for (auto f1 = f0 + 1; f1 != forces.end(); f1++) - { - // make sure each force component is different - if (std::abs(*f0 - *f1) < 1.e-14) - { - std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 - << std::endl; - MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); - } - } + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } + } + mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From 704c9527c1f7ac4edb0185d285f3ff697011e9fc Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 8 Apr 2025 17:50:35 -0700 Subject: [PATCH 44/50] ROM MVP solver in MD for Pinned H2O (#314) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Add commit number temporarily * Fix merge * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Move geometric transforms to tools in library * Add functionalities for extra info in restart file (#312) * Add ROM MVP solver to MD * Changes to scripts * MPI abort (#313) * Fix error code used by MPI_Abort * Partial updates * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Attempt to fix bugs * Fix config file * Debug * Debug * Debug * Debug * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Fix ROM stage * Debug * Add initialization to evaluateDMandEnergyandForces if it is ROM MVP * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Pass by reference * Make global version of setForces * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Use new setLocalForce * Write PinnedH2O into original ordering. This is better for MD * names should be obtained before rotate * Fixing ion names * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Debugging * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ * Add cleanup and MGmolInterface (#331) * Using new constructor for ROM ions * Add function to set local forces (#333) * based on matching coordinates to local ions * Use new setLocalForces with positions instead of names * Cleanup * only pe0 prints bondlengths and bondangle * Fix setLocalForces * Updates * Minor changes * Auxiliary scripts * Auxiliary script * Auxiliary scripts * fix job scripts * Renaming and adding new test * Rename * Minor change in auxiliary script * config files * update tests * Minor changes in names --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- .../PinnedH2O/{coords.in => coords_test1.in} | 0 examples/PinnedH2O/coords_test2.in | 3 + examples/PinnedH2O/get_result.sh | 42 ++-- examples/PinnedH2O/job.basis_1_50 | 8 +- examples/PinnedH2O/job.offline | 13 +- examples/PinnedH2O/job.ref | 36 +++ examples/PinnedH2O/{job.rom_1_50 => job.rom} | 13 +- examples/PinnedH2O/job.rom_3DOF | 35 +++ .../{mgmol.cfg => mgmol_ref_test1.cfg} | 6 +- examples/PinnedH2O/mgmol_ref_test2.cfg | 31 +++ examples/PinnedH2O/mgmol_rom_1_50.cfg | 6 +- examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg | 46 ++++ examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 42 ++++ src/Control.cc | 4 + src/Ions.cc | 23 +- src/MGmol.cc | 27 ++ src/MGmol.h | 1 + src/md.cc | 95 ++++++- src/rom_Control.h | 2 + src/tools/CMakeLists.txt | 1 + src/tools/PinnedH2O.cc | 237 ++++++++++++++++++ src/tools/PinnedH2O.h | 54 ++++ tests/CMakeLists.txt | 3 +- .../{coords.in_rotate1 => coords_rotate1.in} | 0 .../{coords.in_rotate2 => coords_rotate2.in} | 0 tests/PinnedH2O_3DOF/job.basis | 10 +- tests/PinnedH2O_3DOF/job.offline | 20 +- tests/PinnedH2O_3DOF/job.online | 17 +- tests/PinnedH2O_3DOF/job.online_rotate | 8 +- tests/PinnedH2O_3DOF/mgmol_online.cfg | 2 +- tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg | 6 +- tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc | 219 +--------------- tests/PinnedH2O_3DOF/transfer_nlrom.sh | 20 ++ 33 files changed, 725 insertions(+), 305 deletions(-) rename examples/PinnedH2O/{coords.in => coords_test1.in} (100%) create mode 100644 examples/PinnedH2O/coords_test2.in create mode 100644 examples/PinnedH2O/job.ref rename examples/PinnedH2O/{job.rom_1_50 => job.rom} (79%) create mode 100644 examples/PinnedH2O/job.rom_3DOF rename examples/PinnedH2O/{mgmol.cfg => mgmol_ref_test1.cfg} (89%) create mode 100644 examples/PinnedH2O/mgmol_ref_test2.cfg create mode 100644 examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg create mode 100644 examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg create mode 100644 src/tools/PinnedH2O.cc create mode 100644 src/tools/PinnedH2O.h rename tests/PinnedH2O_3DOF/{coords.in_rotate1 => coords_rotate1.in} (100%) rename tests/PinnedH2O_3DOF/{coords.in_rotate2 => coords_rotate2.in} (100%) create mode 100644 tests/PinnedH2O_3DOF/transfer_nlrom.sh diff --git a/examples/PinnedH2O/coords.in b/examples/PinnedH2O/coords_test1.in similarity index 100% rename from examples/PinnedH2O/coords.in rename to examples/PinnedH2O/coords_test1.in diff --git a/examples/PinnedH2O/coords_test2.in b/examples/PinnedH2O/coords_test2.in new file mode 100644 index 00000000..e1e31f8b --- /dev/null +++ b/examples/PinnedH2O/coords_test2.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.57 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/examples/PinnedH2O/get_result.sh b/examples/PinnedH2O/get_result.sh index 443e4cdd..b737960d 100644 --- a/examples/PinnedH2O/get_result.sh +++ b/examples/PinnedH2O/get_result.sh @@ -1,27 +1,37 @@ -filename="offline_PinnedH2O.out" # FOM -#filename="rom39_PinnedH2O.out" # compare MD -#filename="39_force_PinnedH2O.out" # compare force +#filename="offline_PinnedH2O" # FOM +#filename="rom39_PinnedH2O" # ROM compare MD +#filename="39_force_PinnedH2O" # ROM compare force + +#filename="PinnedH2O_test2_ref" # FOM +filename="PinnedH2O_rom_3DOF_test2_2_2_34" # ROM PinnedH2O 3DOF MD + +# Extracting kinetic energy, total energy, temperature from MGmgol output log +awk '/Kinetic/ {print $3}' $filename.out > ke_$filename.txt +awk '/Kinetic/ {print $5}' $filename.out >temp_$filename.txt +awk '/Total/ {print $3}' $filename.out > te_$filename.txt # Extracting H1, H2, F1, F2 from MGmgol output log # if FOM, these files contain the FOM results # if compare MD, these files contain the results with projected orbitals -awk '/H1 / {print $3, $4, $5}' $filename > H1_$filename -awk '/H2 / {print $3, $4, $5}' $filename > H2_$filename -awk '/F1 / {print $6, $7, $8}' $filename > F1_$filename -awk '/F2 / {print $6, $7, $8}' $filename > F2_$filename +awk '/O1 / {print $4, $5, $6}' $filename.out > O1_$filename.txt +awk '/H1 / {print $3, $4, $5}' $filename.out > H1_$filename.txt +awk '/H2 / {print $3, $4, $5}' $filename.out > H2_$filename.txt +awk '/O1 / {print $7, $8, $9}' $filename.out > f_O1_$filename.txt +awk '/H1 / {print $6, $7, $8}' $filename.out > f_H1_$filename.txt +awk '/H2 / {print $6, $7, $8}' $filename.out > f_H2_$filename.txt # if compare force, files with "_fom" contain the FOM results # files with "_rom" contain the results with projected orbitals if [[ "$filename" == *"force_"* ]]; then - sed -n '1~2p' H1_$filename > H1_rom$filename - sed -n '1~2p' H2_$filename > H2_rom$filename - sed -n '1~2p' F1_$filename > F1_rom$filename - sed -n '1~2p' F2_$filename > F2_rom$filename + sed -n '1~2p' H1_$filename.out > H1_rom$filename.txt + sed -n '1~2p' H2_$filename.out > H2_rom$filename.txt + sed -n '1~2p' f_H1_$filename.out > f_H1_rom$filename.txt + sed -n '1~2p' f_H2_$filename.out > f_H2_rom$filename.txt - sed -n '2~2p' H1_$filename > H1_fom$filename - sed -n '2~2p' H2_$filename > H2_fom$filename - sed -n '2~2p' F1_$filename > F1_fom$filename - sed -n '2~2p' F2_$filename > F2_fom$filename + sed -n '2~2p' H1_$filename.out > H1_fom$filename.txt + sed -n '2~2p' H2_$filename.out > H2_fom$filename.txt + sed -n '2~2p' f_H1_$filename.out > f_H1_fom$filename.txt + sed -n '2~2p' f_H2_$filename.out > f_H2_fom$filename.txt fi -rm -rf snapshot0_* +rm -rf snapshot_* diff --git a/examples/PinnedH2O/job.basis_1_50 b/examples/PinnedH2O/job.basis_1_50 index a5069570..bf0a4c49 100644 --- a/examples/PinnedH2O/job.basis_1_50 +++ b/examples/PinnedH2O/job.basis_1_50 @@ -1,16 +1,16 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 0:10:00 -#SBATCH -p pbatch +#SBATCH -p pdebug date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20241012 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.offline b/examples/PinnedH2O/job.offline index 0f8ef90e..64997e09 100644 --- a/examples/PinnedH2O/job.offline +++ b/examples/PinnedH2O/job.offline @@ -1,16 +1,16 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 1:00:00 -#SBATCH -p pbatch +#SBATCH -p pdebug date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20241012 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -18,12 +18,7 @@ set exe = mgmol-opt cp $maindir/install_quartz/bin/$exe . -set datadir = $maindir/examples/PinnedH2O - set cfg_offline = mgmol_offline.cfg -cp $datadir/$cfg_offline . - -cp $datadir/coords.in . ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref new file mode 100644 index 00000000..2c4660cf --- /dev/null +++ b/examples/PinnedH2O/job.ref @@ -0,0 +1,36 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 +set case = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20250219 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set cfg = mgmol_ref_test${case}.cfg +cp $datadir/$cfg . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg -i coords_test${case}.in > PinnedH2O_test${case}_ref.out + +date diff --git a/examples/PinnedH2O/job.rom_1_50 b/examples/PinnedH2O/job.rom similarity index 79% rename from examples/PinnedH2O/job.rom_1_50 rename to examples/PinnedH2O/job.rom index aa6df335..fc4ae1b0 100644 --- a/examples/PinnedH2O/job.rom_1_50 +++ b/examples/PinnedH2O/job.rom @@ -1,16 +1,16 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 1:00:00 -#SBATCH -p pbatch +#SBATCH -p pdebug date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20241012 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -18,16 +18,11 @@ set exe = mgmol-opt cp $maindir/install_quartz/bin/$exe . -set datadir = $maindir/examples/PinnedH2O - set increment_md_steps = 1 set num_md_steps = 50 set basis_file = PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps} set cfg_rom = mgmol_rom_${increment_md_steps}_${num_md_steps}.cfg -cp $datadir/$cfg_rom . - -cp $datadir/coords.in . ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF new file mode 100644 index 00000000..8b328199 --- /dev/null +++ b/examples/PinnedH2O/job.rom_3DOF @@ -0,0 +1,35 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 +set case = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20250219 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set bondlength_num_increments = 2 +set bondangle_num_increments = 2 +set num_basis = 34 + +set cfg_rom = mgmol_rom_3DOF_test${case}.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords_test${case}.in > PinnedH2O_rom_3DOF_test${case}_${bondlength_num_increments}_${bondangle_num_increments}_${num_basis}.out + +date diff --git a/examples/PinnedH2O/mgmol.cfg b/examples/PinnedH2O/mgmol_ref_test1.cfg similarity index 89% rename from examples/PinnedH2O/mgmol.cfg rename to examples/PinnedH2O/mgmol_ref_test1.cfg index ef82f07d..fd456f92 100644 --- a/examples/PinnedH2O/mgmol.cfg +++ b/examples/PinnedH2O/mgmol_ref_test1.cfg @@ -18,12 +18,12 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=50 +num_steps=500 dt=40. thermostat=ON [Thermostat] type=Berendsen -temperature=1000. +temperature=300. relax_time=800. [Quench] max_steps=100 @@ -32,4 +32,4 @@ atol=1.e-8 initial_type=Random initial_width=2. [Restart] -output_level=0 +output_level=4 diff --git a/examples/PinnedH2O/mgmol_ref_test2.cfg b/examples/PinnedH2O/mgmol_ref_test2.cfg new file mode 100644 index 00000000..67aa443c --- /dev/null +++ b/examples/PinnedH2O/mgmol_ref_test2.cfg @@ -0,0 +1,31 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=OFF +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 diff --git a/examples/PinnedH2O/mgmol_rom_1_50.cfg b/examples/PinnedH2O/mgmol_rom_1_50.cfg index 337a86eb..13cc3921 100644 --- a/examples/PinnedH2O/mgmol_rom_1_50.cfg +++ b/examples/PinnedH2O/mgmol_rom_1_50.cfg @@ -1,6 +1,6 @@ verbosity=1 xcFunctional=PBE -FDtype=Mehrstellen +FDtype=4th [Mesh] nx=64 ny=64 @@ -23,7 +23,7 @@ dt=40. thermostat=ON [Thermostat] type=Berendsen -temperature=1000. +temperature=300. relax_time=800. [Quench] max_steps=100 @@ -33,6 +33,8 @@ initial_type=Random initial_width=2. [Restart] output_level=4 +[ROM] +stage=test_orbital [ROM.offline] basis_file=PinnedH2O_orbitals_basis_1_50 [ROM.basis] diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg new file mode 100644 index 00000000..adfdfc43 --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg @@ -0,0 +1,46 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=300. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=30 +[DensityMatrix] +solver=MVP +nb_inner_it=100 +[Restart] +output_level=4 +[ROM] +stage=online_pinned_H2O_3dof +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=34 diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg new file mode 100644 index 00000000..f0368790 --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -0,0 +1,42 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=OFF +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=30 +[DensityMatrix] +solver=MVP +nb_inner_it=100 +[Restart] +output_level=4 +[ROM] +stage=online_pinned_H2O_3dof +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=34 diff --git a/src/Control.cc b/src/Control.cc index d0b1a224..151fe920 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1953,6 +1953,10 @@ void Control::setROMOptions(const boost::program_options::variables_map& vm) rom_pri_option.rom_stage = ROMStage::ONLINE; else if (str.compare("build") == 0) rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("online_pinned_H2O_3dof") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE_PINNED_H2O_3DOF; + else if (str.compare("test_orbital") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_ORBITAL; else if (str.compare("online_poisson") == 0) rom_pri_option.rom_stage = ROMStage::ONLINE_POISSON; else if (str.compare("test_poisson") == 0) diff --git a/src/Ions.cc b/src/Ions.cc index cb6dddf5..072f0950 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1315,6 +1315,22 @@ void Ions::setLocalForces( { if (ion->compareName(*s)) { + //std::cout << "Ion found: " << ion->name() << std::endl; + //std::cout << "Ion force: (" << *it << ", " << *(it + 1) << ", " << *(it + 2) << ")" << std::endl; + //std::cout << "names: "; + //for (int i = 0; i < names.size(); i++) + //{ + // std::cout << names[i]; + // if (i == forces.size() - 1) std::cout << std::endl; + // else std::cout << ", "; + //} + //std::cout << "forces: "; + //for (int i = 0; i < forces.size(); i++) + //{ + // std::cout << forces[i]; + // if (i == forces.size() - 1) std::cout << ")" << std::endl; + // else std::cout << ", "; + //} ion->set_force(0, *it); ion->set_force(1, *(it + 1)); ion->set_force(2, *(it + 2)); @@ -1343,11 +1359,14 @@ void Ions::setLocalForces( double p[3]; ion->getPosition(&p[0]); double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) - + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) - + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + + (p[1] - (*(cit + 1))) * (p[1] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[2] - (*(cit + 2))); double d = std::sqrt(d2); if (d < tol) { + std::cout << "Ion found: " << ion->name() << std::endl; + std::cout << "Ion position:( " << *cit << ", " << *(cit + 1) << ", " << *(cit + 2) << ")" << std::endl; + std::cout << "Ion force: (" << *fit << ", " << *(fit + 1) << ", " << *(fit + 2) << ")" << std::endl; ion->set_force(0, *fit); ion->set_force(1, *(fit + 1)); ion->set_force(2, *(fit + 2)); diff --git a/src/MGmol.cc b/src/MGmol.cc index 4ccb8e90..facd362e 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1422,6 +1422,33 @@ void MGmol::getAtomicNumbers(std::vector& an) ions_->getAtomicNumbers(an); } +template +void MGmol::updateDMandEnergy(OrbitalsType& orbitals, Ions& ions, double& eks) +{ + // initialize electronic density + rho_->update(orbitals); + + // initialize potential + update_pot(ions); + + // initialize projected matrices + updateHmatrix(orbitals, ions); + proj_matrices_->updateThetaAndHB(); + + // compute DM + std::shared_ptr> dm_strategy( + DMStrategyFactory>::create(comm_, os_, ions, + rho_.get(), energy_.get(), electrostat_.get(), this, + proj_matrices_.get(), &orbitals)); + + dm_strategy->update(orbitals); + + // evaluate energy and forces + double ts = 0.; + eks = energy_->evaluateTotal(ts, proj_matrices_.get(), ions, orbitals, 2, os_); +} + template double MGmol::evaluateEnergyAndForces( const std::vector& tau, const std::vector& atnumbers, diff --git a/src/MGmol.h b/src/MGmol.h index 799a7edf..c7777a62 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -368,6 +368,7 @@ class MGmol : public MGmolInterface int save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals); void project_orbital(std::string file_path, int rdim, OrbitalsType& orbitals); #endif + void updateDMandEnergy(OrbitalsType& orbitals, Ions& ions, double& eks); }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/md.cc b/src/md.cc index 8ad3d7b8..bc986955 100644 --- a/src/md.cc +++ b/src/md.cc @@ -35,6 +35,12 @@ #include "mgmol_Signal.h" #include "tools.h" +#ifdef MGMOL_HAS_LIBROM +#include "KBPsiMatrixSparse.h" +#include "PinnedH2O.h" +#include "rom_workflows.h" +#endif + #include #include #include @@ -399,8 +405,27 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) h5f_file_.reset(); } + bool ROM_MVP = (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF); +#ifdef MGMOL_HAS_LIBROM + // ROM - initialize orbitals and density matrix + if (ROM_MVP) + { + if (onpe0) os_ << "Setup ROM MVP solver..." << std::endl; + ExtendedGridOrbitals** extended_orbitals = reinterpret_cast(orbitals); + (*extended_orbitals)->set(ct.getROMOptions().basis_file, ct.numst); + (*extended_orbitals)->orthonormalizeLoewdin(); + (*extended_orbitals)->setDataWithGhosts(true); + (*extended_orbitals)->setIterativeIndex(10); + + std::shared_ptr projmatrices + = getProjectedMatrices(); + projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->printDM(os_); + } +#endif + // additional SC steps to compensate random start - if (ct.restart_info < 3) + if (ct.restart_info < 3 && !ROM_MVP) { double eks = 0.; quench(**orbitals, ions, ct.max_electronic_steps, 20, eks); @@ -432,10 +457,43 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) int retval = 0; bool small_move = true; bool last_move_is_small = true; - do + + bool force_on_ions = true; +#ifdef MGMOL_HAS_LIBROM + // Ions for ROM + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + std::vector positions; + std::vector anumbers; + getAtomicPositions(positions); + getAtomicNumbers(anumbers); + Ions* ROM_ions; + + // Pinned H2O 3 DOF + PinnedH2O H2O_molecule; + if (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF) + { + if (onpe0) os_ << "Rotate Pinned H2O molecule in timestep " << mdstep << std::endl; + H2O_molecule.rotate(positions, anumbers); + if (onpe0) H2O_molecule.print(os_); + ROM_ions = new Ions(positions, anumbers, lattice, ions_->getSpecies()); + setupPotentials(*ROM_ions); + force_on_ions = false; + } +#endif + + if (ROM_MVP) + { + updateDMandEnergy(**orbitals, *ROM_ions, eks); + } + else { retval = quench(**orbitals, ions, ct.max_electronic_steps, 0, eks); + } + do + { // update localization regions if (ct.adaptiveLRs()) { @@ -495,10 +553,23 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) << std::endl; // Compute forces - force(**orbitals, ions); + if (force_on_ions) + force(**orbitals, ions); #ifdef MGMOL_HAS_LIBROM - if (ct.getROMOptions().num_orbbasis > 0) + if (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF) + { + force(**orbitals, *ROM_ions); + // Pinned H2O 3 DOF + if (onpe0) os_ << "Transpose rotate the PinnedH2O molecule" << std::endl; + std::vector forces; + ROM_ions->getForces(forces); + H2O_molecule.transpose_rotate(positions, anumbers, forces); + ions.setLocalForces(forces, positions); + delete ROM_ions; + } + + if (ct.getROMOptions().rom_stage == ROMStage::TEST_ORBITAL) { if (onpe0) { @@ -507,26 +578,30 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) os_ << "Loading ROM basis " << ct.getROMOptions().basis_file << std::endl; os_ << "ROM basis dimension = " << ct.getROMOptions().num_orbbasis << std::endl; } + + // Project orbitals to ROM subspace project_orbital(ct.getROMOptions().basis_file, ct.getROMOptions().num_orbbasis, **orbitals); if (ct.getROMOptions().compare_md) { + // overwrite ions force for MD time stepping force(**orbitals, ions); } else { - double shift[3]; - for (short i = 0; i < 3; i++) shift[i] = 0.; - Ions ROM_ions(ions, shift); - force(**orbitals, ROM_ions); + // write ROM_ions force for one-step comparison + ROM_ions = new Ions(positions, anumbers, lattice, ions_->getSpecies()); + force(**orbitals, *ROM_ions); std::string zero = "0"; if (ions_->getNumIons() < 256 || ct.verbose > 2) { - if (ct.verbose > 0) ROM_ions.printForcesGlobal(os_); + if (ct.verbose > 0) ROM_ions->printForcesGlobal(os_); + } else if (zero.compare(ct.md_print_filename) == 0) { - ROM_ions.printForcesLocal(os_); + ROM_ions->printForcesLocal(os_); } + delete ROM_ions; } } #endif diff --git a/src/rom_Control.h b/src/rom_Control.h index 200061cb..2d143d84 100644 --- a/src/rom_Control.h +++ b/src/rom_Control.h @@ -22,6 +22,8 @@ enum class ROMStage ONLINE, RESTORE, // TODO(kevin): what stage is this? BUILD, + ONLINE_PINNED_H2O_3DOF, + TEST_ORBITAL, ONLINE_POISSON, TEST_POISSON, TEST_RHO, diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index baae6cda..ca187a0e 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES mgmol_mpi_tools.cc random.cc coloring.cc SymmetricMatrix.cc + PinnedH2O.cc ) add_library(mgmol_tools ${SOURCES}) diff --git a/src/tools/PinnedH2O.cc b/src/tools/PinnedH2O.cc new file mode 100644 index 00000000..22f4677a --- /dev/null +++ b/src/tools/PinnedH2O.cc @@ -0,0 +1,237 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "PinnedH2O.h" +#include + +PinnedH2O::PinnedH2O() + : flipped_bond(false), + O1_idx(-1), + H1_idx(-1), + H2_idx(-1), + planar_rotation_angle(0.0) +{ + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) { + out_of_plane_rotation_matrix[i][j] = 0.0; + } + } +} + +double PinnedH2O::calculate_bondlength(const double atom1[3], const double atom2[3]) const +{ + return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + pow(atom1[2] - atom2[2], 2)); +} + +double PinnedH2O::calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3]) const +{ + double vector1[3] = {atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2]}; + double vector2[3] = {atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2]}; + + double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + vector1[2] * vector2[2]; + double magnitude_product = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) * + sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); + double angle = acos(dot_product / magnitude_product); + + return angle; +} + +void PinnedH2O::rotation_matrix(const double axis[3], double angle, double matrix[3][3]) const +{ + double cos_theta = cos(angle); + double sin_theta = sin(angle); + double ux = axis[0], uy = axis[1], uz = axis[2]; + + matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); + matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; + matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; + + matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; + matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); + matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; + + matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; + matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; + matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); +} + +void PinnedH2O::normalize(double vec[3]) const +{ + double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; +} + +void PinnedH2O::cross(const double a[3], const double b[3], double result[3]) const +{ + result[0] = a[1] * b[2] - a[2] * b[1]; + result[1] = a[2] * b[0] - a[0] * b[2]; + result[2] = a[0] * b[1] - a[1] * b[0]; +} + +void PinnedH2O::apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) const +{ + result[0] = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; + result[1] = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; + result[2] = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; +} + +void PinnedH2O::apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) const +{ + result[0] = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; + result[1] = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; + result[2] = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; +} + +void PinnedH2O::rotate(std::vector& positions, std::vector& anumbers) +{ + for (int i = 0; i < 3; i++) { + if (positions[3 * i] == 0.0 && positions[3 * i + 1] == 0.0 && positions[3 * i + 2] == 0.0) { + O1_idx = i; + break; + } + } + if (O1_idx == -1) return; + + H1_idx = (O1_idx + 1) % 3; + H2_idx = (O1_idx + 2) % 3; + + double O1[3] = {positions[3 * O1_idx], positions[3 * O1_idx + 1], positions[3 * O1_idx + 2]}; + double H1[3] = {positions[3 * H1_idx], positions[3 * H1_idx + 1], positions[3 * H1_idx + 2]}; + double H2[3] = {positions[3 * H2_idx], positions[3 * H2_idx + 1], positions[3 * H2_idx + 2]}; + + bondlength1 = calculate_bondlength(H1, O1); + bondlength2 = calculate_bondlength(H2, O1); + bondangle = calculate_bondangle(H1, O1, H2); + + double H1_temp[3], H2_temp[3]; + double H1_rotated[3], H2_rotated[3]; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + double target_plane_normal[3] = {0, 0, 1}; + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align = std::acos(std::min(std::max(dot_product, -1.0), 1.0)); + double axis_to_align[3]; + if (abs(dot_product) > 1.0 - 1e-8) + { + axis_to_align[0] = 1.0; + axis_to_align[1] = 0.0; + axis_to_align[2] = 0.0; + } + else + { + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + } + + rotation_matrix(axis_to_align, angle_to_align, out_of_plane_rotation_matrix); + apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); + apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); + + double theta1 = std::atan2(H1_temp[1], H1_temp[0]); + planar_rotation_angle = -theta1 + bondangle / 2.0; + double planar_rotation_matrix[3][3]; + rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); + apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); + + flipped_bond = (bondlength1 < bondlength2); + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + std::swap(H1_rotated, H2_rotated); + std::swap(bondlength1, bondlength2); + } + + positions[0] = H2_rotated[0]; + positions[1] = H2_rotated[1]; + positions[2] = H2_rotated[2]; + positions[3] = 0.0; + positions[4] = 0.0; + positions[5] = 0.0; + positions[6] = H1_rotated[0]; + positions[7] = H1_rotated[1]; + positions[8] = H1_rotated[2]; + + anumbers[0] = 1; + anumbers[1] = 8; + anumbers[2] = 1; +} + +void PinnedH2O::transpose_rotate(std::vector& positions, std::vector& anumbers, std::vector& forces) +{ + double H2_rotated[3] = {positions[0], positions[1], positions[2]}; + double O1_rotated[3] = {positions[3], positions[4], positions[5]}; + double H1_rotated[3] = {positions[6], positions[7], positions[8]}; + + double f_H2_rotated[3] = {forces[0], forces[1], forces[2]}; + double f_O1_rotated[3] = {forces[3], forces[4], forces[5]}; + double f_H1_rotated[3] = {forces[6], forces[7], forces[8]}; + + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + f_O1_rotated[1] *= -1.0; + f_H1_rotated[1] *= -1.0; + f_H2_rotated[1] *= -1.0; + } + + double planar_rotation_matrix[3][3]; + double target_plane_normal[3] = {0, 0, 1}; + rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + + double H1_temp[3], H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); + apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); + + double f_O1_temp[3], f_H1_temp[3], f_H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, f_O1_rotated, f_O1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H1_rotated, f_H1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H2_rotated, f_H2_temp); + + double H1_restored[3], H2_restored[3]; + apply_transpose_rotation(out_of_plane_rotation_matrix, H1_temp, H1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, H2_temp, H2_restored); + + double f_O1_restored[3], f_H1_restored[3], f_H2_restored[3]; + apply_transpose_rotation(out_of_plane_rotation_matrix, f_O1_temp, f_O1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, f_H1_temp, f_H1_restored); + apply_transpose_rotation(out_of_plane_rotation_matrix, f_H2_temp, f_H2_restored); + + positions[3*H2_idx] = H2_restored[0]; + positions[3*H2_idx+1] = H2_restored[1]; + positions[3*H2_idx+2] = H2_restored[2]; + positions[3*O1_idx] = 0.0; + positions[3*O1_idx+1] = 0.0; + positions[3*O1_idx+2] = 0.0; + positions[3*H1_idx] = H1_restored[0]; + positions[3*H1_idx+1] = H1_restored[1]; + positions[3*H1_idx+2] = H1_restored[2]; + + anumbers[H2_idx] = 1; + anumbers[O1_idx] = 8; + anumbers[H1_idx] = 1; + + forces[3*H2_idx] = f_H2_restored[0]; + forces[3*H2_idx+1] = f_H2_restored[1]; + forces[3*H2_idx+2] = f_H2_restored[2]; + forces[3*O1_idx] = f_O1_restored[0]; + forces[3*O1_idx+1] = f_O1_restored[1]; + forces[3*O1_idx+2] = f_O1_restored[2]; + forces[3*H1_idx] = f_H1_restored[0]; + forces[3*H1_idx+1] = f_H1_restored[1]; + forces[3*H1_idx+2] = f_H1_restored[2]; +} diff --git a/src/tools/PinnedH2O.h b/src/tools/PinnedH2O.h new file mode 100644 index 00000000..bee880d8 --- /dev/null +++ b/src/tools/PinnedH2O.h @@ -0,0 +1,54 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef PINNED_H2O_H +#define PINNED_H2O_H + +#include +#include +#include +#include + +class PinnedH2O +{ + +public: + PinnedH2O(); + ~PinnedH2O() = default; + + void rotate(std::vector& positions, std::vector& anumbers); + void transpose_rotate(std::vector& positions, std::vector& anumbers, std::vector& forces); + void print(std::ostream& os) + { + os << "Bondlengths = " << bondlength1 << ", " << bondlength2 << " Bohrs; " + << "Bondangle = " << bondangle * 180.0 / M_PI << " degrees." << std::endl; + } + +private: + double calculate_bondlength(const double atom1[3], const double atom2[3]) const; + double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3]) const; + void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) const; + void normalize(double vec[3]) const; + void cross(const double a[3], const double b[3], double result[3]) const; + void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) const; + void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) const; + + double bondlength1; + double bondlength2; + double bondangle; + + bool flipped_bond; + int O1_idx; + int H1_idx; + int H2_idx; + double planar_rotation_angle; + double out_of_plane_rotation_matrix[3][3]; +}; + +#endif // PINNED_H2O_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 02edbe8b..b4844557 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -291,7 +291,8 @@ add_executable(testDMandEnergyAndForces add_executable(testRestartEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) add_executable(testPinnedH2O_3DOF - ${CMAKE_SOURCE_DIR}/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc) + ${CMAKE_SOURCE_DIR}/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc + ${CMAKE_SOURCE_DIR}/src/tools/PinnedH2O.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload diff --git a/tests/PinnedH2O_3DOF/coords.in_rotate1 b/tests/PinnedH2O_3DOF/coords_rotate1.in similarity index 100% rename from tests/PinnedH2O_3DOF/coords.in_rotate1 rename to tests/PinnedH2O_3DOF/coords_rotate1.in diff --git a/tests/PinnedH2O_3DOF/coords.in_rotate2 b/tests/PinnedH2O_3DOF/coords_rotate2.in similarity index 100% rename from tests/PinnedH2O_3DOF/coords.in_rotate2 rename to tests/PinnedH2O_3DOF/coords_rotate2.in diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis index eaf8bfde..cab929d1 100644 --- a/tests/PinnedH2O_3DOF/job.basis +++ b/tests/PinnedH2O_3DOF/job.basis @@ -1,5 +1,5 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 0:10:00 #SBATCH -p pdebug @@ -8,9 +8,9 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250206 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -19,10 +19,10 @@ set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples set snapshot_files = "" set bondlength_min = 0.95 set bondlength_max = 1.05 -set bondlength_num_increments = 5 +set bondlength_num_increments = 2 set bondangle_min = -5.0 set bondangle_max = 5.0 -set bondangle_num_increments = 5 +set bondangle_num_increments = 2 foreach i (`seq 0 $bondlength_num_increments`) set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline index 3870a3a3..243f563a 100644 --- a/tests/PinnedH2O_3DOF/job.offline +++ b/tests/PinnedH2O_3DOF/job.offline @@ -1,5 +1,5 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 4:00:00 #SBATCH -p pbatch @@ -8,15 +8,14 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250206 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH set exe = mgmol-opt -rm $exe cp $maindir/install_quartz/bin/$exe . set cfg_offline = mgmol_offline.cfg @@ -46,14 +45,13 @@ foreach i (`seq 0 $bondlength_num_increments`) set tag = ${bondlength_one}_${bondlength_two}_${bondangle} if (-e data/${tag}) then continue + else + mkdir data/${tag} endif - cp $cfg_offline ${tag}_${cfg_offline} - sed -i "s/CUSTOM_TAG/${tag}/g" ${tag}_${cfg_offline} - srun -n $ncpus $exe -c ${tag}_${cfg_offline} -i data/coords_${tag}.in > ${tag}_offline_PinnedH2O.out - mv data/coords_${tag}.in ${tag}/coords.in - mv ${tag}_${cfg_offline} ${tag}/${cfg_offline} - mv ${tag}_offline_PinnedH2O.out ${tag}/offline_PinnedH2O.out - mv -f ${tag} data + cp $cfg_offline data/${tag}/${cfg_offline} + mv data/coords_${tag}.in data/${tag}/coords.in + sed -i "s/CUSTOM_TAG/${tag}/g" data/${tag}/${cfg_offline} + srun -n $ncpus $exe -c data/${tag}/${cfg_offline} -i data/${tag}/coords.in > data/${tag}/offline_PinnedH2O.out end end end diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online index 77b8fde0..1373e011 100644 --- a/tests/PinnedH2O_3DOF/job.online +++ b/tests/PinnedH2O_3DOF/job.online @@ -1,5 +1,5 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 1:00:00 #SBATCH -p pdebug @@ -8,15 +8,14 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250206 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH set exe = testPinnedH2O_3DOF -rm $exe cp $maindir/build_quartz/tests/$exe . set cfg_online = mgmol_online.cfg @@ -26,9 +25,9 @@ ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . source $maindir/scripts/modules.quartz -set sample_bondlength_num_increments = 5 -set sample_bondangle_num_increments = 5 -set num_orbital_basis = 36 +set sample_bondlength_num_increments = 2 +set sample_bondangle_num_increments = 2 +set num_orbital_basis = 34 set bondlength_min = 0.95 set bondlength_max = 1.05 @@ -58,9 +57,7 @@ foreach i (`seq 0 $bondlength_num_increments`) set bondangle = `printf "%.1f" $bondangle` echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" set tag = ${bondlength_one}_${bondlength_two}_${bondangle} - - echo $ncpus $exe $case $cfg_online $tag - srun -p pdebug -n $ncpus $exe -c ${case}_${cfg_online} -i data/${tag}/coords.in >& results_${case}/${tag}.log + srun -p pdebug -n $ncpus $exe -c ${case}_${cfg_online} -i /usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_${ncpus}/${tag}/coords.in >& results_${case}/${tag}.log end end end diff --git a/tests/PinnedH2O_3DOF/job.online_rotate b/tests/PinnedH2O_3DOF/job.online_rotate index 11dd7419..3f8a65ad 100644 --- a/tests/PinnedH2O_3DOF/job.online_rotate +++ b/tests/PinnedH2O_3DOF/job.online_rotate @@ -1,5 +1,5 @@ #!/bin/tcsh -#SBATCH -N 2 +#SBATCH -N 1 #SBATCH -t 0:10:00 #SBATCH -p pdebug @@ -8,9 +8,9 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 64 +set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250206 +set maindir = /p/lustre2/cheung26/mgmol-20250219 setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH @@ -21,7 +21,7 @@ source $maindir/scripts/modules.quartz rm -rf snapshot0_* -set coord_file = coords.in_rotate2 +set coord_file = coords_rotate2.in set exe = mgmol-opt set cfg_online = mgmol_ref_rotate.cfg diff --git a/tests/PinnedH2O_3DOF/mgmol_online.cfg b/tests/PinnedH2O_3DOF/mgmol_online.cfg index 5bc5b8b8..6f66067d 100644 --- a/tests/PinnedH2O_3DOF/mgmol_online.cfg +++ b/tests/PinnedH2O_3DOF/mgmol_online.cfg @@ -30,7 +30,7 @@ output_level=4 solver=MVP nb_inner_it=100 [ROM.offline] -basis_file=data/PinnedH2O_3DOF_orbitals_basis_CUSTOM_NL_CUSTOM_NT +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_CUSTOM_NL_CUSTOM_NT [ROM.basis] compare_md=false number_of_orbital_basis=CUSTOM_NB diff --git a/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg index 72288ac3..8c598248 100644 --- a/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg +++ b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg @@ -23,14 +23,14 @@ atol=1.e-8 [Orbitals] initial_type=Random initial_width=2. -nempty=68 +nempty=30 [Restart] output_level=4 [DensityMatrix] solver=MVP nb_inner_it=100 [ROM.offline] -basis_file=data/PinnedH2O_3DOF_orbitals_basis_2_2 +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 [ROM.basis] compare_md=false -number_of_orbital_basis=72 +number_of_orbital_basis=34 diff --git a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc index a0cd51d3..83bfe8a6 100644 --- a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc +++ b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc @@ -14,6 +14,7 @@ #include "MGmol_MPI.h" #include "MPIdata.h" #include "mgmol_run.h" +#include "PinnedH2O.h" #ifdef MGMOL_HAS_LIBROM #include "librom.h" @@ -29,215 +30,6 @@ #include namespace po = boost::program_options; -double calculate_bondlength(const double atom1[3], const double atom2[3]) -{ - return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + pow(atom1[2] - atom2[2], 2)); -} - -double calculate_bondangle(const double atom1[3], const double atom2[3], const double atom3[3]) -{ - double vector1[3] = {atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2]}; - double vector2[3] = {atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2]}; - - double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + vector1[2] * vector2[2]; - double magnitude_product = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) * - sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); - double angle = acos(dot_product / magnitude_product); - - return angle; -} - -void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) -{ - double cos_theta = cos(angle); - double sin_theta = sin(angle); - double ux = axis[0], uy = axis[1], uz = axis[2]; - - matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); - matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; - matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; - - matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; - matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); - matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; - - matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; - matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; - matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); -} - -void normalize(double vec[3]) -{ - double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); - vec[0] /= norm; - vec[1] /= norm; - vec[2] /= norm; -} - -void cross(const double a[3], const double b[3], double result[3]) -{ - result[0] = a[1] * b[2] - a[2] * b[1]; - result[1] = a[2] * b[0] - a[0] * b[2]; - result[2] = a[0] * b[1] - a[1] * b[0]; -} - -void apply_rotation(const double matrix[3][3], const double vec[3], double result[3]) -{ - result[0] = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; - result[1] = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; - result[2] = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; -} - -void apply_transpose_rotation(const double matrix[3][3], const double vec[3], double result[3]) -{ - result[0] = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; - result[1] = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; - result[2] = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; -} - -void rotate_PinnedH2O(std::vector& positions, std::vector& anumbers, - double out_of_plane_rotation_matrix[3][3], double& planar_rotation_angle, bool& flipped_bond) -{ - int O1_idx = -1; - for (int i = 0; i < 3; i++) - { - if (positions[3*i] == 0.0 && positions[3*i+1] == 0.0 && positions[3*i+2] == 0.0) - { - O1_idx = i; - break; - } - } - if (O1_idx == -1) return; - - int H1_idx = (O1_idx + 1) % 3; - int H2_idx = (O1_idx + 2) % 3; - - double O1[3] = {positions[3*O1_idx], positions[3*O1_idx+1], positions[3*O1_idx+2]}; - double H1[3] = {positions[3*H1_idx], positions[3*H1_idx+1], positions[3*H1_idx+2]}; - double H2[3] = {positions[3*H2_idx], positions[3*H2_idx+1], positions[3*H2_idx+2]}; - - double bondlength1 = calculate_bondlength(H1, O1); - double bondlength2 = calculate_bondlength(H2, O1); - double bondangle = calculate_bondangle(H1, O1, H2); - - double H1_temp[3], H2_temp[3]; - double H1_rotated[3], H2_rotated[3]; - - double plane_normal[3]; - cross(H2, H1, plane_normal); - normalize(plane_normal); - double target_plane_normal[3] = {0, 0, 1}; - double dot_product = plane_normal[0] * target_plane_normal[0] + - plane_normal[1] * target_plane_normal[1] + - plane_normal[2] * target_plane_normal[2]; - double angle_to_align = std::acos(std::min(std::max(dot_product, -1.0), 1.0)); - double axis_to_align[3]; - if (abs(dot_product) > 1.0 - 1e-8) - { - axis_to_align[0] = 1.0; - axis_to_align[1] = 0.0; - axis_to_align[2] = 0.0; - } - else - { - cross(plane_normal, target_plane_normal, axis_to_align); - normalize(axis_to_align); - } - rotation_matrix(axis_to_align, angle_to_align, out_of_plane_rotation_matrix); - apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); - apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); - - double theta1 = std::atan2(H1_temp[1], H1_temp[0]); - planar_rotation_angle = -theta1 + bondangle / 2.0; - double planar_rotation_matrix[3][3]; - rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); - apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); - apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); - - flipped_bond = (bondlength1 < bondlength2); - if (flipped_bond) - { - H1_rotated[1] *= -1.0; - H2_rotated[1] *= -1.0; - std::swap(H1_rotated, H2_rotated); - } - - positions[0] = H2_rotated[0]; - positions[1] = H2_rotated[1]; - positions[2] = H2_rotated[2]; - positions[3] = 0.0; - positions[4] = 0.0; - positions[5] = 0.0; - positions[6] = H1_rotated[0]; - positions[7] = H1_rotated[1]; - positions[8] = H1_rotated[2]; - - anumbers[0] = 1; - anumbers[1] = 8; - anumbers[2] = 1; -} - -void transpose_rotate_PinnedH2O(std::vector& positions, std::vector& forces, - const double out_of_plane_rotation_matrix[3][3], - const double planar_rotation_angle, const bool flipped_bond) -{ - double H2_rotated[3] = {positions[0], positions[1], positions[2]}; - double O1_rotated[3] = {positions[3], positions[4], positions[5]}; - double H1_rotated[3] = {positions[6], positions[7], positions[8]}; - - double f_H2_rotated[3] = {forces[0], forces[1], forces[2]}; - double f_O1_rotated[3] = {forces[3], forces[4], forces[5]}; - double f_H1_rotated[3] = {forces[6], forces[7], forces[8]}; - - if (flipped_bond) - { - H1_rotated[1] *= -1.0; - H2_rotated[1] *= -1.0; - f_O1_rotated[1] *= -1.0; - f_H1_rotated[1] *= -1.0; - f_H2_rotated[1] *= -1.0; - } - - double planar_rotation_matrix[3][3]; - double target_plane_normal[3] = {0, 0, 1}; - rotation_matrix(target_plane_normal, planar_rotation_angle, planar_rotation_matrix); - - double H1_temp[3], H2_temp[3]; - apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); - apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); - - double f_O1_temp[3], f_H1_temp[3], f_H2_temp[3]; - apply_transpose_rotation(planar_rotation_matrix, f_O1_rotated, f_O1_temp); - apply_transpose_rotation(planar_rotation_matrix, f_H1_rotated, f_H1_temp); - apply_transpose_rotation(planar_rotation_matrix, f_H2_rotated, f_H2_temp); - - double H1_restored[3], H2_restored[3]; - apply_transpose_rotation(out_of_plane_rotation_matrix, H1_temp, H1_restored); - apply_transpose_rotation(out_of_plane_rotation_matrix, H2_temp, H2_restored); - - double f_O1_restored[3], f_H1_restored[3], f_H2_restored[3]; - apply_transpose_rotation(out_of_plane_rotation_matrix, f_O1_temp, f_O1_restored); - apply_transpose_rotation(out_of_plane_rotation_matrix, f_H1_temp, f_H1_restored); - apply_transpose_rotation(out_of_plane_rotation_matrix, f_H2_temp, f_H2_restored); - - positions[0] = H2_restored[0]; - positions[1] = H2_restored[1]; - positions[2] = H2_restored[2]; - positions[6] = H1_restored[0]; - positions[7] = H1_restored[1]; - positions[8] = H1_restored[2]; - - forces[0] = f_H2_restored[0]; - forces[1] = f_H2_restored[1]; - forces[2] = f_H2_restored[2]; - forces[3] = f_O1_restored[0]; - forces[4] = f_O1_restored[1]; - forces[5] = f_O1_restored[2]; - forces[6] = f_H1_restored[0]; - forces[7] = f_H1_restored[1]; - forces[8] = f_H1_restored[2]; -} - int main(int argc, char** argv) { int mpirc = MPI_Init(&argc, &argv); @@ -340,11 +132,8 @@ int main(int argc, char** argv) } // rotate the molecule to the reference coordinate system - int atom_order[3]; - double out_of_plane_rotation_matrix[3][3]; - double planar_rotation_angle; - bool flipped_bond; - rotate_PinnedH2O(positions, anumbers, out_of_plane_rotation_matrix, planar_rotation_angle, flipped_bond); + PinnedH2O H2O_molecule; + H2O_molecule.rotate(positions, anumbers); Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); @@ -417,7 +206,7 @@ int main(int argc, char** argv) } // rotate the forces to the original coordinate system - transpose_rotate_PinnedH2O(positions, forces, out_of_plane_rotation_matrix, planar_rotation_angle, flipped_bond); + H2O_molecule.transpose_rotate(positions, anumbers, forces); // print out results if (MPIdata::onpe0) diff --git a/tests/PinnedH2O_3DOF/transfer_nlrom.sh b/tests/PinnedH2O_3DOF/transfer_nlrom.sh new file mode 100644 index 00000000..62824de2 --- /dev/null +++ b/tests/PinnedH2O_3DOF/transfer_nlrom.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ncpus=8 +target_dir="/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_${ncpus}" + +if [ ! -d $target_dir ]; then + mkdir -p $target_dir +fi + +for dir in data/*/; do + dirname=$(basename "$dir") + new_dir="$target_dir/${dirname}" + if [ ! -d $new_dir ]; then + mkdir -p $new_dir + fi + mv -f $dir/* $new_dir/ + rm -rf $dir +done + +rm -rf data From f11a40aedc484453d2ad630f3dd8dbe3da9ec967 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Wed, 14 May 2025 17:39:57 -0700 Subject: [PATCH 45/50] Testing ROM MVP MD (#338) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ * Add cleanup and MGmolInterface (#331) * Add function to set local forces (#333) * based on matching coordinates to local ions * Fix setLocalForces (#335) * Fix test WFEnergyAndForces (#336) * writing restart twice in same directory was failing * Add mixing option in MVP (#337) * Timer * Update MVP with mixing (#339) * use tolerance to terminate iterations * tune verbosity * Updates * use setupPotentials * Setup potentials again * setupRhoc is done within setupPotentials * Avoid setPositions * Rescaling ROM ion density * Relax threshold * Increase tolerance for testROMIonDensity --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- examples/PinnedH2O/job.basis_1_50 | 2 +- examples/PinnedH2O/job.offline | 2 +- examples/PinnedH2O/job.ref | 2 +- examples/PinnedH2O/job.rom | 2 +- examples/PinnedH2O/job.rom_3DOF | 2 +- examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg | 6 +- examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 6 +- src/Control.cc | 12 +- src/Ions.cc | 19 --- src/MGmol.cc | 2 + src/MVPSolver.cc | 115 ++++++++++-------- src/MVPSolver.h | 9 +- src/MVP_DMStrategy.cc | 2 +- src/md.cc | 3 + src/read_config.cc | 9 +- src/rom_workflows.cc | 36 ++++-- tests/CMakeLists.txt | 7 ++ tests/MVP/test.py | 2 + tests/MVPmix/h2o.xyz | 6 + tests/MVPmix/mgmol.cfg | 36 ++++++ tests/MVPmix/test.py | 103 ++++++++++++++++ tests/PinnedH2O_3DOF/job.basis | 2 +- tests/PinnedH2O_3DOF/job.offline | 2 +- tests/PinnedH2O_3DOF/job.online | 2 +- tests/PinnedH2O_3DOF/job.online_rotate | 2 +- .../testWFEnergyAndForces.cc | 3 + 26 files changed, 290 insertions(+), 104 deletions(-) create mode 100644 tests/MVPmix/h2o.xyz create mode 100644 tests/MVPmix/mgmol.cfg create mode 100644 tests/MVPmix/test.py diff --git a/examples/PinnedH2O/job.basis_1_50 b/examples/PinnedH2O/job.basis_1_50 index bf0a4c49..5291a7b7 100644 --- a/examples/PinnedH2O/job.basis_1_50 +++ b/examples/PinnedH2O/job.basis_1_50 @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.offline b/examples/PinnedH2O/job.offline index 64997e09..9d7150f8 100644 --- a/examples/PinnedH2O/job.offline +++ b/examples/PinnedH2O/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref index 2c4660cf..32024e6f 100644 --- a/examples/PinnedH2O/job.ref +++ b/examples/PinnedH2O/job.ref @@ -11,7 +11,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 set case = 2 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom index fc4ae1b0..45285136 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.rom @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF index 8b328199..0e38688b 100644 --- a/examples/PinnedH2O/job.rom_3DOF +++ b/examples/PinnedH2O/job.rom_3DOF @@ -11,7 +11,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 set case = 2 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg index adfdfc43..b7a78a0d 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg @@ -1,4 +1,4 @@ -verbosity=1 +verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] @@ -34,7 +34,9 @@ initial_width=2. nempty=30 [DensityMatrix] solver=MVP -nb_inner_it=100 +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 [Restart] output_level=4 [ROM] diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg index f0368790..24273c40 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -1,4 +1,4 @@ -verbosity=1 +verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] @@ -30,7 +30,9 @@ initial_width=2. nempty=30 [DensityMatrix] solver=MVP -nb_inner_it=100 +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 [Restart] output_level=4 [ROM] diff --git a/src/Control.cc b/src/Control.cc index 151fe920..cbf2eadd 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,6 +224,7 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; + os << std::setprecision(4) << std::scientific << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " @@ -439,7 +440,7 @@ void Control::sync(void) memset(&int_buffer[0], 0, size_int_buffer * sizeof(int)); } - const short size_float_buffer = 43; + const short size_float_buffer = 44; float* float_buffer = new float[size_float_buffer]; if (mype_ == 0) { @@ -485,6 +486,7 @@ void Control::sync(void) float_buffer[40] = threshold_eigenvalue_gram_quench_; float_buffer[41] = pair_mlwf_distance_threshold_; float_buffer[42] = e0_; + float_buffer[43] = dm_tol; } else { @@ -680,6 +682,7 @@ void Control::sync(void) threshold_eigenvalue_gram_quench_ = float_buffer[40]; pair_mlwf_distance_threshold_ = float_buffer[41]; e0_ = float_buffer[42]; + dm_tol = float_buffer[43]; max_electronic_steps_loose_ = max_electronic_steps; delete[] short_buffer; @@ -699,8 +702,8 @@ void Control::setDefaultValues() void Control::adjust() { - // change dm_mix default to 1. if not using Davidson - if (it_algo_type_ != 2 && dm_mix < 0.) dm_mix = 1.; + // change dm_mix default to 1. if not using Davidson or MVP + if ((it_algo_type_ != 2 && DM_solver_ != 1) && dm_mix < 0.) dm_mix = 1.; if (nel_ - 2 * numst == 0) { @@ -1720,6 +1723,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) lrs_extrapolation = 10; dm_mix = vm["DensityMatrix.mixing"].as(); + dm_tol = vm["DensityMatrix.tol"].as(); dm_inner_steps = vm["DensityMatrix.nb_inner_it"].as(); dm_use_old_ = vm["DensityMatrix.use_old"].as() ? 1 : 0; str = vm["DensityMatrix.algo"].as(); @@ -1739,8 +1743,6 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else dm_algo_ = 2; - dm_tol = vm["DensityMatrix.tol"].as(); - str = vm["DensityMatrix.solver"].as(); if (str.compare("Mixing") == 0) DM_solver_ = 0; if (str.compare("MVP") == 0) DM_solver_ = 1; diff --git a/src/Ions.cc b/src/Ions.cc index 072f0950..1879e1ba 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1315,22 +1315,6 @@ void Ions::setLocalForces( { if (ion->compareName(*s)) { - //std::cout << "Ion found: " << ion->name() << std::endl; - //std::cout << "Ion force: (" << *it << ", " << *(it + 1) << ", " << *(it + 2) << ")" << std::endl; - //std::cout << "names: "; - //for (int i = 0; i < names.size(); i++) - //{ - // std::cout << names[i]; - // if (i == forces.size() - 1) std::cout << std::endl; - // else std::cout << ", "; - //} - //std::cout << "forces: "; - //for (int i = 0; i < forces.size(); i++) - //{ - // std::cout << forces[i]; - // if (i == forces.size() - 1) std::cout << ")" << std::endl; - // else std::cout << ", "; - //} ion->set_force(0, *it); ion->set_force(1, *(it + 1)); ion->set_force(2, *(it + 2)); @@ -1364,9 +1348,6 @@ void Ions::setLocalForces( double d = std::sqrt(d2); if (d < tol) { - std::cout << "Ion found: " << ion->name() << std::endl; - std::cout << "Ion position:( " << *cit << ", " << *(cit + 1) << ", " << *(cit + 2) << ")" << std::endl; - std::cout << "Ion force: (" << *fit << ", " << *(fit + 1) << ", " << *(fit + 2) << ")" << std::endl; ion->set_force(0, *fit); ion->set_force(1, *(fit + 1)); ion->set_force(2, *(fit + 2)); diff --git a/src/MGmol.cc b/src/MGmol.cc index facd362e..ec780d0e 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -102,6 +102,7 @@ extern Timer md_moveVnuc_tm; extern Timer md_updateMasks_tm; extern Timer md_extrapolateOrbitals_tm; extern Timer md_updateRhoAndPot_tm; +extern Timer md_updateDMandEnergy_tm; extern Timer quench_tm; extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; @@ -928,6 +929,7 @@ void MGmol::printTimers() init_nuc_tm_.print(os_); md_updateMasks_tm.print(os_); md_extrapolateOrbitals_tm.print(os_); + md_updateDMandEnergy_tm.print(os_); quench_tm.print(os_); evnl_tm_.print(os_); ions_setupInteractingIons_tm.print(os_); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6990e29d..1c093e53 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -47,13 +47,16 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm) + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm) : comm_(comm), os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), ions_(ions), - numst_(numst) + numst_(numst), + mixing_(mixing), + tol_de0_(tol_de0) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -207,7 +210,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) kbpsi.computeHvnlMatrix(&kbpsi, ions_, h11_nl); - const double tol_de0 = 1.e-12; for (int inner_it = 0; inner_it < n_inner_steps_; inner_it++) { if (onpe0 && ct.verbose > 1) @@ -270,7 +272,8 @@ int MVPSolver::solve(OrbitalsType& orbitals) double de0 = evaluateDerivative(dmInit, delta_dm, ts0); - if (std::abs(de0) < tol_de0 && inner_it > 0) + // check for convergence + if (std::abs(de0) < tol_de0_ && inner_it > 0) { if (onpe0 && ct.verbose > 0) std::cout << "MVP: de0 = " << de0 @@ -278,54 +281,70 @@ int MVPSolver::solve(OrbitalsType& orbitals) break; } - // - // evaluate free energy at beta=1 - // - if (onpe0 && ct.verbose > 2) - std::cout << "MVP --- Target energy..." << std::endl; - proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); - proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); - const double nel = proj_mat_work_->getNel(); - if (onpe0 && ct.verbose > 1) - os_ << "MVP --- Number of electrons at beta=1 : " << nel - << std::endl; - - rho_->computeRho(orbitals, target); - - mgmol_strategy_->update_pot(vh_init, ions_); - - energy_->saveVofRho(); - - // update h11 + double beta = 0.; + if (mixing_ > 0.) { - h11 = h11_nl; - mgmol_strategy_->addHlocal2matrix(orbitals, orbitals, h11); + beta = mixing_; + if (onpe0 && ct.verbose > 0) + { + if (ct.verbose > 1) + os_ << "MVP with beta = " << beta << std::endl; + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << std::endl; + } } - - proj_mat_work_->assignH(h11); - proj_mat_work_->setHB2H(); - - const double ts1 - = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); - - // line minimization - const double beta - = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); - assert(!std::isnan(beta)); - - if (onpe0 && ct.verbose > 0) + else { - os_ << std::setprecision(12); - os_ << std::fixed << "MVP inner iteration " << inner_it - << ", E0=" << e0 << ", E1=" << e1; - os_ << std::scientific << ", E0'=" << de0 - << " -> beta=" << beta; - os_ << std::endl; - } + // + // evaluate free energy at beta=1 + // + if (onpe0 && ct.verbose > 2) + std::cout << "MVP --- Target energy..." << std::endl; + proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); + proj_mat_work_->computeOccupationsFromDM(); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); + const double nel = proj_mat_work_->getNel(); + if (onpe0 && ct.verbose > 1) + os_ << "MVP --- Number of electrons at beta=1 : " << nel + << std::endl; + + rho_->computeRho(orbitals, target); + + mgmol_strategy_->update_pot(vh_init, ions_); + + energy_->saveVofRho(); + + // update h11 + { + h11 = h11_nl; + mgmol_strategy_->addHlocal2matrix( + orbitals, orbitals, h11); + } + + proj_mat_work_->assignH(h11); + proj_mat_work_->setHB2H(); + + const double ts1 + = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); + const double e1 = energy_->evaluateTotal(ts1, + proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); + + // line minimization + beta + = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); + assert(!std::isnan(beta)); + if (onpe0 && ct.verbose > 0) + { + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << ", E1=" << e1; + os_ << std::scientific << ", E0'=" << de0 + << " -> beta=" << beta; + os_ << std::endl; + } + } // update DM *work_ = dmInit; work_->axpy(beta, delta_dm); diff --git a/src/MVPSolver.h b/src/MVPSolver.h index 8e7a7c70..77a6a345 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -33,6 +33,12 @@ class MVPSolver Ions& ions_; int numst_; + double mixing_; + + /*! + * tolerance on energy slope in inner iterations + */ + double tol_de0_; Rho* rho_; Energy* energy_; @@ -56,7 +62,8 @@ class MVPSolver Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm); + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm); ~MVPSolver(); int solve(OrbitalsType& orbitals); diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 8712ae92..3fb53dee 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -54,7 +54,7 @@ int MVP_DMStrategy::update(OrbitalsType& orbitals) MVPSolver solver(comm_, os_, ions_, rho_, energy_, electrostat_, mgmol_strategy_, ct.numst, ct.occ_width, global_indexes_, - ct.dm_inner_steps, use_old_dm_); + ct.dm_inner_steps, ct.dm_mix, ct.dm_tol, use_old_dm_); return solver.solve(orbitals); } diff --git a/src/md.cc b/src/md.cc index bc986955..e8d06133 100644 --- a/src/md.cc +++ b/src/md.cc @@ -50,6 +50,7 @@ Timer md_tau_tm("md_tau"); Timer md_moveVnuc_tm("md_moveVnuc"); Timer md_updateMasks_tm("md_updateMasks"); Timer md_extrapolateOrbitals_tm("md_extrapolateOrbitals"); +Timer md_updateDMandEnergy_tm("md_updateDMandEnergy"); #define DUMP_MAX_NUM_TRY 5 @@ -485,7 +486,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ROM_MVP) { + md_updateDMandEnergy_tm.start(); updateDMandEnergy(**orbitals, *ROM_ions, eks); + md_updateDMandEnergy_tm.stop(); } else { diff --git a/src/read_config.cc b/src/read_config.cc index 884aa8d3..9dbd56d3 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -302,7 +302,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(0), "Flag for computing new centers from extrapolated orbitals.")( "DensityMatrix.mixing", po::value()->default_value(-1.), - "Mixing coefficient for Density Matrix")("DensityMatrix.solver", + "Mixing coefficient for Density Matrix")("DensityMatrix.tol", + po::value()->default_value(1.e-12), + "Tolerance for Density Matrix convergence")("DensityMatrix.solver", po::value()->default_value("Mixing"), "Algorithm for updating Density Matrix: Mixing, MVP, HMVP")( "DensityMatrix.nb_inner_it", po::value()->default_value(3), @@ -324,10 +326,7 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(100), "Maximum number of iterations for power method " "to compute interval for Chebyshev " - "approximation of density matrix. ")("DensityMatrix.tol", - po::value()->default_value(1.e-7), - "tolerance, used in iterative DM computation convergence " - "criteria"); + "approximation of density matrix. "); po::options_description cmdline_options; cmdline_options.add(generic); diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 8fbd4cb3..f8d70eec 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -841,6 +841,8 @@ void testROMIonDensity(MGmolInterface *mgmol_) mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); } + /* Artificial ions object to avoid repeated setPositions */ + Ions* new_ions; /* Collect fictitious ion density based on each configuration */ std::vector> fom_rhoc(num_snap); /* Sanity check for overlappingVL_ions */ @@ -848,24 +850,25 @@ void testROMIonDensity(MGmolInterface *mgmol_) for (int idx = 0; idx < num_snap; idx++) { /* set ion positions */ - ions->setPositions(cfgs[idx], atnumbers); + new_ions = new Ions(cfgs[idx], atnumbers, lattice, ions->getSpecies()); /* save overlapping ions for sanity check */ - fom_overlap_ions[idx].resize(ions->overlappingVL_ions().size()); - for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + fom_overlap_ions[idx].resize(new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) { fom_overlap_ions[idx][k].resize(3); for (int d = 0; d < 3; d++) - fom_overlap_ions[idx][k][d] = ions->overlappingVL_ions()[k]->position(d); + fom_overlap_ions[idx][k][d] = new_ions->overlappingVL_ions()[k]->position(d); } /* compute resulting ion density */ /* NOTE: we exclude rescaling for the sake of verification */ - pot.initialize(*ions); + mgmol->setupPotentials(*new_ions); - mgmol->electrostat_->setupRhoc(pot.rho_comp()); fom_rhoc[idx].resize(dim); mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); + + delete new_ions; } /* Initialize libROM classes */ @@ -911,23 +914,32 @@ void testROMIonDensity(MGmolInterface *mgmol_) if (rank == 0) printf("test index: %d\n", test_idx); /* set ion positions */ - ions->setPositions(cfgs[test_idx], atnumbers); + new_ions = new Ions(cfgs[test_idx], atnumbers, lattice, ions->getSpecies()); /* Sanity check for overlapping ions */ - CAROM_VERIFY(fom_overlap_ions[test_idx].size() == ions->overlappingVL_ions().size()); - for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + CAROM_VERIFY(fom_overlap_ions[test_idx].size() == new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) for (int d = 0; d < 3; d++) - CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - new_ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + + /* set up potentials */ + mgmol->setupPotentials(*new_ions); /* eval ion density on sample grid points */ std::vector sampled_rhoc(sampled_row.size()); - pot.evalIonDensityOnSamplePts(*ions, sampled_row, sampled_rhoc); + pot.evalIonDensityOnSamplePts(*new_ions, sampled_row, sampled_rhoc); + // For now, we relax the threshold to allow the slight difference of the values at the sampled indices. + // This is because the rescaling procedure after getting the radial data on mesh requires global information, + // thus cannot be done in the ROM level only with sampled values. + // After we have the DEIM reconstruction, we will do rescaling and revisit this. for (int d = 0; d < sampled_row.size(); d++) { printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); - CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-12); + CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-2); } + + delete new_ions; } template void readRestartFiles(MGmolInterface *mgmol_); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b4844557..f55ce699 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -511,6 +511,13 @@ add_test(NAME testMVP ${CMAKE_CURRENT_SOURCE_DIR}/MVP/mvp.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MVP/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMVPmix + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME BandGapN2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/BandGapN2/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/test.py b/tests/MVP/test.py index bd5d708e..57d9522e 100644 --- a/tests/MVP/test.py +++ b/tests/MVP/test.py @@ -51,6 +51,8 @@ energies=[] print("Check forces are smaller than tol = {}".format(tol)) for line in lines: + if line.count(b'MVP'): + print(line) if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/MVPmix/h2o.xyz b/tests/MVPmix/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/MVPmix/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/MVPmix/mgmol.cfg b/tests/MVPmix/mgmol.cfg new file mode 100644 index 00000000..10edc464 --- /dev/null +++ b/tests/MVPmix/mgmol.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-8 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=random +initial_width=1.5 +temperature=10. +nempty=5 +[Restart] +output_level=0 +[DensityMatrix] +solver=MVP +nb_inner_it=2 +mixing=0.5 diff --git a/tests/MVPmix/test.py b/tests/MVPmix/test.py new file mode 100644 index 00000000..8ffafdbb --- /dev/null +++ b/tests/MVPmix/test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test MVP solver with mixing coefficient...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-4): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-4] +inp = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +cwd = os.getcwd() + +dst = 'pseudo.O_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +dst = 'pseudo.H_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True) + +#analyse mgmol standard output +#make sure force is below tolerance +lines=output.split(b'\n') + +convergence=0 +for line in lines: + if line.count(b'DFTsolver:') and line.count(b'convergence'): + convergence=1 + break + +if convergence==0: + print("MVP Solver did not converge") + sys.exit(1) + +flag = 0 +eigenvalues=[] +energies=[] +ecount=0 +for line in lines: + if line.count(b'FERMI'): + flag = 0 + if flag==1: + words=line.split() + for w in words: + eigenvalues.append(eval(w)) + if line.count(b'Eigenvalues'): + flag = 1 + eigenvalues=[] + if line.count(b'%%'): + words=line.split() + e=words[5][0:-1] + print(e) + ecount=ecount+1 + energies.append(eval(e)) +print(energies) + +print(eigenvalues) +tol = 1.e-4 +eigenvalue0 = -0.916 +if abs(eigenvalues[0]-eigenvalue0)>tol: + print("Expected eigenvalue 0 to be {}".format(eigenvalue0)) + sys.exit(1) +eigenvalue8 = 0.219 +if abs(eigenvalues[8]-eigenvalue8)>tol: + print("Expected eigenvalue 8 to be {}".format(eigenvalue8)) + sys.exit(1) + +niterations = ecount +print("MVP solver ran for {} iterations".format(niterations)) +if niterations>180: + print("MVP test FAILED for taking too many iterations") + sys.exit(1) + +print("Check energy...") +last_energy = energies[-1] +print("Energy = {}".format(last_energy)) +if last_energy>-17.16269: + print("Last energy = {}".format(last_energy)) + sys.exit(1) + +print("Test PASSED") +sys.exit(0) diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis index cab929d1..35f564a3 100644 --- a/tests/PinnedH2O_3DOF/job.basis +++ b/tests/PinnedH2O_3DOF/job.basis @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline index 243f563a..5b4224f0 100644 --- a/tests/PinnedH2O_3DOF/job.offline +++ b/tests/PinnedH2O_3DOF/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online index 1373e011..ea75e59f 100644 --- a/tests/PinnedH2O_3DOF/job.online +++ b/tests/PinnedH2O_3DOF/job.online @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online_rotate b/tests/PinnedH2O_3DOF/job.online_rotate index 3f8a65ad..a2065b72 100644 --- a/tests/PinnedH2O_3DOF/job.online_rotate +++ b/tests/PinnedH2O_3DOF/job.online_rotate @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index 64765c7a..f9039abe 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,9 @@ int main(int argc, char** argv) } } + // rename output restart file + ct.out_restart_file = ct.out_restart_file + "1"; + mgmol->cleanup(); delete mgmol; From 81ab133cffa4c33a4be4b541059de2a07f785b4c Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Tue, 15 Jul 2025 13:30:56 -0700 Subject: [PATCH 46/50] Merge release (#351) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ * Add cleanup and MGmolInterface (#331) * Add function to set local forces (#333) * based on matching coordinates to local ions * Fix setLocalForces (#335) * Fix test WFEnergyAndForces (#336) * writing restart twice in same directory was failing * Add mixing option in MVP (#337) * Update MVP with mixing (#339) * use tolerance to terminate iterations * tune verbosity * Update DensityMatrix class (#342) * add extrapolation function * remove unused function * simplify use of iterative index * Fixing DM tolerance (#341) * Fix Density Matrix tolerance read --------- Co-authored-by: Jean-Luc Fattebert * Rm oversubscribe from CMakeLists.txt (#346) * already set in build script * Adapt to new environment on condo @ ORNL (#348) --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- scripts/build_condo-mod.sh | 8 +- scripts/build_condo.sh | 21 ++--- scripts/modules.condo | 9 +- scripts/modules.condo-mod | 10 +-- src/Control.cc | 3 +- src/DavidsonSolver.cc | 12 +-- src/DensityMatrix.cc | 85 ++++++++++--------- src/DensityMatrix.h | 40 +++++---- src/DensityMatrixSparse.cc | 15 ++-- src/DensityMatrixSparse.h | 16 ++-- src/EigenDMStrategy.cc | 2 +- src/HamiltonianMVPSolver.cc | 22 ++--- src/MGmol.cc | 4 +- src/MVPSolver.cc | 12 +-- src/NonOrthoDMStrategy.cc | 6 +- src/ProjectedMatrices.cc | 52 ++++++------ src/ProjectedMatrices.h | 40 ++++----- src/ProjectedMatrices2N.cc | 4 +- src/ProjectedMatrices2N.h | 4 +- src/ProjectedMatricesInterface.h | 22 ++--- src/ProjectedMatricesSparse.h | 9 +- src/md.cc | 2 +- tests/CMakeLists.txt | 4 +- .../testDMandEnergyAndForces.cc | 2 +- tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc | 2 +- .../testRestartEnergyAndForces.cc | 2 +- tests/testDensityMatrix.cc | 4 +- 27 files changed, 181 insertions(+), 231 deletions(-) diff --git a/scripts/build_condo-mod.sh b/scripts/build_condo-mod.sh index d9abe034..4dfe5fc6 100755 --- a/scripts/build_condo-mod.sh +++ b/scripts/build_condo-mod.sh @@ -3,9 +3,6 @@ #Before compiling, load the following modules: source scripts/modules.condo-mod -# We need to define the cmake blas vendor option here to find the right one. -BLAS_VENDOR=OpenBLAS - MGMOL_ROOT=`pwd` INSTALL_DIR=${MGMOL_ROOT}/install @@ -15,16 +12,17 @@ BUILD_DIR=${MGMOL_ROOT}/build mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} +SCALAPACK_DIR=/home/q8j/Software/ScaLapack/scalapack-2.2.2 + # call cmake cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ - -DBLA_VENDOR=${BLAS_VENDOR} \ -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.3" \ + -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.5" \ -DMPIEXEC_EXECUTABLE=${OPENMPI_DIR}/bin/mpiexec \ .. diff --git a/scripts/build_condo.sh b/scripts/build_condo.sh index 3a349cbe..68aa16dc 100755 --- a/scripts/build_condo.sh +++ b/scripts/build_condo.sh @@ -1,41 +1,30 @@ #/bin/bash ## An example script to build on ONRL condo systems (CADES). -## This script assumes intel/ mkl libraries are being used. #Before compiling, load the following modules: source scripts/modules.condo -# set some environment variables using loaded module path -export SCALAPACK_ROOT=${MKLROOT} - -# We need to define the cmake blas vendor option here to find the right one. -BLAS_VENDOR=Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -BLACS_LIB=${MKLROOT}/lib/intel64 - MGMOL_ROOT=`pwd` -INSTALL_DIR=${MGMOL_ROOT}/mgmol_install +INSTALL_DIR=${MGMOL_ROOT}/install mkdir -p ${INSTALL_DIR} BUILD_DIR=${MGMOL_ROOT}/build mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} +SCALAPACK_DIR=/home/q8j/Software/ScaLapack/scalapack-2.2.2 + # call cmake cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ - -DHDF5_LIBRARIES=${HDF5_DIR}/lib/libhdf5.so \ - -DHDF5_HL_LIBRARIES=${HDF5_DIR}/lib/libhdf5_hl.so \ - -DHDF5_INCLUDE_DIRS=${HDF5_DIR}/include \ - -DBLA_VENDOR=${BLAS_VENDOR} \ + -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ + -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.5" \ -DMPIEXEC_EXECUTABLE=${OPENMPI_DIR}/bin/mpiexec \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_openmpi_lp64.so \ .. # call make install diff --git a/scripts/modules.condo b/scripts/modules.condo index 6f27e62b..de36562f 100644 --- a/scripts/modules.condo +++ b/scripts/modules.condo @@ -1,6 +1,5 @@ -module load PE-intel/3.0 -module load boost/1.67.0-pe3 -module load mkl -module load hdf5_parallel/1.10.3 -module load cmake/3.18.4 +module load hdf5 +module load boost +module load cmake module load python +module load openblas diff --git a/scripts/modules.condo-mod b/scripts/modules.condo-mod index 18377905..de36562f 100644 --- a/scripts/modules.condo-mod +++ b/scripts/modules.condo-mod @@ -1,7 +1,5 @@ -module load PE-gnu/3.0 -module load hdf5-parallel/1.8.20 +module load hdf5 module load boost -module load cmake/3.20.3 -module load python/3.6.6 -module load openBLAS/0.2.19 -module load scalapack/2.0.2 +module load cmake +module load python +module load openblas diff --git a/src/Control.cc b/src/Control.cc index cbf2eadd..980c1abe 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,7 +224,8 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; - os << std::setprecision(4) << std::scientific << " Density matrix tol = " << dm_tol << std::endl; + os << std::setprecision(4) << std::scientific + << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index a049fde7..c34b6396 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -206,8 +206,8 @@ double DavidsonSolver::evaluateDerivative( const double dbeta = 0.0001; *work2N_ = dm2Ninit; work2N_->axpy(dbeta, delta_dm); - // proj_mat2N_->setDM(*work2N_,orbitals.getIterativeIndex()); - proj_mat2N_->setDM(*work2N_, -1); + + proj_mat2N_->setDM(*work2N_); proj_mat2N_->computeOccupationsFromDM(); const double tsd0e = evalEntropy(proj_mat2N_.get(), false, os_); @@ -250,7 +250,7 @@ void DavidsonSolver::buildTarget2N_MVP( // if( onpe0 )os_<<"Compute X2N..."<setHB2H(); - proj_mat2N_->updateDM(orbitals_index); + proj_mat2N_->updateDM(); target = proj_mat2N_->dm(); @@ -584,7 +584,7 @@ int DavidsonSolver::solve( // if (mmpi.PE0() && ct.verbose > 2) os_ << "Target energy..." << std::endl; - proj_mat2N_->setDM(target, orbitals.getIterativeIndex()); + proj_mat2N_->setDM(target); proj_mat2N_->computeOccupationsFromDM(); double nel = proj_mat2N_->getNel(); if (mmpi.PE0() && ct.verbose > 2) @@ -643,7 +643,7 @@ int DavidsonSolver::solve( // update DM *work2N_ = dm2Ninit; work2N_->axpy(beta, delta_dm); - proj_mat2N_->setDM(*work2N_, orbitals.getIterativeIndex()); + proj_mat2N_->setDM(*work2N_); if (inner_it < ct.dm_inner_steps - 1) { @@ -809,7 +809,7 @@ int DavidsonSolver::solve( = dynamic_cast*>( orbitals.getProjMatrices()); assert(pmat); - pmat->buildDM(new_occ, orbitals.getIterativeIndex()); + pmat->buildDM(new_occ); if (retval == 0) break; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 78d7008e..83d715d5 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -22,7 +22,11 @@ const double factor_kernel4dot = 10.; -#define PROCRUSTES 0 +#define MGMOL_DENSITYMATRIX_FAIL(X) \ + { \ + std::cerr << "DensityMatrix failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } #define MGMOL_DENSITYMATRIX_FAIL(X) \ { \ @@ -36,7 +40,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) : dim_(ndim), - orbitals_index_(-1), + update_index_(-1), occ_uptodate_(false), uniform_occ_(false), stripped_(false) @@ -66,8 +70,8 @@ DensityMatrix::~DensityMatrix() } template -void DensityMatrix::build(const MatrixType& zmat, - const std::vector& occ, const int new_orbitals_index) +void DensityMatrix::build( + const MatrixType& zmat, const std::vector& occ) { #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -98,32 +102,30 @@ void DensityMatrix::build(const MatrixType& zmat, work_->symm('r', 'l', 1., gamma, zmat, 0.); kernel4dot_->gemm('n', 't', 1., *work_, zmat, 0.); - stripped_ = false; - orbitals_index_ = new_orbitals_index; + stripped_ = false; + update_index_++; } template -void DensityMatrix::build( - const MatrixType& zmat, const int new_orbitals_index) +void DensityMatrix::build(const MatrixType& zmat) { - build(zmat, occupation_, new_orbitals_index); + build(zmat, occupation_); } // build diagonal matrix template -void DensityMatrix::build( - const std::vector& occ, const int new_orbitals_index) +void DensityMatrix::build(const std::vector& occ) { assert(dm_ != nullptr); assert(!occ.empty()); setOccupations(occ); - build(new_orbitals_index); + build(); } template -void DensityMatrix::build(const int new_orbitals_index) +void DensityMatrix::build() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); #ifdef PRINT_OPERATIONS @@ -148,13 +150,12 @@ void DensityMatrix::build(const int new_orbitals_index) * std::min(1., factor_kernel4dot * occupation_[i])); kernel4dot_->setDiagonal(w); - stripped_ = false; - orbitals_index_ = new_orbitals_index; + stripped_ = false; + update_index_++; } template -void DensityMatrix::setUniform( - const double nel, const int new_orbitals_index) +void DensityMatrix::setUniform(const double nel) { assert(!occupation_.empty()); @@ -171,22 +172,13 @@ void DensityMatrix::setUniform( uniform_occ_ = true; - build(occupation_, new_orbitals_index); -} - -template -void DensityMatrix::buildFromBlock(const MatrixType& block00) -{ - dm_->clear(); - dm_->assign(block00, 0, 0); - dm_->print(std::cout, 0, 0, 25, 25); + build(occupation_); } template void DensityMatrix::rotate( const MatrixType& rotation_matrix, const bool flag_eigen) { - if (!flag_eigen) { MatrixType invU(rotation_matrix); @@ -204,6 +196,8 @@ void DensityMatrix::rotate( invU.getrs('n', tmp, ipiv); *dm_ = tmp; + + update_index_++; } } @@ -370,8 +364,7 @@ double DensityMatrix::computeEntropy() const } template -void DensityMatrix::setto2InvS( - const MatrixType& invS, const int orbitals_index) +void DensityMatrix::setto2InvS(const MatrixType& invS) { *dm_ = invS; dm_->scal(orbital_occupation_); @@ -382,8 +375,8 @@ void DensityMatrix::setto2InvS( occupation_[st] = 1.; occ_uptodate_ = true; } - uniform_occ_ = false; - orbitals_index_ = orbitals_index; + uniform_occ_ = false; + update_index_++; } template @@ -400,8 +393,7 @@ void DensityMatrix::stripS(const MatrixType& ls) } template -void DensityMatrix::dressUpS( - const MatrixType& ls, const int new_orbitals_index) +void DensityMatrix::dressUpS(const MatrixType& ls) { assert(stripped_); @@ -410,10 +402,11 @@ void DensityMatrix::dressUpS( *dm_ = *work_; ls.trtrs('l', 't', 'n', *dm_); - orbitals_index_ = new_orbitals_index; - occ_uptodate_ = false; - uniform_occ_ = false; - stripped_ = false; + update_index_++; + + occ_uptodate_ = false; + uniform_occ_ = false; + stripped_ = false; } // dm_ -> u*dm_*u^T @@ -424,6 +417,8 @@ void DensityMatrix::transform(const MatrixType& u) { work_->gemm('n', 't', 1., *dm_, u, 0.); dm_->gemm('n', 'n', 1., u, *work_, 0.); + + update_index_++; } template @@ -434,13 +429,21 @@ double DensityMatrix::getExpectation(const MatrixType& A) } template -void DensityMatrix::mix( - const double mix, const MatrixType& matA, const int new_orbitals_index) +void DensityMatrix::mix(const double mix, const MatrixType& matA) { dm_->scal(1. - mix); - dm_->axpy(mix, matA); - orbitals_index_ = new_orbitals_index; + + update_index_++; +} + +template +void DensityMatrix::linearExtrapolate(const MatrixType& previous_dm) +{ + dm_->scal(2.); + dm_->axpy(-1., previous_dm); + + update_index_++; } template diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 1d09e79e..8ce74542 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -31,7 +31,10 @@ class DensityMatrix MatrixType* kernel4dot_; MatrixType* work_; - int orbitals_index_; + /*! + * Keep track of changes, incremented every time dm_ is updated + */ + int update_index_; bool occ_uptodate_; bool uniform_occ_; @@ -46,16 +49,16 @@ class DensityMatrix DensityMatrix& operator=(const DensityMatrix&); DensityMatrix(const DensityMatrix&); - void build(const int new_orbitals_index); + void build(); public: DensityMatrix(const int ndim); ~DensityMatrix(); - void setUniform(const double nel, const int new_orbitals_index); + void setUniform(const double nel); - int getOrbitalsIndex() const { return orbitals_index_; } + int getIndex() const { return update_index_; } bool occupationsUptodate() const { return occ_uptodate_; } bool fromUniformOccupations() const { return uniform_occ_; } @@ -84,10 +87,10 @@ class DensityMatrix const MatrixType& kernel4dot() const { return *kernel4dot_; } - void setMatrix(const MatrixType& mat, const int orbitals_index) + void setMatrix(const MatrixType& mat) { - *dm_ = mat; - orbitals_index_ = orbitals_index; + *dm_ = mat; + update_index_++; setDummyOcc(); @@ -112,7 +115,7 @@ class DensityMatrix uniform_occ_ = false; stripped_ = false; - orbitals_index_ = 0; + update_index_ = 0; } void getOccupations(std::vector& occ) const @@ -126,22 +129,19 @@ class DensityMatrix void setOccupations(const std::vector& occ); - void setto2InvS(const MatrixType& invS, const int orbitals_index); + void setto2InvS(const MatrixType& invS); void stripS(const MatrixType& ls); - void dressUpS(const MatrixType& ls, const int new_orbitals_index); + void dressUpS(const MatrixType& ls); // dm_ -> u*dm_*u^T void transform(const MatrixType& u); - void buildFromBlock(const MatrixType& block00); - double computeEntropy() const; void computeOccupations(const MatrixType& ls); - void build(const std::vector& occ, const int new_orbitals_index); - void build(const MatrixType& z, const int new_orbitals_index); - void build(const MatrixType& z, const std::vector& occ, - const int new_orbitals_index); + void build(const std::vector& occ); + void build(const MatrixType& z); + void build(const MatrixType& z, const std::vector& occ); void rotate(const MatrixType& rotation_matrix, const bool flag_eigen); void printOccupations(std::ostream& os) const; @@ -149,8 +149,12 @@ class DensityMatrix void diagonalize( const char eigv, std::vector& occ, MatrixType& vect); double getExpectation(const MatrixType& A); - void mix( - const double mix, const MatrixType& matA, const int new_orbitals_index); + void mix(const double mix, const MatrixType& matA); + + /*! + * dm <- dm + (dm-previous_dm) = 2.*dm - previous_dm + */ + void linearExtrapolate(const MatrixType& previous_dm); int write(HDFrestart& h5f_file, std::string& name); int read(HDFrestart& h5f_file, std::string& name); diff --git a/src/DensityMatrixSparse.cc b/src/DensityMatrixSparse.cc index 4217b8ac..e554501a 100644 --- a/src/DensityMatrixSparse.cc +++ b/src/DensityMatrixSparse.cc @@ -26,7 +26,7 @@ DensityMatrixSparse::DensityMatrixSparse( MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; - orbitals_index_ = -1; + update_index_ = -1; if (dim_ > 0) { @@ -44,27 +44,24 @@ DensityMatrixSparse::~DensityMatrixSparse() } } -void DensityMatrixSparse::setUniform(const double nel, const int orbitals_index) +void DensityMatrixSparse::setUniform(const double nel) { const double occ = (double)((double)nel / (double)dim_); assert(occ < 1.01); - orbitals_index_ = orbitals_index; + update_index_++; (*dm_).reset(); const double uval = (double)occ * orbital_occupation_; for (std::vector::const_iterator st = locvars_.begin(); st != locvars_.end(); ++st) (*dm_).insertMatrixElement(*st, *st, uval, INSERT, true); - - return; } -void DensityMatrixSparse::setto2InvS( - const VariableSizeMatrix& invS, const int orbitals_index) +void DensityMatrixSparse::setto2InvS(const VariableSizeMatrix& invS) { *dm_ = invS; dm_->scale(orbital_occupation_); - orbitals_index_ = orbitals_index; + update_index_++; } // build density matrix, given computed locally centered data void DensityMatrixSparse::assembleMatrixFromCenteredData( @@ -92,7 +89,7 @@ void DensityMatrixSparse::assembleMatrixFromCenteredData( dtor_DM.updateLocalRows((*dm_)); gather_DM_tm_.stop(); - orbitals_index_ = orbitals_index; + update_index_ = orbitals_index; } // compute trace of dot product dm_ . vsmat double DensityMatrixSparse::getTraceDotProductWithMat( diff --git a/src/DensityMatrixSparse.h b/src/DensityMatrixSparse.h index b574bd0b..170057fa 100644 --- a/src/DensityMatrixSparse.h +++ b/src/DensityMatrixSparse.h @@ -32,7 +32,7 @@ class DensityMatrixSparse VariableSizeMatrix* dm_; - int orbitals_index_; + int update_index_; double orbital_occupation_; @@ -43,16 +43,14 @@ class DensityMatrixSparse ~DensityMatrixSparse(); - void setUniform(const double nel, const int new_orbitals_index); + void setUniform(const double nel); - void setto2InvS( - const VariableSizeMatrix& invS, const int orbitals_index); - int getOrbitalsIndex() const { return orbitals_index_; } - void setMatrix( - const VariableSizeMatrix& mat, const int orbitals_index) + void setto2InvS(const VariableSizeMatrix& invS); + int getIndex() const { return update_index_; } + void setMatrix(const VariableSizeMatrix& mat) { - *dm_ = mat; - orbitals_index_ = orbitals_index; + *dm_ = mat; + update_index_++; } void assembleMatrixFromCenteredData(const std::vector& data, const std::vector& locRowIds, const int* globalColIds, diff --git a/src/EigenDMStrategy.cc b/src/EigenDMStrategy.cc index 8fd8443a..358ed5f9 100644 --- a/src/EigenDMStrategy.cc +++ b/src/EigenDMStrategy.cc @@ -37,7 +37,7 @@ int EigenDMStrategy::update(OrbitalsType& orbitals) = dynamic_cast< ProjectedMatrices>*>( proj_matrices_); - pmat->updateDMwithEigenstatesAndRotate(orbitals.getIterativeIndex(), zz); + pmat->updateDMwithEigenstatesAndRotate(zz); // if( onpe0 && ct.verbose>2 ) // (*MPIdata::sout)<<"get_dm_diag: rotate orbitals "<::solve( ProjMatrixType* projmatrices = dynamic_cast(orbitals.getProjMatrices()); - int iterative_index = 0; - // save computed vh for a fair energy "comparison" with vh computed // in close neigborhood const pb::GridFunc vh_init(electrostat_->getVh()); @@ -134,13 +132,11 @@ int HamiltonianMVPSolver::solve( // // evaluate energy at origin // - iterative_index++; - projmatrices->assignH(*hmatrix_); projmatrices->setHB2H(); // update DM and compute entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double ts0 = evalEntropyMVP(projmatrices, true, os_); // Update density rho_->update(orbitals); @@ -168,10 +164,8 @@ int HamiltonianMVPSolver::solve( // MatrixType htarget(projmatrices->getH()); - iterative_index++; - // update DM and compute entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double ts1 = evalEntropyMVP(projmatrices, true, os_); // Update density rho_->update(orbitals); @@ -204,10 +198,8 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(h11); projmatrices->setHB2H(); - iterative_index++; - // update DM and entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double tsi = evalEntropyMVP(projmatrices, true, os_); // Update density @@ -262,10 +254,8 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(h11); projmatrices->setHB2H(); - iterative_index++; - // update DM and entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); tsi = evalEntropyMVP(projmatrices, true, os_); // Update density @@ -325,9 +315,7 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(*hmatrix_); projmatrices->setHB2H(); - iterative_index++; - - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); // Generate new density rho_->update(orbitals); diff --git a/src/MGmol.cc b/src/MGmol.cc index ec780d0e..ac022d91 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -365,9 +365,7 @@ int MGmol::initial() // initialize Rho if (ct.verbose > 0) printWithTimeStamp("Initialize Rho...", os_); - if (ct.restart_info <= 1) - proj_matrices_->setDMuniform( - ct.getNelSpin(), current_orbitals_->getIterativeIndex()); + if (ct.restart_info <= 1) proj_matrices_->setDMuniform(ct.getNelSpin()); rho_->setup(ct.getOrthoType(), current_orbitals_->getOverlappingGids()); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 1c093e53..4f9165a5 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -104,8 +104,8 @@ double MVPSolver::evaluateDerivative( const double dbeta = 0.001; *work_ = dmInit; work_->axpy(dbeta, delta_dm); - // proj_mat_work_->setDM(*work_,orbitals.getIterativeIndex()); - proj_mat_work_->setDM(*work_, -1); + + proj_mat_work_->setDM(*work_); proj_mat_work_->computeOccupationsFromDM(); const double tsd0e = evalEntropyMVP(proj_mat_work_, false, os_); @@ -145,7 +145,7 @@ void MVPSolver::buildTarget_MVP( // proj_mat_work_->setHB2H(); - proj_mat_work_->updateDM(orbitals_index); + proj_mat_work_->updateDM(); target = proj_mat_work_->dm(); if (ct.verbose > 2) @@ -301,7 +301,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) // if (onpe0 && ct.verbose > 2) std::cout << "MVP --- Target energy..." << std::endl; - proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); + proj_mat_work_->setDM(target); proj_mat_work_->computeOccupationsFromDM(); if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); const double nel = proj_mat_work_->getNel(); @@ -353,7 +353,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) { *work_ = target; } - proj_mat_work_->setDM(*work_, orbitals.getIterativeIndex()); + proj_mat_work_->setDM(*work_); if (inner_it < n_inner_steps_ - 1) { @@ -373,7 +373,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) ProjectedMatrices* projmatrices = dynamic_cast*>( orbitals.getProjMatrices()); - projmatrices->setDM(*work_, orbitals.getIterativeIndex()); + projmatrices->setDM(*work_); projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); projmatrices->assignH(proj_mat_work_->getH()); projmatrices->setHB2H(); diff --git a/src/NonOrthoDMStrategy.cc b/src/NonOrthoDMStrategy.cc index 3835b039..28a8f1d5 100644 --- a/src/NonOrthoDMStrategy.cc +++ b/src/NonOrthoDMStrategy.cc @@ -31,7 +31,7 @@ void NonOrthoDMStrategy::initialize(OrbitalsType& orbitals) (*MPIdata::sout) << "NonOrthoDMStrategy::initialize()..." << std::endl; } - proj_matrices_->updateDM(orbitals.getIterativeIndex()); + proj_matrices_->updateDM(); } template @@ -59,11 +59,11 @@ int NonOrthoDMStrategy::update(OrbitalsType& orbitals) if (mix_ < 1.) proj_matrices_->saveDM(); // compute new density matrix - proj_matrices_->updateDM(orbitals.getIterativeIndex()); + proj_matrices_->updateDM(); if (mix_ < 1.) { - proj_matrices_->updateDMwithRelax(mix_, orbitals.getIterativeIndex()); + proj_matrices_->updateDMwithRelax(mix_); } if (ct.verbose > 2) diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 74ff01ed..87c9fd48 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -271,7 +271,7 @@ void ProjectedMatrices::setDMto2InvS() if (mmpi.instancePE0() && ct.verbose > 1) std::cout << "ProjectedMatrices::setDMto2InvS()..." << std::endl; - dm_->setto2InvS(gm_->getInverse(), gm_->getAssociatedOrbitalsIndex()); + dm_->setto2InvS(gm_->getInverse()); } template @@ -296,30 +296,28 @@ void ProjectedMatrices::solveGenEigenProblem( } template -void ProjectedMatrices::buildDM( - const MatrixType& z, const int orbitals_index) +void ProjectedMatrices::buildDM(const MatrixType& z) { - dm_->build(z, orbitals_index); + dm_->build(z); } template -void ProjectedMatrices::buildDM(const MatrixType& z, - const std::vector& occ, const int orbitals_index) +void ProjectedMatrices::buildDM( + const MatrixType& z, const std::vector& occ) { - dm_->build(z, occ, orbitals_index); + dm_->build(z, occ); } template void ProjectedMatrices::buildDM( - const std::vector& occ, const int orbitals_index) + const std::vector& occ) { - dm_->build(occ, orbitals_index); + dm_->build(occ); } // Use Chebyshev approximation to compute chemical potential and density matrix template -void ProjectedMatrices::updateDMwithChebApproximation( - const int iterative_index) +void ProjectedMatrices::updateDMwithChebApproximation() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -354,15 +352,14 @@ void ProjectedMatrices::updateDMwithChebApproximation( } // compute chemical potential and density matrix with Chebyshev // approximation. - double final_mu = computeChemicalPotentialAndDMwithChebyshev( - order, emin, emax, iterative_index); + double final_mu + = computeChemicalPotentialAndDMwithChebyshev(order, emin, emax); if (mmpi.instancePE0() && ct.verbose > 1) std::cout << "Final mu_ = " << final_mu << " [Ha]" << std::endl; } template -void ProjectedMatrices::updateDMwithEigenstates( - const int iterative_index) +void ProjectedMatrices::updateDMwithEigenstates() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -381,14 +378,14 @@ void ProjectedMatrices::updateDMwithEigenstates( // Build the density matrix X // X = Z * gamma * Z^T - buildDM(zz, iterative_index); + buildDM(zz); } //"replicated" implementation of SP2. // Theta is replicated on each MPI task, and SP2 solve run independently // by each MPI task template -void ProjectedMatrices::updateDMwithSP2(const int iterative_index) +void ProjectedMatrices::updateDMwithSP2() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -433,21 +430,21 @@ void ProjectedMatrices::updateDMwithSP2(const int iterative_index) MatrixType dm("dm", dim_, dim_); sp2.getDM(dm, gm_->getInverse()); - dm_->setMatrix(dm, iterative_index); + dm_->setMatrix(dm); } template -void ProjectedMatrices::updateDM(const int iterative_index) +void ProjectedMatrices::updateDM() { Control& ct = *(Control::instance()); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (ct.DMEigensolver() == DMEigensolverType::Eigensolver) - updateDMwithEigenstates(iterative_index); + updateDMwithEigenstates(); else if (ct.DMEigensolver() == DMEigensolverType::Chebyshev) - updateDMwithChebApproximation(iterative_index); + updateDMwithChebApproximation(); else if (ct.DMEigensolver() == DMEigensolverType::SP2) - updateDMwithSP2(iterative_index); + updateDMwithSP2(); else { std::cerr << "Eigensolver not available in " @@ -470,7 +467,7 @@ void ProjectedMatrices::updateDM(const int iterative_index) template void ProjectedMatrices::updateDMwithEigenstatesAndRotate( - const int iterative_index, MatrixType& zz) + MatrixType& zz) { // solves generalized eigenvalue problem // and return solution in zz @@ -479,7 +476,7 @@ void ProjectedMatrices::updateDMwithEigenstatesAndRotate( rotateAll(zz, true); - dm_->build(zz, iterative_index); + dm_->build(zz); } template @@ -614,7 +611,7 @@ void ProjectedMatrices::dressupDM() if (mmpi.instancePE0()) std::cout << "ProjectedMatrices::dressupDM()" << std::endl; #endif - dm_->dressUpS(gm_->getCholeskyL(), gm_->getAssociatedOrbitalsIndex()); + dm_->dressUpS(gm_->getCholeskyL()); } template @@ -999,8 +996,7 @@ double ProjectedMatrices::computeTraceInvSmultMatMultTheta( template double ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( - const int order, const double emin, const double emax, - const int iterative_index) + const int order, const double emin, const double emax) { assert(emax > emin); assert(nel_ >= 0.); @@ -1175,7 +1171,7 @@ ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( dm.gemm('N', 'N', 1., tmp, gm_->getInverse(), 0.); double orbital_occupation = mmpi.nspin() > 1 ? 1. : 2.; dm.scal(orbital_occupation); - dm_->setMatrix(dm, iterative_index); + dm_->setMatrix(dm); return mu_; } diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index f2881763..142dbf70 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -80,8 +80,8 @@ class ProjectedMatrices : public ProjectedMatricesInterface { computeChemicalPotentialAndOccupations(width_, dim_); } - double computeChemicalPotentialAndDMwithChebyshev(const int order, - const double emin, const double emax, const int iterative_index); + double computeChemicalPotentialAndDMwithChebyshev( + const int order, const double emin, const double emax); protected: // indexes corresponding to valid function in each subdomain @@ -197,12 +197,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface int getDMMatrixIndex() const override { assert(dm_); - return dm_->getOrbitalsIndex(); - } - void setDMuniform(const double nel, const int orbitals_index) override - { - dm_->setUniform(nel, orbitals_index); + return dm_->getIndex(); } + void setDMuniform(const double nel) override { dm_->setUniform(nel); } int dim() const { return dim_; } void computeInvS() override; @@ -280,10 +277,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface } void setDMto2InvS() override; - void buildDM(const MatrixType& z, const int orbitals_index); - void buildDM(const MatrixType& z, const std::vector&, - const int orbitals_index); - void buildDM(const std::vector&, const int orbitals_index); + void buildDM(const MatrixType& z); + void buildDM(const MatrixType& z, const std::vector&); + void buildDM(const std::vector&); double getEigSum() override; double getExpectation(const MatrixType& A); @@ -323,12 +319,11 @@ class ProjectedMatrices : public ProjectedMatricesInterface int readDM(HDFrestart& h5f_file) override; int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; - void updateDM(const int iterative_index) override; - void updateDMwithEigenstates(const int iterative_index); - void updateDMwithSP2(const int iterative_index); - void updateDMwithEigenstatesAndRotate( - const int iterative_index, MatrixType& zz); - void updateDMwithChebApproximation(const int iterative_index) override; + void updateDM() override; + void updateDMwithEigenstates(); + void updateDMwithSP2(); + void updateDMwithEigenstatesAndRotate(MatrixType& zz); + void updateDMwithChebApproximation() override; void computeChemicalPotentialAndOccupations( const double width, const int max_numst) { @@ -354,16 +349,16 @@ class ProjectedMatrices : public ProjectedMatricesInterface void resetDM() override { - dm_->setMatrix(*mat_X_old_, 0); + dm_->setMatrix(*mat_X_old_); dm_->stripS(*mat_L_old_); } - void updateDMwithRelax(const double mix, const int itindex) override + void updateDMwithRelax(const double mix) override { // cout<<"ProjectedMatrices::updateDMwithRelax()..."<mix(mix, *mat_X_old_, itindex); + dm_->mix(mix, *mat_X_old_); } SquareLocalMatrices getReplicatedDM(); @@ -414,10 +409,7 @@ class ProjectedMatrices : public ProjectedMatricesInterface pmat.gemm('n', 'n', 1.0, mat, *theta_, 0.); } MatrixType& getMatHB() { return *matHB_; } - void setDM(const MatrixType& mat, const int orbitals_index) - { - dm_->setMatrix(mat, orbitals_index); - } + void setDM(const MatrixType& mat) { dm_->setMatrix(mat); } void setEigenvalues(const std::vector& eigenvalues) { memcpy(eigenvalues_.data(), eigenvalues.data(), diff --git a/src/ProjectedMatrices2N.cc b/src/ProjectedMatrices2N.cc index b0696f94..479ee4f0 100644 --- a/src/ProjectedMatrices2N.cc +++ b/src/ProjectedMatrices2N.cc @@ -39,7 +39,7 @@ void ProjectedMatrices2N::assignBlocksH( template void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( - const double occ_width, const int iterative_index, const bool flag_reduce_T) + const double occ_width, const bool flag_reduce_T) { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -64,7 +64,7 @@ void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( (*MPIdata::sout) << "MVP target with mu = " << ProjectedMatricesInterface::mu_ << " [Ry]" << std::endl; - ProjectedMatrices::buildDM(*work2N_, iterative_index); + ProjectedMatrices::buildDM(*work2N_); } template class ProjectedMatrices2N>; diff --git a/src/ProjectedMatrices2N.h b/src/ProjectedMatrices2N.h index 69ca2838..a87bf83f 100644 --- a/src/ProjectedMatrices2N.h +++ b/src/ProjectedMatrices2N.h @@ -26,8 +26,8 @@ class ProjectedMatrices2N : public ProjectedMatrices void assignBlocksH(MatrixType&, MatrixType&, MatrixType&, MatrixType&); - void iterativeUpdateDMwithEigenstates(const double occ_width, - const int iterative_index, const bool flag_reduce_T = true); + void iterativeUpdateDMwithEigenstates( + const double occ_width, const bool flag_reduce_T = true); void diagonalizeDM(std::vector& occ, MatrixType& vect) { // we are assuming Gram matrix=identity diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index c52595e1..53508a2e 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -199,10 +199,10 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction virtual double computeEntropy() = 0; virtual double getEigSum() = 0; - virtual void updateTheta() = 0; - virtual void computeInvB() = 0; - virtual void printGramMM(std::ofstream& tfile) = 0; - virtual void setDMuniform(const double nel, const int orbitals_index) = 0; + virtual void updateTheta() = 0; + virtual void computeInvB() = 0; + virtual void printGramMM(std::ofstream& tfile) = 0; + virtual void setDMuniform(const double nel) = 0; virtual double dotProductWithInvS( const SquareLocalMatrices& ss) @@ -258,10 +258,9 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction } virtual void saveDM() { exitWithErrorMessage("saveDM"); } virtual void resetDM() { exitWithErrorMessage("resetDM"); } - virtual void updateDMwithRelax(const double mix, const int itindex) + virtual void updateDMwithRelax(const double mix) { (void)mix; - (void)itindex; exitWithErrorMessage("updateDMwithRelax"); } @@ -297,18 +296,11 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } - virtual void updateDMwithChebApproximation(const int iterative_index) + virtual void updateDMwithChebApproximation() { - (void)iterative_index; - exitWithErrorMessage("updateDMwithChebApproximation"); } - virtual void updateDM(const int iterative_index) - { - (void)iterative_index; - - exitWithErrorMessage("updateDM"); - } + virtual void updateDM() { exitWithErrorMessage("updateDM"); } virtual void setDMto2InvS() { exitWithErrorMessage("setDMto2InvS"); } virtual void initializeMatB( diff --git a/src/ProjectedMatricesSparse.h b/src/ProjectedMatricesSparse.h index fd3f6918..fdbb4f88 100644 --- a/src/ProjectedMatricesSparse.h +++ b/src/ProjectedMatricesSparse.h @@ -170,15 +170,12 @@ class ProjectedMatricesSparse : public ProjectedMatricesInterface /* scale H */ // (*matHB_).scale(vel_); } - void setDMuniform(const double nel, const int orbitals_index) override - { - dm_->setUniform(nel, orbitals_index); - } + void setDMuniform(const double nel) override { dm_->setUniform(nel); } int getDMMatrixIndex() const override { assert(dm_ != nullptr); - return dm_->getOrbitalsIndex(); + return dm_->getIndex(); } int getGramMatrixIndex() const { @@ -253,7 +250,7 @@ class ProjectedMatricesSparse : public ProjectedMatricesInterface void setDMto2InvS() override { assert(invS_ != nullptr); - dm_->setto2InvS(invS_->getInvS(), invS_->getGramMatrixOrbitalsIndex()); + dm_->setto2InvS(invS_->getInvS()); } void computeInvS() override diff --git a/src/md.cc b/src/md.cc index e8d06133..ac5aa8c1 100644 --- a/src/md.cc +++ b/src/md.cc @@ -420,7 +420,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) std::shared_ptr projmatrices = getProjectedMatrices(); - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); projmatrices->printDM(os_); } #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f55ce699..38cdc4c1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -558,7 +558,7 @@ add_test(NAME ReplicatedSP2 ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testMD_D72 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/test.py - ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/mgmol_quench.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/mgmol_md.cfg @@ -612,7 +612,7 @@ add_test(NAME ChebyshevMVP if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py - ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/quench.cfg ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/md.cfg diff --git a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc index 1cf1cf88..a5a086fe 100644 --- a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc +++ b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc @@ -172,7 +172,7 @@ int main(int argc, char** argv) // // reset initial DM to test iterative solve for it - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); ct.dm_inner_steps = 50; eks = mgmol->evaluateDMandEnergyAndForces( &orbitals, positions, anumbers, forces); diff --git a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc index 83bfe8a6..9cb58f33 100644 --- a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc +++ b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc @@ -166,7 +166,7 @@ int main(int argc, char** argv) orbitals.setIterativeIndex(10); // set initial DM with uniform occupations - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); projmatrices->printDM(std::cout); // diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index b1fb0094..058b0a8e 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -162,7 +162,7 @@ int main(int argc, char** argv) orbitals.setIterativeIndex(1); // set initial DM with uniform occupations - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); projmatrices->printDM(std::cout); // swap H and O to make sure order of atoms in list does not matter diff --git a/tests/testDensityMatrix.cc b/tests/testDensityMatrix.cc index 8a3649c1..6a65c995 100644 --- a/tests/testDensityMatrix.cc +++ b/tests/testDensityMatrix.cc @@ -95,10 +95,10 @@ TEST_CASE( // setup density matrix DensityMatrix dm(n); - dm.setMatrix(matK, 0); + dm.setMatrix(matK); dm.stripS(ls); - dm.dressUpS(ls, 1); + dm.dressUpS(ls); const MatrixType& newM = dm.getMatrix(); if (myrank == 0) std::cout << "new M" << std::endl; From cce45b987b15a75a19853c2158eaf7242d3522e3 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 31 Jul 2025 08:02:20 -0700 Subject: [PATCH 47/50] Merge release (#354) * Fix EnergyAndForces tests (#277) * have them work in debug mode too * Move factor 4pi out og linear solvers (#278) * Move some code into PoissonSolverFactory (#279) * Clean up class Potentials (#280) * Clean up class Ions, add test for it (#281) * Add test MD_MVP (#290) * Clean up code related to DM restart data (#292) * Write dm (#291) * Update use of DM in restart * Remove unused function in Control (#294) * Change symlink to restart in tests (#295) * enable tests when old link present * Extract number empty orbitals from restart file (#296) * Clean up MD_IonicStepper restart data write (#297) * Clean up code based on compiler warnings (#299) * Add getForces for evaluateDMandEnergyAndForces (#300) * Add check for compatibility MVP/Mehrstellen (#301) * Add support for Br atom (#302) * Clean up some HDFrestart functions (#303) * Fix and test restart single hdf5 file (#305) * Save Hartree potential for write in restart file (#306) * Speed-up recently added tests (#309) * simply use a smaller domain/mesh * Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing * Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry * Add functionalities for extra info in restart file (#312) * MPI abort (#313) * Fix error code used by MPI_Abort * Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung * Strenghten testIons (#315) * More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes * Misc code fixes (#317) * exposed by porting on new environment * Use unique restart filenames in test HDF5single (#318) * Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces * Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() * Added functionalities to set local forces (#321) * Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code * Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private * Fixes for build without HDF5P (#324) * Fix testRhoVhRestart (#325) * Fix a few more issues with class Ions (#328) * Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ * Add cleanup and MGmolInterface (#331) * Add function to set local forces (#333) * based on matching coordinates to local ions * Fix setLocalForces (#335) * Fix test WFEnergyAndForces (#336) * writing restart twice in same directory was failing * Add mixing option in MVP (#337) * Update MVP with mixing (#339) * use tolerance to terminate iterations * tune verbosity * Update DensityMatrix class (#342) * add extrapolation function * remove unused function * simplify use of iterative index * Fixing DM tolerance (#341) * Fix Density Matrix tolerance read --------- Co-authored-by: Jean-Luc Fattebert * Rm oversubscribe from CMakeLists.txt (#346) * already set in build script * Adapt to new environment on condo @ ORNL (#348) * Remove unused function * Clean up use of coords file when restarting * use coordinates file to override atomic coordinates when using restart wavefunctions * Add test restart with override atomic coordinates * Add some missing include files * required for gcc12 * Print MVP timers --------- Co-authored-by: Jean-Luc Fattebert Co-authored-by: Seung Whan Chung --- src/Control.cc | 74 ++++++++++++++-------------- src/Control.h | 1 - src/DensityMatrix.cc | 6 --- src/MGmol.cc | 4 ++ src/MGmol.h | 1 - src/MVPSolver.cc | 8 +-- src/MVPSolver.h | 3 +- src/main.cc | 10 ++-- src/md.cc | 2 +- src/readInput.cc | 63 ++++-------------------- src/read_config.cc | 17 +++---- tests/CMakeLists.txt | 9 ++++ tests/LBFGS/test.py | 2 +- tests/MD_D72/test.py | 2 +- tests/MD_MVP/test.py | 2 +- tests/OverrideCoords/mgmol.cfg | 27 +++++++++++ tests/OverrideCoords/restart.cfg | 24 +++++++++ tests/OverrideCoords/shifted.xyz | 8 +++ tests/OverrideCoords/sih4.xyz | 8 +++ tests/OverrideCoords/test.py | 83 ++++++++++++++++++++++++++++++++ tests/RhoVhRestart/test.py | 4 +- tests/testSetGhostValues.cc | 2 + tests/testTradeGhostValues.cc | 2 + 23 files changed, 234 insertions(+), 128 deletions(-) create mode 100644 tests/OverrideCoords/mgmol.cfg create mode 100644 tests/OverrideCoords/restart.cfg create mode 100644 tests/OverrideCoords/shifted.xyz create mode 100644 tests/OverrideCoords/sih4.xyz create mode 100755 tests/OverrideCoords/test.py diff --git a/src/Control.cc b/src/Control.cc index 980c1abe..b7d23203 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -112,7 +112,6 @@ Control::Control() out_restart_file_type = -1; spread_radius = -1.; iprint_residual = -1; - override_restart = -1; dot_product_type = -1; spread_penalty_damping_ = -1; spread_penalty_alpha_ = -1; @@ -386,7 +385,7 @@ void Control::sync(void) short_buffer[47] = out_restart_file_naming_strategy; short_buffer[48] = enforceVmass0; short_buffer[49] = dm_inner_steps; - short_buffer[50] = override_restart; + short_buffer[50] = -1; short_buffer[51] = fgmres_kim; short_buffer[52] = fgmres_maxits; short_buffer[53] = ilu_type; @@ -599,41 +598,41 @@ void Control::sync(void) out_restart_file_naming_strategy = short_buffer[47]; enforceVmass0 = short_buffer[48]; dm_inner_steps = short_buffer[49]; - override_restart = short_buffer[50]; - fgmres_kim = short_buffer[51]; - fgmres_maxits = short_buffer[52]; - ilu_type = short_buffer[53]; - ilu_lof = short_buffer[54]; - ilu_maxfil = short_buffer[55]; - coloring_algo_ = short_buffer[56]; - diel_flag_ = short_buffer[57]; - poisson_pc_nu1 = short_buffer[58]; - poisson_pc_nu2 = short_buffer[59]; - poisson_pc_nlev = short_buffer[60]; - system_charge_ = short_buffer[61]; - md_print_freq = short_buffer[62]; - use_kernel_functions = short_buffer[63]; - ngpts_[0] = short_buffer[64]; - ngpts_[1] = short_buffer[65]; - ngpts_[2] = short_buffer[66]; - computeCondGram_ = short_buffer[67]; - lrs_extrapolation = short_buffer[68]; - parallel_transport = (bool)short_buffer[69]; - with_spin_ = (bool)short_buffer[70]; - conv_criterion_ = short_buffer[71]; - load_balancing_max_iterations = short_buffer[72]; - load_balancing_modulo = short_buffer[73]; - write_clusters = short_buffer[74]; - DM_solver_ = short_buffer[75]; - dm_algo_ = short_buffer[80]; - dm_approx_order = short_buffer[81]; - dm_approx_ndigits = short_buffer[82]; - dm_approx_power_maxits = short_buffer[83]; - spread_penalty_type_ = short_buffer[84]; - dm_use_old_ = short_buffer[85]; - max_electronic_steps_tight_ = short_buffer[86]; - hartree_reset_ = short_buffer[88]; - MD_last_step_ = short_buffer[89]; + //... = short_buffer[50]; + fgmres_kim = short_buffer[51]; + fgmres_maxits = short_buffer[52]; + ilu_type = short_buffer[53]; + ilu_lof = short_buffer[54]; + ilu_maxfil = short_buffer[55]; + coloring_algo_ = short_buffer[56]; + diel_flag_ = short_buffer[57]; + poisson_pc_nu1 = short_buffer[58]; + poisson_pc_nu2 = short_buffer[59]; + poisson_pc_nlev = short_buffer[60]; + system_charge_ = short_buffer[61]; + md_print_freq = short_buffer[62]; + use_kernel_functions = short_buffer[63]; + ngpts_[0] = short_buffer[64]; + ngpts_[1] = short_buffer[65]; + ngpts_[2] = short_buffer[66]; + computeCondGram_ = short_buffer[67]; + lrs_extrapolation = short_buffer[68]; + parallel_transport = (bool)short_buffer[69]; + with_spin_ = (bool)short_buffer[70]; + conv_criterion_ = short_buffer[71]; + load_balancing_max_iterations = short_buffer[72]; + load_balancing_modulo = short_buffer[73]; + write_clusters = short_buffer[74]; + DM_solver_ = short_buffer[75]; + dm_algo_ = short_buffer[80]; + dm_approx_order = short_buffer[81]; + dm_approx_ndigits = short_buffer[82]; + dm_approx_power_maxits = short_buffer[83]; + spread_penalty_type_ = short_buffer[84]; + dm_use_old_ = short_buffer[85]; + max_electronic_steps_tight_ = short_buffer[86]; + hartree_reset_ = short_buffer[88]; + MD_last_step_ = short_buffer[89]; poisson_lap_type_ = static_cast(short_buffer[90]); numst = int_buffer[0]; @@ -1787,7 +1786,6 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // options not available in configure file lr_updates_type = 0; precond_factor_computed = false; - override_restart = 0; mix_pot = 1.; project_out_psd = 0; multipole_order = 1; diff --git a/src/Control.h b/src/Control.h index 77932fb3..7e27d1b8 100644 --- a/src/Control.h +++ b/src/Control.h @@ -562,7 +562,6 @@ class Control short out_restart_file_naming_strategy; short restart_file_type; short out_restart_file_type; - short override_restart; short verbose; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 83d715d5..7be19aab 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -22,12 +22,6 @@ const double factor_kernel4dot = 10.; -#define MGMOL_DENSITYMATRIX_FAIL(X) \ - { \ - std::cerr << "DensityMatrix failure:" << std::endl; \ - std::cerr << "Error Message: " << X << std::endl; \ - } - #define MGMOL_DENSITYMATRIX_FAIL(X) \ { \ std::cerr << "DensityMatrix failure:" << std::endl; \ diff --git a/src/MGmol.cc b/src/MGmol.cc index ac022d91..5c590819 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -45,6 +45,7 @@ #include "MGmol.h" #include "MLWFTransform.h" #include "MPIdata.h" +#include "MVPSolver.h" #include "MasksSet.h" #include "Mesh.h" #include "OrbitalsPreconditioning.h" @@ -901,6 +902,9 @@ void MGmol::printTimers() AndersonMix::update_tm().print(os_); proj_matrices_->printTimers(os_); ShortSightedInverse::printTimers(os_); + if (std::is_same::value) + MVPSolver>::printTimers(os_); VariableSizeMatrixInterface::printTimers(os_); DataDistribution::printTimers(os_); PackedCommunicationBuffer::printTimers(os_); diff --git a/src/MGmol.h b/src/MGmol.h index c7777a62..a561e900 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -229,7 +229,6 @@ class MGmol : public MGmolInterface void printEigAndOcc(); - int readCoordinates(std::ifstream* tfile, const bool cell_relative); int readCoordinates(const std::string& filename, const bool cell_relative); double computeConstraintResidual(OrbitalsType& orbitals, const OrbitalsType& hphi, OrbitalsType& res, const bool print_residual, diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 4f9165a5..699a01f3 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -398,12 +398,8 @@ int MVPSolver::solve(OrbitalsType& orbitals) template void MVPSolver::printTimers(std::ostream& os) { - if (onpe0) - { - os << std::setprecision(2) << std::fixed << std::endl; - solve_tm_.print(os); - target_tm_.print(os); - } + solve_tm_.print(os); + target_tm_.print(os); } template class MVPSolver class ProjectedMatrices; @@ -67,7 +66,7 @@ class MVPSolver ~MVPSolver(); int solve(OrbitalsType& orbitals); - void printTimers(std::ostream& os); + static void printTimers(std::ostream& os); }; #endif diff --git a/src/main.cc b/src/main.cc index 3c6830a0..621b72de 100644 --- a/src/main.cc +++ b/src/main.cc @@ -58,7 +58,7 @@ int main(int argc, char** argv) /* * read runtime parameters */ - std::string input_filename(""); + std::string coords_filename(""); std::string lrs_filename; std::string constraints_filename(""); @@ -70,7 +70,7 @@ int main(int argc, char** argv) // read from PE0 only if (MPIdata::onpe0) { - read_config(argc, argv, vm, input_filename, lrs_filename, + read_config(argc, argv, vm, coords_filename, lrs_filename, constraints_filename, total_spin, with_spin); } @@ -89,7 +89,7 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; - mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(coords_filename); mmpi.bcastGlobal(lrs_filename); // Enter main scope @@ -97,10 +97,10 @@ int main(int argc, char** argv) MGmolInterface* mgmol; if (ct.isLocMode()) mgmol = new MGmol(global_comm, *MPIdata::sout, - input_filename, lrs_filename, constraints_filename); + coords_filename, lrs_filename, constraints_filename); else mgmol = new MGmol(global_comm, *MPIdata::sout, - input_filename, lrs_filename, constraints_filename); + coords_filename, lrs_filename, constraints_filename); mgmol->setup(); diff --git a/src/md.cc b/src/md.cc index ac5aa8c1..06a45e51 100644 --- a/src/md.cc +++ b/src/md.cc @@ -340,7 +340,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) constraints_->printConstraints(os_); - if (ct.restart_info > 0 && !ct.override_restart) + if (ct.restart_info > 0) { if (onpe0) os_ << "Use restart file to initialize MD..." << std::endl; stepper->init(*h5f_file_); diff --git a/src/readInput.cc b/src/readInput.cc index 79fbecf1..bd6a7dd1 100644 --- a/src/readInput.cc +++ b/src/readInput.cc @@ -164,7 +164,7 @@ int MGmol::readLRsFromInput(std::ifstream* tfile) template int MGmol::readCoordinates( - std::ifstream* tfile, const bool cell_relative) + const std::string& coords_filename, const bool cell_relative) { Control& ct = *(Control::instance()); if (ct.verbose > 0) printWithTimeStamp("Read atomic coordinates...", os_); @@ -177,71 +177,26 @@ int MGmol::readCoordinates( const std::vector& sp(ct.getSpecies()); ions_.reset(new Ions(lattice, sp)); - if (ct.restart_info > 0 - && ct.override_restart == 0) // read restart ionic positions + if (ct.restart_info > 0 && coords_filename.empty()) { - if (ct.restart_info > 0) - { - if (onpe0 && ct.verbose > 0) - { - os_ << "Initialize ionic positions from restart file " - << ct.restart_file << std::endl; - } - ions_->initFromRestartFile(*h5f_file_); - } - } - else - { - // Coordinates and species type for each ion. - int info = ions_->readAtoms(tfile, cell_relative); - - return info; - } - - const int num_ions = ions_->getNumIons(); - if (onpe0) os_ << num_ions << " ions in simulation" << std::endl; - - return 0; -} - -template -int MGmol::readCoordinates( - const std::string& filename, const bool cell_relative) -{ - Control& ct = *(Control::instance()); - if (ct.verbose > 0) printWithTimeStamp("Read atomic coordinates...", os_); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; - - // setup ions - const std::vector& sp(ct.getSpecies()); - ions_.reset(new Ions(lattice, sp)); - - if (ct.restart_info > 0 - && ct.override_restart == 0) // read restart ionic positions - { - if (ct.restart_info > 0) + // read restart atomic positions + if (onpe0 && ct.verbose > 0) { - if (onpe0 && ct.verbose > 0) - { - os_ << "Initialize ionic positions from restart file " - << ct.restart_file << std::endl; - } - ions_->initFromRestartFile(*h5f_file_); + os_ << "Initialize atomic positions from restart file " + << ct.restart_file << std::endl; } + ions_->initFromRestartFile(*h5f_file_); } else { // Coordinates and species type for each ion. - int info = ions_->readAtoms(filename, cell_relative); + int info = ions_->readAtoms(coords_filename, cell_relative); return info; } const int num_ions = ions_->getNumIons(); - if (onpe0) os_ << num_ions << " ions in simulation" << std::endl; + if (onpe0) os_ << num_ions << " atoms in simulation" << std::endl; return 0; } diff --git a/src/read_config.cc b/src/read_config.cc index 9dbd56d3..aefb59ae 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -20,7 +20,7 @@ namespace po = boost::program_options; int read_config(int argc, char** argv, po::variables_map& vm, - std::string& input_file, std::string& lrs_filename, + std::string& coords_filename, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin) { // use configure file if it can be found @@ -36,9 +36,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, "help,h", "produce help message")("check", "check input")( "config,c", po::value(&config_file)->default_value("mgmol.cfg"), - "name of a file of a configuration.")("atomicCoordinates,i", - po::value>(), - "coordinates filename")("LRsFilename,l", + "name of configuration file")("atomicCoordinates,i", + po::value(&coords_filename), + "atomic coordinates filename")("LRsFilename,l", po::value(&lrs_filename), "LRs filename"); // Declare a group of options (with default when appropriate) that @@ -150,9 +150,8 @@ int read_config(int argc, char** argv, po::variables_map& vm, "Tolerance on forces for Geometry optimization")( "GeomOpt.max_steps", po::value()->default_value(1), "max. number of Geometry optimization steps")("GeomOpt.dt", - po::value(), "Delta t for trial pseudo-time steps")( - "atomicCoordinates", po::value>(), - "coordinates filename")("Thermostat.type", + po::value(), + "Delta t for trial pseudo-time steps")("Thermostat.type", po::value()->default_value("Langevin"), "Thermostat type")("Thermostat.temperature", po::value()->default_value(-1.), "Thermostat temperature")( @@ -384,9 +383,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, } if (vm.count("atomicCoordinates")) { - input_file + coords_filename = vm["atomicCoordinates"].as>()[0]; - std::cout << "Input files is: " << input_file << "\n"; + std::cout << "Coordinates files is: " << coords_filename << "\n"; } else { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 38cdc4c1..72e603b6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -467,6 +467,15 @@ add_test(NAME testSiH4 ${CMAKE_CURRENT_SOURCE_DIR}/SiH4/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/SiH4/sih4.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testOverrideCoords + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/sih4.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/shifted.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testCl2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/Cl2_ONCVPSP_LDA/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/LBFGS/test.py b/tests/LBFGS/test.py index 324fd129..7150b3ce 100755 --- a/tests/LBFGS/test.py +++ b/tests/LBFGS/test.py @@ -56,7 +56,7 @@ os.remove('wave.out') os.symlink(restart_file, 'wave.out') -command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp2,coords,lrs) +command = "{} {} -c {} -l {}".format(mpicmd,exe,inp2,lrs) print(command) output = subprocess.check_output(command,shell=True) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index 3f98b2bd..22773b6b 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -54,7 +54,7 @@ os.remove('wave.out') os.symlink(restart_file, 'wave.out') -command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) +command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) #remove created files diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py index 47b5462d..c767031e 100755 --- a/tests/MD_MVP/test.py +++ b/tests/MD_MVP/test.py @@ -51,7 +51,7 @@ os.rename(restart_file, 'snapshotMVP') #run MGmol - command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) + command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) #remove used restart files diff --git a/tests/OverrideCoords/mgmol.cfg b/tests/OverrideCoords/mgmol.cfg new file mode 100644 index 00000000..fad77b2d --- /dev/null +++ b/tests/OverrideCoords/mgmol.cfg @@ -0,0 +1,27 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=45 +atol=1.e-9 +num_lin_iterations=2 +[Orbitals] +initial_type=Gaussian +initial_width=2. +[Restart] +output_level=3 diff --git a/tests/OverrideCoords/restart.cfg b/tests/OverrideCoords/restart.cfg new file mode 100644 index 00000000..0715e6b1 --- /dev/null +++ b/tests/OverrideCoords/restart.cfg @@ -0,0 +1,24 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=25 +atol=1.e-8 +[Restart] +input_level=3 +input_filename=wave.out diff --git a/tests/OverrideCoords/shifted.xyz b/tests/OverrideCoords/shifted.xyz new file mode 100644 index 00000000..089658f8 --- /dev/null +++ b/tests/OverrideCoords/shifted.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.935 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/OverrideCoords/sih4.xyz b/tests/OverrideCoords/sih4.xyz new file mode 100644 index 00000000..b3f921e3 --- /dev/null +++ b/tests/OverrideCoords/sih4.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.885 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/OverrideCoords/test.py b/tests/OverrideCoords/test.py new file mode 100755 index 00000000..2de3fd4f --- /dev/null +++ b/tests/OverrideCoords/test.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string +import shutil + +print("Test Override coordinates...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-6] +inp1 = sys.argv[nargs-5] +inp2 = sys.argv[nargs-4] +coords1 = sys.argv[nargs-3] +coords2 = sys.argv[nargs-2] +print("coordinates file: %s"%coords1) + +#create links to potentials files +dst1 = 'pseudo.Si' +dst2 = 'pseudo.H' +src1 = sys.argv[nargs-1] + '/' + dst1 +src2 = sys.argv[nargs-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run quench +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp1,coords1) +print("Run command: {}".format(command)) +output1 = subprocess.check_output(command,shell=True) +lines=output1.split(b'\n') + +#analyse output of quench +for line in lines: + if line.count(b'%%'): + print(line) + +#run quench with shifted coordinates +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" +print(command) +restart_file = subprocess.check_output(command,shell=True) +restart_file=str(restart_file[:-1],'utf-8') +print(restart_file) + +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') + +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords2) +print(command) +output2 = subprocess.check_output(command,shell=True) + +#remove created files +shutil.rmtree(restart_file) +os.remove('wave.out') + +#analyse mgmol standard output +lines=output2.split(b'\n') + +flag = 0 +for line in lines: + if line.count(b'%%'): + print(line) + if line.count(b'achieved'): + flag=1 + +if flag==0: + print("second run did not converge...") + sys.exit(1) + +sys.exit(0) diff --git a/tests/RhoVhRestart/test.py b/tests/RhoVhRestart/test.py index a34b962f..0d14383e 100755 --- a/tests/RhoVhRestart/test.py +++ b/tests/RhoVhRestart/test.py @@ -53,7 +53,7 @@ sys.exit(1) #run MD -command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +command = "{} {} -c {}".format(mpicmd,mgmol_exe,input2) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') @@ -68,7 +68,7 @@ sys.exit(1) #run test -command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input3,coords) +command = "{} {} -c {}".format(mpicmd,test_exe,input3) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') diff --git a/tests/testSetGhostValues.cc b/tests/testSetGhostValues.cc index c3680087..90aafde0 100644 --- a/tests/testSetGhostValues.cc +++ b/tests/testSetGhostValues.cc @@ -12,6 +12,8 @@ #include "catch.hpp" +#include + TEST_CASE("Set ghost values", "[set ghosts") { const double origin[3] = { 0., 0., 0. }; diff --git a/tests/testTradeGhostValues.cc b/tests/testTradeGhostValues.cc index 069b9374..8294aaec 100644 --- a/tests/testTradeGhostValues.cc +++ b/tests/testTradeGhostValues.cc @@ -13,6 +13,8 @@ #include "catch.hpp" +#include + // function of periodicity nx, ny, nz double cos3(const int i, const int j, const int k, const int nx, const int ny, const int nz) From 5908a8097969f19a1956199717852ad85d6ea1c5 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 16 Dec 2025 11:19:02 -0800 Subject: [PATCH 48/50] Resolve errors --- CMakeLists.txt | 2 +- src/ExtendedGridOrbitals.cc | 26 +++++++++++++++++++++++++ src/ExtendedGridOrbitals.h | 2 ++ src/rom.cc | 38 ++++++++++++++++++------------------- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dab46d4..23d058f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,7 +151,7 @@ if(USE_LIBROM) find_package(libROM REQUIRED) if(libROM_FOUND) - set(MGMOL_HAS_LIBROM 1) + add_definitions(-DMGMOL_HAS_LIBROM) endif(libROM_FOUND) endif(USE_LIBROM) diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 589e1f5b..48eafce6 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -30,6 +30,10 @@ #include #include +#ifdef MGMOL_HAS_LIBROM +#include "librom.h" +#endif + #define ORBITAL_OCCUPATION 2. std::string getDatasetName(const std::string& name, const int color); @@ -1810,6 +1814,28 @@ void ExtendedGridOrbitals::initWF( #endif } +#ifdef MGMOL_HAS_LIBROM +template +void ExtendedGridOrbitals::set(std::string file_path, int rdim) +{ + const int dim = getLocNumpt(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + pb::GridFunc gf_psi(mymesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector psi; + for (int i = 0; i < rdim; ++i) + { + orbital_basis->getColumn(i, psi); + gf_psi.assign(psi.getData()); + setPsi(gf_psi, i); + } +} +#endif + template void ExtendedGridOrbitals::axpy( const ORBDTYPE alpha, const ExtendedGridOrbitals&); diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index 77887bb4..e1fb667c 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -402,9 +402,11 @@ class ExtendedGridOrbitals : public Orbitals const pb::Grid& mygrid = mymesh->grid(); return mygrid.maxDomainSize(); } + #ifdef MGMOL_HAS_LIBROM void set(std::string file_path, int rdim); #endif + }; #endif diff --git a/src/rom.cc b/src/rom.cc index 5085f1d3..863d76e9 100644 --- a/src/rom.cc +++ b/src/rom.cc @@ -92,25 +92,25 @@ void MGmol::project_orbital(std::string file_path, int rdim, Orbit } } -template -void ExtendedGridOrbitals::set(std::string file_path, int rdim) -{ - const int dim = getLocNumpt(); - - CAROM::BasisReader reader(file_path); - CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); - - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - pb::GridFunc gf_psi(mymesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); - CAROM::Vector psi; - for (int i = 0; i < rdim; ++i) - { - orbital_basis->getColumn(i, psi); - gf_psi.assign(psi.getData()); - setPsi(gf_psi, i); - } -} +//template +//void ExtendedGridOrbitals::set(std::string file_path, int rdim) +//{ +// const int dim = getLocNumpt(); + +// CAROM::BasisReader reader(file_path); +// CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + +// Control& ct = *(Control::instance()); +// Mesh* mymesh = Mesh::instance(); +// pb::GridFunc gf_psi(mymesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); +// CAROM::Vector psi; +// for (int i = 0; i < rdim; ++i) +// { +// orbital_basis->getColumn(i, psi); +// gf_psi.assign(psi.getData()); +// setPsi(gf_psi, i); +// } +//} template class MGmol>; template class MGmol>; From 5f0ed94db450ff53b3c418430eef07bda66f87b8 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Wed, 17 Dec 2025 08:39:08 -0800 Subject: [PATCH 49/50] Minor change to scripts --- examples/PinnedH2O/job.ref | 2 +- examples/PinnedH2O/job.rom_3DOF | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref index 32024e6f..f9826e8e 100644 --- a/examples/PinnedH2O/job.ref +++ b/examples/PinnedH2O/job.ref @@ -8,7 +8,7 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 8 +set ncpus = 1 set case = 2 set maindir = /p/lustre2/cheung26/mgmol diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF index 0e38688b..b7dc5d6c 100644 --- a/examples/PinnedH2O/job.rom_3DOF +++ b/examples/PinnedH2O/job.rom_3DOF @@ -8,7 +8,7 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 8 +set ncpus = 1 set case = 2 set maindir = /p/lustre2/cheung26/mgmol From aa6217f15cd6c1563343b38164993e0a09f0dfde Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Wed, 17 Dec 2025 14:55:19 -0800 Subject: [PATCH 50/50] Minor fix in names --- examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg index 24273c40..bb2912d9 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -38,7 +38,7 @@ output_level=4 [ROM] stage=online_pinned_H2O_3dof [ROM.offline] -basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_1/PinnedH2O_3DOF_orbitals_basis_2_2 [ROM.basis] compare_md=false number_of_orbital_basis=34