From 1ad1292720f89c93e4d8a4046d68028fdee449fe Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 26 Sep 2024 08:27:05 -0400 Subject: [PATCH 01/53] Fix EnergyAndForces tests (#277) * have them work in debug mode too --- src/DensityMatrix.cc | 7 ++++++- src/DensityMatrix.h | 13 +++++++++++-- src/ProjectedMatrices.cc | 1 + tests/WFEnergyAndForces/mgmol.cfg | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 6e6ea652..e2c7b473 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -28,7 +28,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) { - assert(ndim >= 0); + assert(ndim > 0); dim_ = ndim; @@ -45,6 +45,7 @@ DensityMatrix::DensityMatrix(const int ndim) kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); occupation_.resize(dim_); + setDummyOcc(); } template @@ -109,6 +110,7 @@ void DensityMatrix::build( const std::vector& occ, const int new_orbitals_index) { assert(dm_ != nullptr); + assert(!occ.empty()); setOccupations(occ); @@ -149,6 +151,8 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { + assert(!occupation_.empty()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) @@ -314,6 +318,7 @@ void DensityMatrix::computeOccupations(const MatrixType& ls) template void DensityMatrix::setOccupations(const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (mmpi.instancePE0()) diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 960ad18e..84804a86 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -84,17 +84,24 @@ class DensityMatrix *dm_ = mat; orbitals_index_ = orbitals_index; - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; stripped_ = false; } + // set occupations to meaningless values to catch uninitialized use + void setDummyOcc() + { + for (auto& occ : occupation_) + occ = -1.; + } + void initMatrix(const double* const val) { dm_->init(val, dim_); - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; @@ -105,8 +112,10 @@ class DensityMatrix void getOccupations(std::vector& occ) const { + assert(!occupation_.empty()); assert(occ_uptodate_); assert((int)occ.size() == dim_); + memcpy(&occ[0], &occupation_[0], dim_ * sizeof(double)); } diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e0e679e9..3606aa40 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -509,6 +509,7 @@ template void ProjectedMatrices::setOccupations( const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS if (mmpi.instancePE0()) (*MPIdata::sout) << "ProjectedMatrices::setOccupations()" diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg index d543b626..12002703 100644 --- a/tests/WFEnergyAndForces/mgmol.cfg +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H type=QUENCH [Quench] max_steps=50 -atol=1.e-9 +atol=1.e-8 num_lin_iterations=2 [Orbitals] initial_type=Gaussian From 8ac6cd60cd2c06f87c8be6a5806478e316f33c53 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 30 Sep 2024 11:53:23 -0400 Subject: [PATCH 02/53] Move factor 4pi out og linear solvers (#278) --- src/Hartree.cc | 9 ++--- src/Hartree.h | 2 -- src/Hartree_CG.cc | 1 + src/PCGSolver.cc | 59 ++++++++++++++++----------------- src/PCGSolver.h | 52 ++++++++++++++--------------- src/PCGSolver_Diel.cc | 68 +++++++++++++++++++------------------- src/PCGSolver_Diel.h | 30 +++++++++-------- src/pb/Mgm.h | 3 -- src/pb/Solver.h | 7 ++-- src/pb/SolverLap.cc | 77 ++----------------------------------------- src/pb/SolverLap.h | 2 +- src/pb/SolverPB.cc | 5 ++- src/pb/SolverPB.h | 4 +-- 13 files changed, 120 insertions(+), 199 deletions(-) diff --git a/src/Hartree.cc b/src/Hartree.cc index b3984b87..7d0ba4d8 100644 --- a/src/Hartree.cc +++ b/src/Hartree.cc @@ -7,10 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #include #include -using namespace std; #include "Control.h" #include "Hartree.h" @@ -82,6 +80,8 @@ void Hartree::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + // Hartree units + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else @@ -92,10 +92,11 @@ void Hartree::solve( double final_residual = poisson_solver_->getFinalResidual(); if (onpe0) - (*MPIdata::sout) << setprecision(2) << scientific + (*MPIdata::sout) << std::setprecision(2) << std::scientific << "Hartree: residual reduction = " << residual_reduction - << ", final residual = " << final_residual << endl; + << ", final residual = " << final_residual + << std::endl; Poisson::Int_vhrho_ = vel * Poisson::vh_->gdot(rho); Poisson::Int_vhrhoc_ = vel * Poisson::vh_->gdot(rhoc); diff --git a/src/Hartree.h b/src/Hartree.h index a7050e24..b87ff01e 100644 --- a/src/Hartree.h +++ b/src/Hartree.h @@ -7,14 +7,12 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #ifndef included_Hartree #define included_Hartree #include "Poisson.h" #include "PoissonInterface.h" -// pb #include "SolverLap.h" template diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index f9bf3860..375237f8 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -81,6 +81,7 @@ void Hartree_CG::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else diff --git a/src/PCGSolver.cc b/src/PCGSolver.cc index 12103aaa..b3bd7ec3 100644 --- a/src/PCGSolver.cc +++ b/src/PCGSolver.cc @@ -11,14 +11,13 @@ #include #include -using namespace std; -template -void PCGSolver::clear() +template +void PCGSolver::clear() { - for (short i = 0; i < (short)pc_oper_.size(); i++) + for (short i = 0; i < (short)precond_oper_.size(); i++) { - delete pc_oper_[i]; + delete precond_oper_[i]; } for (short i = 0; i < (short)gf_work_.size(); i++) { @@ -35,26 +34,26 @@ void PCGSolver::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { delete grid_[i]; } - pc_oper_.clear(); + precond_oper_.clear(); grid_.clear(); gf_work_.clear(); gf_rcoarse_.clear(); gf_newv_.clear(); } -template -void PCGSolver::setupPrecon() +template +void PCGSolver::setupPrecon() { // check if precon is already setup // Assumes operator does not change, hence // a single setup is sufficient - if (is_pc_setup_) return; + if (is_precond_setup_) return; // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -63,7 +62,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*grid_[0], lap_type_); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); pb::GridFunc* gf_work = new pb::GridFunc( @@ -92,7 +91,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*coarse_grid, 1); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); gf_work = new pb::GridFunc( *coarse_grid, bc_[0], bc_[1], bc_[2]); @@ -109,12 +108,13 @@ void PCGSolver::setupPrecon() mygrid = coarse_grid; } - is_pc_setup_ = true; + is_precond_setup_ = true; } // MG V-cycle with no mask -template -void PCGSolver::preconSolve(pb::GridFunc& gf_v, +template +void PCGSolver::preconSolve( + pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve(pb::GridFunc& gf_v, ncycl = 4 > (nu1_ + nu2_) ? 4 : (nu1_ + nu2_); } - pb::Lap* myoper = pc_oper_[level]; + pb::Lap* myoper = precond_oper_[level]; // SMOOTHING for (short it = 0; it < ncycl; it++) @@ -161,25 +161,23 @@ void PCGSolver::preconSolve(pb::GridFunc& gf_v, } // Left Preconditioned CG -template -bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +template +bool PCGSolver::solve( + pb::GridFunc& gf_phi, const pb::GridFunc& gf_rhs) { bool converged = false; const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual: r := b - Ax /* compute Ax */ oper_.apply(gf_phi, lhs); /* set r = b */ - pb::GridFunc res(gf_rhs); + pb::GridFunc res(gf_rhs); oper_.transform(res); - // Hartree units - const double hu = 4. * M_PI; - res *= hu; /* compute r = r - Ax */ res -= lhs; @@ -199,11 +197,11 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) /* preconditioning step */ prec_z.setValues(0.); preconSolve(prec_z, prec_res, 0); - pb::GridFunc z(prec_z); + pb::GridFunc z(prec_z); // conjugate vectors - pb::GridFunc p(prec_z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(prec_z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -251,13 +249,14 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) } // Left Preconditioned CG -template -bool PCGSolver::solve(T2* phi, T2* rhs, const char dis) +template +bool PCGSolver::solve( + ScalarType* phi, ScalarType* rhs, const char dis) { - pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_phi.assign(phi, dis); - pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_work.assign(rhs, dis); bool converged = solve(gf_phi, gf_work); diff --git a/src/PCGSolver.h b/src/PCGSolver.h index 8ba76158..04e1028f 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_H_ -#define _PCG_SOLVER_H_ +#ifndef MGMOL_PCG_SOLVER_H +#define MGMOL_PCG_SOLVER_H #include "Control.h" #include "Lap.h" @@ -17,32 +17,41 @@ #include -template +template class PCGSolver { - private: std::vector grid_; short lap_type_; short bc_[3]; bool fully_periodic_; - // operators + + // operator to solve for T oper_; - std::vector*> pc_oper_; + + // preconditioner operator for each MG level + std::vector*> precond_oper_; std::vector*> gf_work_; std::vector*> gf_rcoarse_; std::vector*> gf_newv_; - // solver params + + // solver parameters int maxiters_; double tol_; double final_residual_; double residual_reduction_; - // precon params + + // preconditioner parameters short nu1_; short nu2_; short max_nlevels_; short nlevels_; - bool is_pc_setup_; + bool is_precond_setup_; + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + void setupPrecon(); + void clear(); public: PCGSolver(T& oper, const short px, const short py, const short pz) @@ -61,15 +70,10 @@ class PCGSolver bc_[1] = py; bc_[2] = pz; fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); - // fine grid info - // pb::Grid* mygrid=new pb::Grid(oper.grid()); - // grid_.push_back(mygrid); - // fine grid operator - Control& ct = *(Control::instance()); - lap_type_ = ct.lap_type; - is_pc_setup_ = false; - // pb::Lap* myoper = LapFactory::createLap(*grid_[0],lap_type_); - // pc_oper_.push_back(myoper); + + Control& ct = *(Control::instance()); + lap_type_ = ct.lap_type; + is_precond_setup_ = false; }; void setup(const short nu1, const short nu2, const short max_sweeps, @@ -83,16 +87,10 @@ class PCGSolver setupPrecon(); } - void clear(); - - void setupPrecon(); - - void preconSolve(pb::GridFunc& gf_v, - const pb::GridFunc& gf_f, const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve(pb::GridFunc& gf_phi, + const pb::GridFunc& gf_rhs); - bool solve(T2* phi, T2* rhs, const char dis); + bool solve(ScalarType* phi, ScalarType* rhs, const char dis); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PCGSolver_Diel.cc b/src/PCGSolver_Diel.cc index 0cdc5aed..4e7c88a9 100644 --- a/src/PCGSolver_Diel.cc +++ b/src/PCGSolver_Diel.cc @@ -9,10 +9,8 @@ #include "PCGSolver_Diel.h" -using namespace std; - -template -void PCGSolver_Diel::clear() +template +void PCGSolver_Diel::clear() { for (short i = 0; i < (short)pc_oper_.size(); i++) { @@ -33,7 +31,7 @@ void PCGSolver_Diel::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { @@ -46,8 +44,8 @@ void PCGSolver_Diel::clear() gf_newv_.clear(); } -template -void PCGSolver_Diel::setupPrecon() +template +void PCGSolver_Diel::setupPrecon() { // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -57,8 +55,8 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(oper_); pc_oper_.push_back(myoper); - pb::GridFunc* gf_work - = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_work + = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); // coarse levels @@ -86,24 +84,25 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(pc_oper_[ln - 1]->coarseOp(*mygrid)); pc_oper_.push_back(myoper); - gf_work = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + gf_work = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); - pb::GridFunc* gf_rcoarse - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_rcoarse = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_rcoarse_.push_back(gf_rcoarse); - pb::GridFunc* gf_newv - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_newv = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_newv_.push_back(gf_newv); mygrid = coarse_grid; } } -template +template // MG V-cycle with no mask -void PCGSolver_Diel::preconSolve( - pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) +void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve( // COARSE GRID CORRECTION // restrictions - pb::GridFunc* rcoarse = gf_rcoarse_[level]; + pb::GridFunc* rcoarse = gf_rcoarse_[level]; gf_work_[level]->restrict3D(*rcoarse); // storage functions for coarse grid - pb::GridFunc* newv = gf_newv_[level]; + pb::GridFunc* newv = gf_newv_[level]; // call mgrid solver on a coarser level newv->resetData(); @@ -149,15 +148,16 @@ void PCGSolver_Diel::preconSolve( if (bc_[0] != 1 || bc_[2] != 1 || bc_[2] != 1) gf_v.trade_boundaries(); } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve( - pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +bool PCGSolver_Diel::solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) { if (!oper_.initialized()) { - cout << "Error in PCGSolver_Diel::solve: operator not initialized" - << endl; + std::cout + << "Error in PCGSolver_Diel::solve: operator not initialized" + << std::endl; return 0.; } @@ -165,13 +165,13 @@ bool PCGSolver_Diel::solve( const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); - pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual oper_.apply(gf_phi, lhs); - pb::GridFunc rhs(gf_rhs); + pb::GridFunc rhs(gf_rhs); oper_.transform(rhs); // Hartree units rhs *= (4. * M_PI); @@ -180,13 +180,13 @@ bool PCGSolver_Diel::solve( double rnorm = init_rnorm; // preconditioned residual - pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); // preconditioning step z = 0.; preconSolve(z, res, 0); // conjugate vectors - pb::GridFunc p(z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -225,11 +225,11 @@ bool PCGSolver_Diel::solve( return converged; } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, - pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, - pb::GridFunc& gf_vks) +bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks) { // initialize the linear system operator and the preconditioner oper_.init(gf_rhod); diff --git a/src/PCGSolver_Diel.h b/src/PCGSolver_Diel.h index 9fd97914..f2001a4c 100644 --- a/src/PCGSolver_Diel.h +++ b/src/PCGSolver_Diel.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_DIEL_H_ -#define _PCG_SOLVER_DIEL_H_ +#ifndef MGMOL_PCG_SOLVER_DIEL_H_ +#define MGMOL_PCG_SOLVER_DIEL_H_ #include "Control.h" #include "PB.h" @@ -21,7 +21,7 @@ #include -template +template class PCGSolver_Diel { @@ -32,9 +32,9 @@ class PCGSolver_Diel // operators T oper_; std::vector pc_oper_; - std::vector*> gf_work_; - std::vector*> gf_rcoarse_; - std::vector*> gf_newv_; + std::vector*> gf_work_; + std::vector*> gf_rcoarse_; + std::vector*> gf_newv_; // solver params int maxiters_; double tol_; @@ -47,6 +47,11 @@ class PCGSolver_Diel short nlevels_; void setupPrecon(); + void clear(); + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + public: PCGSolver_Diel(T& oper, const short px, const short py, const short pz) : oper_(oper) @@ -78,15 +83,12 @@ class PCGSolver_Diel max_nlevels_ = max_nlevels; } - void clear(); - - void preconSolve(pb::GridFunc& gf_v, const pb::GridFunc& gf_f, - const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs, - pb::GridFunc& gf_rhod, pb::GridFunc& gf_vks); + bool solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/pb/Mgm.h b/src/pb/Mgm.h index d16f5610..4b5ad68d 100644 --- a/src/pb/Mgm.h +++ b/src/pb/Mgm.h @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Mgm.h,v 1.13 2010/01/28 22:56:47 jeanluc Exp $ #ifndef PB_MGM_H #define PB_MGM_H @@ -41,8 +40,6 @@ bool Mgm(T1& A, T2& vh, const GridFunc& rho, const short cogr, A.rhs(res, rhs); // Hartree units - rhs *= (4. * M_PI); - // work GridFunc GridFunc lhs(finegrid, bcx, bcy, bcz); diff --git a/src/pb/Solver.h b/src/pb/Solver.h index 062fbaa2..f82b3b6a 100644 --- a/src/pb/Solver.h +++ b/src/pb/Solver.h @@ -7,9 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Solver.h,v 1.14 2010/01/28 22:56:47 jeanluc Exp $ -#ifndef SOLVE_H -#define SOLVE_H +#ifndef PB_SOLVER_H +#define PB_SOLVER_H #include "GridFunc.h" @@ -33,7 +32,7 @@ class Solver fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); } - virtual bool solve(GridFunc&, GridFunc&) = 0; + virtual bool solve(GridFunc&, const GridFunc&) = 0; virtual void setup(const short nu1, const short nu2, const short max_sweeps, const double tol, const short max_nlevels, diff --git a/src/pb/SolverLap.cc b/src/pb/SolverLap.cc index 5a85fefe..062755a8 100644 --- a/src/pb/SolverLap.cc +++ b/src/pb/SolverLap.cc @@ -7,8 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverLap.cc,v 1.15 2010/01/28 22:56:31 jeanluc Exp $ - #include "SolverLap.h" #include "Laph2.h" #include "Laph4.h" @@ -26,9 +24,8 @@ Timer vcycle_repvcycle_tm("vcycle_repvcycle"); namespace pb { - // explicit instantiation declaration -#if 1 + // double template class SolverLap, double>; template class SolverLap, double>; @@ -45,77 +42,7 @@ template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; -#else -// double -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); - -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -// float -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -#endif -/* -template bool Mgm(Laph4MP&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4M&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph2&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph6&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(ShiftedLaph4M&, GridFunc&, const GridFunc&, -const short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -*/ -/* -template int Vcycle(ShiftedLaph4M&, GridFunc&, const GridFunc -&, const short, const short, const short, const bool); template int -Vcycle(Laph4MP&, GridFunc&, const GridFunc &, const short, -const short, const short, const bool); template int Vcycle(Laph4M&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); template int Vcycle(Laph2&, GridFunc&, const -GridFunc &, const short, const short, const short, const bool); -template int Vcycle(Laph4&, GridFunc&, const GridFunc &, const -short, const short, const short, const bool); template int Vcycle(Laph6&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); -*/ template bool SolverLap::solve(T2* phi, T2* rhs, const char dis) { @@ -139,7 +66,7 @@ bool SolverLap::solve(T2* phi, T2* rhs, const char dis) } template -bool SolverLap::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverLap::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { bool conv = Mgm(oper_, gf_phi, gf_rhs, max_nlevels_, max_sweeps_, tol_, nu1_, nu2_, gather_coarse_level_, final_residual_, diff --git a/src/pb/SolverLap.h b/src/pb/SolverLap.h index 939e15bc..567bed05 100644 --- a/src/pb/SolverLap.h +++ b/src/pb/SolverLap.h @@ -64,7 +64,7 @@ class SolverLap : public Solver bool solve(T2* phi, T2* rhs, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; short getNbSweeps() const override { return nb_sweeps_; } double getFinalResidual() const override { return final_residual_; } diff --git a/src/pb/SolverPB.cc b/src/pb/SolverPB.cc index 53252c26..756b49d8 100644 --- a/src/pb/SolverPB.cc +++ b/src/pb/SolverPB.cc @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverPB.cc,v 1.13 2010/01/28 22:56:32 jeanluc Exp $ #include "SolverPB.h" #include "Mgm.h" #include "PBh2.h" @@ -85,7 +84,7 @@ bool SolverPB::solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis) } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks) { oper_.init(gf_rhod); @@ -101,7 +100,7 @@ bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { if (!oper_.initialized()) { diff --git a/src/pb/SolverPB.h b/src/pb/SolverPB.h index a05e1d49..99eae0e3 100644 --- a/src/pb/SolverPB.h +++ b/src/pb/SolverPB.h @@ -64,9 +64,9 @@ class SolverPB : public Solver } bool solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs, + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; ~SolverPB() override{}; From a2ece1adc7c16334d03d785c63018259cd80208b Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 30 Sep 2024 14:08:59 -0400 Subject: [PATCH 03/53] Move some code into PoissonSolverFactory (#279) --- src/Electrostatic.cc | 200 ++------------------------------ src/Electrostatic.h | 1 + src/PoissonSolverFactory.h | 228 +++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 192 deletions(-) create mode 100644 src/PoissonSolverFactory.h diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 80cb754b..6ac4fef3 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -11,6 +11,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "GridFactory.h" +#include "GridFunc.h" #include "Hartree.h" #include "Hartree_CG.h" #include "Ions.h" @@ -23,15 +24,6 @@ #include "ShiftedHartree.h" #include "mputils.h" -#include "GridFunc.h" -#include "Laph2.h" -#include "Laph4.h" -#include "Laph4M.h" -#include "Laph4MP.h" -#include "Laph6.h" -#include "Laph8.h" -#include "ShiftedLaph4M.h" - Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], @@ -49,109 +41,9 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Mesh* mymesh = Mesh::instance(); const pb::Grid& myGrid = mymesh->grid(); - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } - else // use PCG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "PCG Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "PCG Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } + // create Poisson solver + poisson_solver_ = PoissonSolverFactory::create( + myGrid, lap_type, bcPoisson, screening_const); grhoc_ = nullptr; diel_flag_ = false; @@ -244,73 +136,8 @@ void Electrostatic::setupPB( ngpts, origin, cell, static_cast(laptype_), true, myPEenv); if (poisson_solver_ != nullptr) delete poisson_solver_; - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } - else // use PCG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } + poisson_solver_ = PoissonSolverFactory::createDiel( + *pbGrid_, laptype_, bc_, e0, rho0, drho0); if (grhoc_ != nullptr) { @@ -330,6 +157,7 @@ void Electrostatic::setupPB( poisson_solver_->set_vh(gf_vh); } +// This function is only useful for Hartree problem with dielectric continuum void Electrostatic::fillFuncAroundIons(const Ions& ions) { assert(grhod_ != nullptr); @@ -352,7 +180,6 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) std::vector::const_iterator ion = rc_ions.begin(); while (ion != rc_ions.end()) { - double rc = (*ion)->getRC(); // Special case: silicon if ((*ion)->isMass28()) rc = 2.0; @@ -373,43 +200,32 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) #endif for (unsigned int ix = 0; ix < pbGrid_->dim(0); ix++) { - xc[1] = pbGrid_->start(1); const int ix1 = (ix + shift) * incx; for (unsigned int iy = 0; iy < pbGrid_->dim(1); iy++) { - xc[2] = pbGrid_->start(2); const int iy1 = ix1 + (iy + shift) * incy; for (unsigned int iz = 0; iz < pbGrid_->dim(2); iz++) { - const double r = (*ion)->minimage(xc, lattice, bc_); if (r < rc) { const double alpha = 0.2 * (1. + cos(r * pi_rc)); - - const int iz1 = iy1 + iz + shift; + const int iz1 = iy1 + iz + shift; vv[iz1] += alpha; } - xc[2] += pbGrid_->hgrid(2); } - xc[1] += pbGrid_->hgrid(1); - } // end for iy - xc[0] += pbGrid_->hgrid(0); - } // end for ix } - ion++; - } // end loop on list of ions return; diff --git a/src/Electrostatic.h b/src/Electrostatic.h index a3b86879..072ce0cc 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -13,6 +13,7 @@ #include "Control.h" #include "GridFunc.h" #include "Poisson.h" +#include "PoissonSolverFactory.h" #include "Rho.h" #include "Timer.h" diff --git a/src/PoissonSolverFactory.h b/src/PoissonSolverFactory.h new file mode 100644 index 00000000..91b99bf2 --- /dev/null +++ b/src/PoissonSolverFactory.h @@ -0,0 +1,228 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE +#ifndef MGMOL_PoissonSolverFactory +#define MGMOL_PoissonSolverFactory + +#include "Control.h" +#include "Hartree.h" +#include "Hartree_CG.h" +#include "Mesh.h" +#include "PBdiel.h" +#include "PBdiel_CG.h" +#include "ShiftedHartree.h" +#include "mputils.h" + +#include "GridFunc.h" +#include "Laph2.h" +#include "Laph4.h" +#include "Laph4M.h" +#include "Laph4MP.h" +#include "Laph6.h" +#include "Laph8.h" +#include "ShiftedLaph4M.h" + +class PoissonSolverFactory +{ + +public: + /*! + * return specific Poisson solver needed to solve Hartree problem + */ + static Poisson* create(const pb::Grid& myGrid, PoissonFDtype lap_type, + const short bc[3], const double screening_const) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + else // use PCG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + + return poisson_solver; + } + + static Poisson* createDiel(pb::Grid& pbGrid, PoissonFDtype lap_type, + const short bc[3], const double e0, const double rho0, + const double drho0) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + else // use PCG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + return poisson_solver; + } +}; + +#endif From 8b577ea75f805f6288e0c9e91fba75b960a59a39 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 3 Oct 2024 10:03:02 -0400 Subject: [PATCH 04/53] Clean up class Potentials (#280) --- src/MGmol.cc | 2 +- src/Potentials.cc | 24 +++----------------- src/Potentials.h | 56 ++++++++++++++++++++++++++++++++--------------- src/tools.cc | 17 +++++++++++++- src/tools.h | 1 + 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index be4f28aa..4aa1fa26 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -837,7 +837,7 @@ void MGmol::initNuc(Ions& ions) pot.initialize(ions); // Check compensating charges - double comp_rho = pot.getCharge(pot.rho_comp()); + double comp_rho = getCharge(pot.rho_comp()); if (onpe0 && ct.verbose > 1) { diff --git a/src/Potentials.cc b/src/Potentials.cc index ee874f10..380f05a0 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -27,6 +27,7 @@ #include using namespace std; +// unit conversion factor Ha -> Ry const double ha2ry = 2.; Potentials::~Potentials() @@ -57,13 +58,9 @@ Potentials::Potentials(const bool vh_frozen) size_ = dim_[0] * dim_[1] * dim_[2]; - mix_ = 1.; - scf_dvrho_ = 1000.; scf_dv_ = 1000.; - vh_frozen_ = vh_frozen; - vtot_.resize(size_); vtot_old_.resize(size_); @@ -101,7 +98,7 @@ void Potentials::initWithVnuc() double one = 1.; LinearAlgebraUtils::MPaxpy( size_, one, &v_ext_[0], &vtot_[0]); - // factor 2 to get total potential in [Ry] for calculations + // factor ha2ry to get total potential in [Ry] for calculations LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); } @@ -568,9 +565,8 @@ template void Potentials::setVxc(const T* const vxc, const int iterativeIndex) { assert(iterativeIndex >= 0); - // int ione=1; + itindex_vxc_ = iterativeIndex; - // Tcopy(&size_, vxc, &ione, &vxc_rho_[0], &ione); MPcpy(&vxc_rho_[0], vxc, size_); } void Potentials::setVh(const POTDTYPE* const vh, const int iterativeIndex) @@ -849,20 +845,6 @@ void Potentials::rescaleRhoComp() if (comp_rho < 0.) mmpi.abort(); } -double Potentials::getCharge(RHODTYPE* rho) -{ - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - double charge = mygrid.integralOverMesh(rho); - - if (onpe0 && ct.verbose > 0) - cout << setprecision(8) << fixed << "Charge: " << charge << endl; - - return charge; -} - void Potentials::addBackgroundToRhoComp() { if (fabs(background_charge_) > 0.) diff --git a/src/Potentials.h b/src/Potentials.h index 0cd7a8a6..205fb538 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -30,7 +30,6 @@ class Potentials int gdim_[3]; int dim_[3]; bool diel_; - double mix_; double scf_dvrho_; double scf_dv_; @@ -39,8 +38,9 @@ class Potentials double charge_in_cell_; double ionic_charge_; - bool vh_frozen_; - + /*! + * Total KS potential seen by electrons + */ std::vector vtot_; std::vector vtot_old_; @@ -48,16 +48,30 @@ class Potentials std::vector vh_rho_; std::vector vxc_rho_; - // nuclei local potential + /* + * Potential contribution from atomic cores (local pseudopotential) + */ std::vector v_nuc_; - // external potential (read from input) + /*! + * Optional external potential (read from input) + * Used only in special cases. + */ std::vector v_ext_; #ifdef HAVE_TRICUBIC pb::TriCubic* vext_tricubic_; #endif + /*! + * Potential associated with the sum of Gaussian charge distributions + * compensating the Coulomb potential of each atom + */ std::vector v_comp_; + + /*! + * Sum of Gaussian charge distributions compensating the Coulomb potential + * of each atom + */ std::vector rho_comp_; std::vector dv_; @@ -121,14 +135,11 @@ class Potentials void turnOnDiel() { diel_ = true; } int size() const { return size_; } - bool vh_frozen() const { return vh_frozen_; } - void freeze_vh() { vh_frozen_ = true; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return &vtot_[0]; } POTDTYPE* vh_rho() { return &vh_rho_[0]; } - POTDTYPE* vxc_rho() { return &vxc_rho_[0]; } RHODTYPE* rho_comp() { return &rho_comp_[0]; } const std::vector& vnuc() const { return v_nuc_; } @@ -136,12 +147,6 @@ class Potentials POTDTYPE* vext() { return &v_ext_[0]; } POTDTYPE* vepsilon() { return &vepsilon_[0]; } - void set_vcomp(const POTDTYPE val) - { - const int n = (int)v_comp_.size(); - for (int i = 0; i < n; i++) - v_comp_[i] = val; - } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -154,13 +159,29 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + /*! + * initialize total potential as local pseudopotential + */ void initWithVnuc(); void getVofRho(std::vector& vrho) const; - double delta_v(const std::vector>&); - double update(const std::vector>&); - void update(const double); + /*! + * evaluate potential correction associated with a new rho + */ + double delta_v(const std::vector>& rho); + + /*! + * update potentials based on argument rho + */ + double update(const std::vector>& rho); + + /*! + * update potentials based on potential correction delta v and mixing + * parameter + */ + void update(const double mix); + double max() const; double min() const; void readAll(std::vector& sp); @@ -172,7 +193,6 @@ class Potentials void initialize(Ions& ions); void rescaleRhoComp(); - double getCharge(RHODTYPE* rho); void initBackground(Ions& ions); void addBackgroundToRhoComp(); diff --git a/src/tools.cc b/src/tools.cc index 4e1bd314..95790c59 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -228,7 +228,7 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) int mpierr=MPI_Reduce(&s, &r, 1, MPI_INT, MPI_SUM, 0, mmpi.commGlobal()); if( mpierr!=MPI_SUCCESS ) { - cerr << " Error in MPI!!!" << endl; + cerr << " Error in MPI!!!" << std::endl; MPI_Abort(mmpi.commGlobal(),1); } if( r!=mmpi.size()*s && onpe0 ) @@ -388,3 +388,18 @@ void getkvector(const int index, const int kmax, int kvector[3]) // std::cout << " k=(" << kvector[0] << "," << kvector[1] << "," // << kvector[2] << ")" << std::endl; } + +double getCharge(double* rho) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double charge = mygrid.integralOverMesh(rho); + + if (onpe0 && ct.verbose > 0) + std::cout << std::setprecision(8) << std::fixed << "Charge: " << charge + << std::endl; + + return charge; +} diff --git a/src/tools.h b/src/tools.h index 4e8374da..be3d13e0 100644 --- a/src/tools.h +++ b/src/tools.h @@ -37,5 +37,6 @@ double minQuadPolynomial(const double e0, const double e1, const double de0, double minQuadPolynomialFrom3values(const double e0, const double e1, const double e12, const bool print_flag, std::ostream& os); void getkvector(const int index, const int kmax, int kvector[3]); +double getCharge(double* rho); #endif From 7ac691c7126e45223c5ea33d043f256de1a52519 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 11 Oct 2024 21:44:42 -0400 Subject: [PATCH 05/53] Clean up class Ions, add test for it (#281) --- src/Control.cc | 1 + src/Ions.cc | 324 ++++++++++++++++----------------------- src/Ions.h | 23 ++- src/KBprojectorSparse.cc | 4 +- tests/CMakeLists.txt | 8 + tests/testIons.cc | 104 +++++++++++++ 6 files changed, 265 insertions(+), 199 deletions(-) create mode 100644 tests/testIons.cc diff --git a/src/Control.cc b/src/Control.cc index af3d8ed9..dd786678 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -70,6 +70,7 @@ Control::Control() dm_approx_ndigits = 1; dm_approx_power_maxits = 100; wf_extrapolation_ = 1; + verbose = 0; // undefined values dm_algo_ = -1; diff --git a/src/Ions.cc b/src/Ions.cc index 940e12e8..852151c7 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -584,11 +584,9 @@ void Ions::printPositionsLocal(std::ostream& os, const int root) const os.setf(std::ios::right, std::ios::adjustfield); os.setf(std::ios::fixed, std::ios::floatfield); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->printPosition(os); - ion++; + ion->printPosition(os); } os << std::endl; @@ -627,14 +625,12 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - assert((*ion)->atomic_number() > 0); - assert((*ion)->atomic_number() < 200); + assert(ion->atomic_number() > 0); + assert(ion->atomic_number() < 200); - data.push_back((*ion)->atomic_number()); - ion++; + data.push_back(ion->atomic_number()); } } @@ -685,17 +681,15 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) void Ions::lockAtom(const std::string& name) { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - std::string name_ion((*ion)->name()); + std::string name_ion(ion->name()); if (name.compare(name_ion) == 0) { - (*ion)->lock(); + ion->lock(); if (onpe0) (*MPIdata::sout) << "Lock atom " << name << std::endl; break; } - ion++; } } @@ -975,12 +969,10 @@ void Ions::readRestartPositions(HDFrestart& h5_file) std::vector data; h5_file.readAtomicPositions(data); - int i = 0; - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1109,14 +1101,12 @@ void Ions::removeMassCenterMotion() #endif } - ion = local_ions_.begin(); - i = 0; - while (ion != local_ions_.end()) + i = 0; + for (auto& ion : local_ions_) { const int threei = 3 * i; - (*ion)->setVelocity(velocities[threei] - tmp[0], + ion->setVelocity(velocities[threei] - tmp[0], velocities[threei + 1] - tmp[1], velocities[threei + 2] - tmp[2]); - ion++; i++; } @@ -1124,16 +1114,15 @@ void Ions::removeMassCenterMotion() mv[0] = 0.; mv[1] = 0.; mv[2] = 0.; - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - velocities[3 * i] = (*ion)->velocity(0); - velocities[3 * i + 1] = (*ion)->velocity(1); - velocities[3 * i + 2] = (*ion)->velocity(2); + velocities[3 * i] = ion->velocity(0); + velocities[3 * i + 1] = ion->velocity(1); + velocities[3 * i + 2] = ion->velocity(2); mv[0] += mass[i] * velocities[3 * i]; mv[1] += mass[i] * velocities[3 * i + 1]; mv[2] += mass[i] * velocities[3 * i + 2]; - ion++; i++; } @@ -1155,12 +1144,10 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) std::vector data; h5_file.readAtomicVelocities(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1174,12 +1161,10 @@ void Ions::readRestartRandomStates(HDFrestart& h5f_file) std::vector data; h5f_file.readRestartRandomStates(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1202,17 +1187,14 @@ void Ions::writeForces(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get position of local ion double force[3]; - (*ion)->getForce(&force[0]); + ion->getForce(&force[0]); data.push_back(force[0]); data.push_back(force[1]); data.push_back(force[2]); - - ++ion; } } @@ -1364,64 +1346,54 @@ void Ions::printForcesLocal(std::ostream& os, const int root) const << "FX" << std::setw(10) << "FY" << std::setw(10) << "FZ" << std::endl; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { + ion->printPositionAndForce(os); - (*ion)->printPositionAndForce(os); - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } // global statistics @@ -1479,12 +1451,10 @@ int Ions::countIonsHere() const { return (int)local_ions_.size(); } int Ions::countProjectorsHere() const { - int count = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int count = 0; + for (auto& ion : local_ions_) { - count += (*ion)->nProjectors(); - ion++; + count += ion->nProjectors(); } return count; } @@ -1497,12 +1467,10 @@ int Ions::countProjectors() const Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - int nproj = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int nproj = 0; + for (auto& ion : local_ions_) { - nproj += (*ion)->nProjectors(); - ion++; + nproj += ion->nProjectors(); } int tmp = nproj; MPI_Allreduce(&tmp, &nproj, 1, MPI_INT, MPI_SUM, myPEenv.comm()); @@ -1569,13 +1537,10 @@ void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); - - ion++; + ion->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); ia++; } @@ -1780,45 +1745,8 @@ int Ions::setAtoms( if (ia < 1000) aname.append("0"); if (ia < 10000) aname.append("0"); aname.append(ss.str()); - Ion* new_ion - = new Ion(species_[isp], aname, &crds[3 * ia], velocity, locked); - new_ion->bcast(mmpi.commGlobal()); - - // Populate list_ions_ list - // std::cout<<"crds: "< 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " << crds[3 * ia + 0] - << "," << crds[3 * ia + 1] << "," << crds[3 * ia + 2] - << " added to the list... on PE" << mmpi.mypeGlobal() - << std::endl; - // populate local_ions_ list - if (inLocalIons( - crds[3 * ia + 0], crds[3 * ia + 1], crds[3 * ia + 2])) - { - (new_ion)->set_here(true); - local_ions_.push_back(new_ion); - if (onpe0 && ct.verbose > 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " - << crds[3 * ia + 0] << "," << crds[3 * ia + 1] << "," - << crds[3 * ia + 2] - << " added to the list of local ions... on PE" - << mmpi.mypeGlobal() << std::endl; - } - else - (new_ion)->set_here(false); - } - else - { - //(*MPIdata::sout)<<"Ion "<set_here(true); + local_ions_.push_back(new_ion); + if (onpe0 && ct.verbose > 2) + (*MPIdata::sout) << "Ion " << name << " at position " << crds[0] + << "," << crds[1] << "," << crds[2] + << " added to the list of local ions... on PE" + << mmpi.mypeGlobal() << std::endl; + } + else + (new_ion)->set_here(false); + } + else + { + // delete Ion if not put in list + delete new_ion; + } +} + int Ions::readNatoms(const std::string& filename, const bool cell_relative) { Control& ct(*(Control::instance())); @@ -2117,9 +2085,8 @@ void Ions::setVelocities(const std::vector& tau0, assert(tau0.size() == 3 * local_ions_.size()); assert(taup.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { double v[3]; for (short i = 0; i < 3; i++) @@ -2128,8 +2095,7 @@ void Ions::setVelocities(const std::vector& tau0, v[i] -= tau0[3 * ia + i]; v[i] /= dt; } - (*iion)->setVelocity(v[0], v[1], v[2]); - iion++; + ion->setVelocity(v[0], v[1], v[2]); ia++; } } @@ -2138,12 +2104,10 @@ void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau[3 * ia]); - iion++; + ion->getPosition(&tau[3 * ia]); ia++; } } @@ -2188,12 +2152,10 @@ void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau0_[3 * ia]); - iion++; + ion->getPosition(&tau0_[3 * ia]); ia++; } } @@ -2202,13 +2164,11 @@ void Ions::setPositionsToTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - iion++; ia++; } } @@ -2244,13 +2204,11 @@ void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { assert(3 * ia + 2 < (int)tau.size()); - (*iion)->getForce(&tau[3 * ia]); - iion++; + ion->getForce(&tau[3 * ia]); ia++; } } @@ -2351,12 +2309,10 @@ double Ions::computeMaxVlRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& ion : local_ions_) { - double r = (*iion)->computeRadiusVl(); + double r = ion->computeRadiusVl(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -2393,13 +2349,11 @@ void Ions::gatherNames(std::map& names, const int root, std::vector data; std::vector local_names; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get local name and index - local_names.push_back((*ion)->name()); - local_indexes.push_back((*ion)->index()); - ++ion; + local_names.push_back(ion->name()); + local_indexes.push_back(ion->index()); } // gather data to PE root @@ -2695,15 +2649,13 @@ bool Ions::hasLockedAtoms() const return (flag == 1); } -double Ions::getMaxNLradius() const +double Ions::getSpeciesMaxNLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double nlradius = spi->nlradius(); + const double nlradius = spi.nlradius(); radius = radius > nlradius ? radius : nlradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2711,15 +2663,13 @@ double Ions::getMaxNLradius() const return radius; } -double Ions::getMaxLradius() const +double Ions::getSpeciesMaxLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double lradius = spi->lradius(); + const double lradius = spi.lradius(); radius = radius > lradius ? radius : lradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2918,8 +2868,8 @@ void Ions::computeNumIons(void) double Ions::getMaxListRadius() const { // get radius of projectors - const double nlradius = getMaxNLradius(); - const double lradius = getMaxLradius(); + const double nlradius = getSpeciesMaxNLradius(); + const double lradius = getSpeciesMaxLradius(); double rmax = nlradius > lradius ? nlradius : lradius; @@ -3289,21 +3239,15 @@ void Ions::updateIons() Control& ct(*(Control::instance())); // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - // (*ion)->setForce(fion_[3*ia+0], - // fion_[3*ia+1], - // fion_[3*ia+2]); - - (*ion)->setRandomState(rand_states_[3 * ia + 0], - rand_states_[3 * ia + 1], rand_states_[3 * ia + 2]); + ion->setRandomState(rand_states_[3 * ia + 0], rand_states_[3 * ia + 1], + rand_states_[3 * ia + 2]); - ion++; ia++; } diff --git a/src/Ions.h b/src/Ions.h index 772c51d1..ae4f6adb 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -71,13 +71,14 @@ class Ions void setMapVL(); double computeMaxVlRadius() const; double computeMaxNLprojRadius() const; - double getMaxNLradius() const; - double getMaxLradius() const; - void updateListIons(); - // compute boundary for box containing all atoms to be known on local - // processor that is values for list_boundary_left_, list_boundary_right_ - void setupListIonsBoundaries(const double rmax); + /* + * Evaluate maximum pseudopotential radius among all species in class + */ + double getSpeciesMaxNLradius() const; + double getSpeciesMaxLradius() const; + + void updateListIons(); void augmentIonsData(const int nsteps, const int dir, const int disp, const int locSize, const int maxLocSize, std::vector& data, @@ -163,6 +164,8 @@ class Ions bool hasLockedAtoms() const; void clearLists(); + void rescaleVelocities(const double factor); + public: Ions(const double lat[3], const std::vector& sp); @@ -172,6 +175,10 @@ class Ions void setup(); + // compute boundary for box containing all atoms to be known on local + // processor that is values for list_boundary_left_, list_boundary_right_ + void setupListIonsBoundaries(const double rmax); + std::vector& getLocalNames() { return local_names_; } std::vector& getTau0() { return tau0_; } std::vector& getTaup() { return taup_; } @@ -334,13 +341,15 @@ class Ions void updateForcesInteractingIons(); void updateTaupInteractingIons(); - void rescaleVelocities(const double factor); /*! * Calculate minimum distance between local pairs */ double computeMinLocalSpacing() const; + void addIonToList(const Species& sp, const std::string& name, + const double crds[3], const double velocity[3], const bool lock); + // void checkUnicityLocalIons(); }; diff --git a/src/KBprojectorSparse.cc b/src/KBprojectorSparse.cc index afd8871c..937b95d2 100644 --- a/src/KBprojectorSparse.cc +++ b/src/KBprojectorSparse.cc @@ -103,7 +103,7 @@ void KBprojectorSparse::setNLindex( for (int i = 0; i < size_nl; i++) { - assert(i < lnumpt); + // assert(i < lnumpt); if ((pvec[i] < iloc * lnumpt) || (pvec[i] >= (iloc + 1) * lnumpt)) { (*MPIdata::sout) << " iloc=" << iloc << ", i=" << i @@ -960,7 +960,7 @@ bool KBprojectorSparse::setIndexesAndProjectors() // get "pvec" and "is_in_domain" int icount = get_index_array(pvec, iloc, index_low, index_high); assert(icount <= nl3); - assert(icount <= mymesh->npointsPatch()); + // assert(icount <= mymesh->npointsPatch()); assert(icount > 0); setNLindex(iloc, icount, pvec); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc205167..6de34971 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -209,6 +209,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +add_executable(testIons + ${CMAKE_SOURCE_DIR}/tests/testIons.cc) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -334,6 +336,10 @@ add_test(NAME testBatchLaph4 add_test(NAME testtMGkernels COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMGkernels) +add_test(NAME testIons + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testIons + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) @@ -541,6 +547,7 @@ target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -550,6 +557,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testIons PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/testIons.cc b/tests/testIons.cc new file mode 100644 index 00000000..30b2ad0c --- /dev/null +++ b/tests/testIons.cc @@ -0,0 +1,104 @@ +#include "Control.h" +#include "Ions.h" +#include "MGmol_MPI.h" +#include "Mesh.h" +#include "Species.h" + +#include + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + + MPI_Comm comm = MPI_COMM_WORLD; + + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + MGmol_MPI::setup(comm, std::cout); + Control::setup(comm, false, 0.); + + // create a domain [0.10.]^3 + const double origin[3] = { 0., 0., 0. }; + const double ll = 10.; + const double lattice[3] = { ll, ll, ll }; + const unsigned ngpts[3] = { 32, 24, 20 }; + short lap_type = 0; + + Mesh::setup(comm, ngpts, origin, lattice, lap_type); + + const double h[3] = { ll / (double(ngpts[0])), ll / (double(ngpts[1])), + ll / (double(ngpts[2])) }; + + // random number generator + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_real_distribution<> dis(0.0, 1.0); + + // create one species + Species sp(MPI_COMM_WORLD); + + // read species info from pseudopotential file + std::string file_path = argv[1]; + std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); + std::cout << "Potential = " << filename << std::endl; + + sp.read_1species(filename); + sp.set_dim_nl(h[0]); + sp.set_dim_l(h[0]); + sp.initPotentials('f', h[0], true); + + // put species into a vector + std::vector vsp; + vsp.push_back(sp); + + Ions ions(lattice, vsp); + ions.setupListIonsBoundaries(10000.); + + double velocity[3] = { 0., 0., 0. }; + + // set "na" atoms coordinates and add them to "ions" + const int na = 10; + for (int i = 0; i < na; i++) + { + double x[3] = { origin[0] + lattice[0] * dis(gen), + origin[1] + lattice[1] * dis(gen), + origin[2] + lattice[2] * dis(gen) }; + if (myrank == 0) + std::cout << "x,y,z = " << x[0] << ", " << x[1] << ", " << x[2] + << std::endl; + + // set all x to the values of PE0 + MPI_Bcast(&x[0], 3, MPI_DOUBLE, 0, comm); + + // make a name for atom based on species and order of reading in + std::string stri = std::to_string(i); + std::string aname("C" + stri); + + ions.addIonToList(sp, aname, &x[0], velocity, false); + } + + ions.setup(); + + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + return 1; + } + + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + + return 0; +} From c05ba7f4cccb1077faa226e9bb79fb7294c93130 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 25 Nov 2024 13:35:01 -0500 Subject: [PATCH 06/53] Add test MD_MVP (#290) --- tests/CMakeLists.txt | 8 ++++ tests/MD_MVP/li2.xyz | 4 ++ tests/MD_MVP/md.cfg | 39 ++++++++++++++++++ tests/MD_MVP/quench.cfg | 34 ++++++++++++++++ tests/MD_MVP/test.py | 89 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+) create mode 100644 tests/MD_MVP/li2.xyz create mode 100644 tests/MD_MVP/md.cfg create mode 100644 tests/MD_MVP/quench.cfg create mode 100755 tests/MD_MVP/test.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6de34971..72378704 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -491,6 +491,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMD_MVP + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/quench.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/li2.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testLBFGS COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/LBFGS/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/MD_MVP/li2.xyz b/tests/MD_MVP/li2.xyz new file mode 100644 index 00000000..fce3cd72 --- /dev/null +++ b/tests/MD_MVP/li2.xyz @@ -0,0 +1,4 @@ +2 + +Li 1.33 0.0 0.0 +Li -1.33 0.0 0.0 diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg new file mode 100644 index 00000000..83d5aa4c --- /dev/null +++ b/tests/MD_MVP/md.cfg @@ -0,0 +1,39 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=MD +[MD] +num_steps=5 +dt=15. +[Quench] +solver=PSD +max_steps=30 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +input_filename=wave.out +input_level=3 +output_level=3 diff --git a/tests/MD_MVP/quench.cfg b/tests/MD_MVP/quench.cfg new file mode 100644 index 00000000..135dbde7 --- /dev/null +++ b/tests/MD_MVP/quench.cfg @@ -0,0 +1,34 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py new file mode 100755 index 00000000..489c9c8c --- /dev/null +++ b/tests/MD_MVP/test.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string +import shutil + +print("Test MD with MVP solver...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-5): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp1 = sys.argv[nargs-4] +inp2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst = 'pseudo.Li_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run quench +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp1,coords) +print("Run command: {}".format(command)) +output1 = subprocess.check_output(command,shell=True) +lines=output1.split(b'\n') + +#analyse output of quench +for line in lines: + num_matches = line.count(b'%%') + if num_matches: + print(line) + +#run MD +for i in range(2): + command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + print(command) + restart_file = subprocess.check_output(command,shell=True) + restart_file=str(restart_file[:-1],'utf-8') + print(restart_file) + + os.rename(restart_file, 'wave.out') + + #run MGmol + command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) + output2 = subprocess.check_output(command,shell=True) + + #remove used restart files + shutil.rmtree('wave.out') + + #analyse mgmol standard output + lines=output2.split(b'\n') + + print("Check energy conservation...") + tol = 1.e-4 + energy = 0. + count = 0 + for line in lines: + if line.count(b'%%'): + print(line) + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +#remove last restart files +command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +restart_file = subprocess.check_output(command,shell=True) +restart_file=str(restart_file[:-1],'utf-8') +shutil.rmtree(restart_file) + +sys.exit(0) From bb8051d2cce7299c4baf61f863a51488476aea1a Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 2 Dec 2024 08:17:13 -0500 Subject: [PATCH 07/53] Clean up code related to DM restart data (#292) --- src/DensityMatrix.cc | 58 +++++- src/DensityMatrix.h | 10 +- src/ExtendedGridOrbitals.cc | 28 +-- src/ExtendedGridOrbitals.h | 3 +- src/HDFrestart.cc | 8 +- src/HDFrestart.h | 32 ++-- src/LBFGS_IonicStepper.cc | 13 +- src/LocGridOrbitals.cc | 27 +-- src/LocGridOrbitals.h | 3 +- src/MGmol.h | 2 +- src/ProjectedMatrices.cc | 106 +---------- src/ProjectedMatrices.h | 11 +- src/ProjectedMatricesInterface.h | 10 +- src/hdf_tools.cc | 292 ++++++++++++++++++++----------- src/hdf_tools.h | 6 +- src/md.cc | 31 ++-- src/restart.cc | 7 +- tests/CMakeLists.txt | 5 +- 18 files changed, 338 insertions(+), 314 deletions(-) diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index e2c7b473..78d7008e 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -12,6 +12,8 @@ #include "DistMatrix.h" #include "MGmol_MPI.h" #include "ReplicatedMatrix.h" +#include "ReplicatedWorkSpace.h" +#include "hdf_tools.h" #include #include @@ -22,25 +24,28 @@ const double factor_kernel4dot = 10.; #define PROCRUSTES 0 +#define MGMOL_DENSITYMATRIX_FAIL(X) \ + { \ + std::cerr << "DensityMatrix failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } + // occupations in [0,1] // DM eigenvalues in [0,orbital_occupation] template DensityMatrix::DensityMatrix(const int ndim) + : dim_(ndim), + orbitals_index_(-1), + occ_uptodate_(false), + uniform_occ_(false), + stripped_(false) { assert(ndim > 0); - dim_ = ndim; - - occ_uptodate_ = false; - stripped_ = false; - uniform_occ_ = false; - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; - orbitals_index_ = -1; - dm_ = new MatrixType("DM", ndim, ndim); kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); @@ -438,6 +443,43 @@ void DensityMatrix::mix( orbitals_index_ = new_orbitals_index; } +template +int DensityMatrix::write(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + + wspace.initSquareMatrix(*dm_); + + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); +} + +template +int DensityMatrix::read(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + int ierr = 0; + if (mmpi.instancePE0()) + { + hid_t file_id = h5f_file.file_id(); + ierr = mgmol_tools::read_matrix(file_id, name, work_matrix); + } + mmpi.bcast(&ierr, 1); + + if (ierr >= 0) wspace.mpiBcastSquareMatrix(); + if (ierr >= 0) initMatrix(work_matrix); + + return ierr; +} + template class DensityMatrix>; #ifdef HAVE_MAGMA template class DensityMatrix; diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 84804a86..1d09e79e 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -10,6 +10,7 @@ #ifndef MGMOL_DENSITYMATRIX_H #define MGMOL_DENSITYMATRIX_H +#include "HDFrestart.h" #include "MGmol_MPI.h" #include "global.h" @@ -23,7 +24,7 @@ template class DensityMatrix { - int dim_; + const int dim_; std::vector occupation_; MatrixType* dm_; @@ -35,6 +36,10 @@ class DensityMatrix bool occ_uptodate_; bool uniform_occ_; bool stripped_; + + /*! + * Max. occupation of an orbital: 1 with spin, 2 otherwise + */ double orbital_occupation_; DensityMatrix(); @@ -146,6 +151,9 @@ class DensityMatrix double getExpectation(const MatrixType& A); void mix( const double mix, const MatrixType& matA, const int new_orbitals_index); + + int write(HDFrestart& h5f_file, std::string& name); + int read(HDFrestart& h5f_file, std::string& name); }; #endif diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 12e5b04c..19c0d26b 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -584,7 +584,6 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -603,7 +602,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -618,28 +617,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int ExtendedGridOrbitals::write_hdf5( - HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int ExtendedGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int ExtendedGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { if (onpe0) (*MPIdata::sout) << "ExtendedGridOrbitals::write_func_hdf5()...\n"; @@ -829,7 +807,7 @@ int ExtendedGridOrbitals::read_func_hdf5( const std::string datasetname(getDatasetName(name, icolor)); // check if dataset exists... - int err_id = h5f_file.dset_exists(datasetname); + int err_id = h5f_file.checkDataExists(datasetname); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index fb7a4b42..c1a718d0 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -374,8 +374,7 @@ class ExtendedGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, ExtendedGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index f95292ac..08018d47 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -933,12 +933,12 @@ int HDFrestart::getLRCenters(std::multimap& centers, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) @@ -1052,12 +1052,12 @@ int HDFrestart::getLRs(std::shared_ptr lrs, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) { // dataset does not exists diff --git a/src/HDFrestart.h b/src/HDFrestart.h index fef634c2..7ffadff7 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -155,28 +155,30 @@ class HDFrestart return 0; } - hid_t dset_exists(const std::string& datasetname) const + hid_t checkDataExists(const std::string& datasetname) const { - if (active_) - { - return dset_exists(datasetname.c_str()); - } - return 0; + return checkDataExists(datasetname.c_str()); } - herr_t dset_exists(const char* const datasetname) const + herr_t checkDataExists(const char* const datasetname) const + { + herr_t err_id = checkDataExistsLocal(datasetname); + + short id = (short)err_id; + MPI_Bcast(&id, 1, MPI_SHORT, 0, comm_data_); + return (herr_t)id; + } + + hid_t checkDataExistsLocal(const std::string& datasetname) const + { + return checkDataExistsLocal(datasetname.c_str()); + } + + herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; if (active_) { err_id = H5LTfind_dataset(file_id_, datasetname); - - if (err_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::dset_exists() failed for dataset " - << datasetname << std::endl; - } } return err_id; } diff --git a/src/LBFGS_IonicStepper.cc b/src/LBFGS_IonicStepper.cc index caea5f84..93ce03f5 100644 --- a/src/LBFGS_IonicStepper.cc +++ b/src/LBFGS_IonicStepper.cc @@ -328,11 +328,11 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) short check_data = 0; std::vector attr_d(16); + // Open an existing dataset. + std::string datasetname("/LBFGS"); + int err_id = h5f_file.checkDataExists(datasetname); if (onpe0) { - // Open an existing dataset. - std::string datasetname("/LBFGS"); - int err_id = h5f_file.dset_exists(datasetname); if (err_id < 0) { // dataset does not exists (*MPIdata::sout) << "Warning: no dataset " << datasetname @@ -343,10 +343,9 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) dataset_id = H5Dopen2(file_id, "/LBFGS", H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " - "no restart info for LBFGS" - << std::endl; + (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " + "no restart info for LBFGS" + << std::endl; } else { diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index ae8a4d1d..4a1b9442 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -966,7 +966,6 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -984,7 +983,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -997,27 +996,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int LocGridOrbitals::write_hdf5(HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int LocGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int LocGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { Control& ct = *(Control::instance()); hid_t file_id = h5f_file.file_id(); @@ -1249,7 +1228,7 @@ int LocGridOrbitals::read_func_hdf5( const std::string key(itcenter->first); // checkif dataset exists... - int err_id = h5f_file.dset_exists(key); + int err_id = h5f_file.checkDataExistsLocal(key); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/LocGridOrbitals.h b/src/LocGridOrbitals.h index 5616c6b6..951e418f 100644 --- a/src/LocGridOrbitals.h +++ b/src/LocGridOrbitals.h @@ -410,8 +410,7 @@ class LocGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, LocGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/MGmol.h b/src/MGmol.h index aee2b660..4d82dd64 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -130,7 +130,7 @@ class MGmol : public MGmolInterface KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij); void computeHnlPhiAndAdd2HPhi(Ions& ions, OrbitalsType& phi, OrbitalsType& hphi, const KBPsiMatrixSparse* const kbpsi); - int dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, + int dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count); diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 3606aa40..e50e98bc 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -26,6 +26,7 @@ #include "SparseDistMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include "fermi.h" +#include "hdf_tools.h" #include #include @@ -732,114 +733,23 @@ double ProjectedMatrices::checkCond( (*MPIdata::sout) << " CONDITION NUMBER OF THE OVERLAP MATRIX EXCEEDS TOL: " << rcond << "!!!" << std::endl; - Control& ct = *(Control::instance()); if (flag) mmpi.abort(); } return rcond; } -////// TEMPLATE THIS FOR FLOAT OPTION ?? template -int ProjectedMatrices::writeDM_hdf5(HDFrestart& h5f_file) +int ProjectedMatrices::writeDM(HDFrestart& h5f_file) { - hid_t file_id = h5f_file.file_id(); - - ReplicatedWorkSpace& wspace( - ReplicatedWorkSpace::instance()); - - wspace.initSquareMatrix(dm_->getMatrix()); - - if (file_id < 0) return 0; - - hsize_t dims[2] = { dim_, dim_ }; - - // filespace identifier - hid_t dataspace = H5Screate_simple(2, dims, nullptr); - - hid_t dset_id = H5Dcreate2(file_id, "/Density_Matrix", H5T_NATIVE_DOUBLE, - dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5: " - "H5Dcreate2 failed!!!" - << std::endl; - return -1; - } - - hid_t memspace = dataspace; - hid_t filespace = dataspace; - - DISTMATDTYPE* work_matrix = wspace.square_matrix(); - herr_t status = H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, - H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) << "Orbitals: H5Dwrite failed!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Dclose failed!!!" - << std::endl; - return -1; - } - status = H5Sclose(dataspace); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Sclose failed!!!" - << std::endl; - return -1; - } - - return 0; + std::string name("/Density_Matrix"); + return dm_->write(h5f_file, name); } -////// TEMPLATE THIS FOR FLOAT OPTION ?? + template -int ProjectedMatrices::read_dm_hdf5(hid_t file_id) +int ProjectedMatrices::readDM(HDFrestart& h5f_file) { - ReplicatedWorkSpace& wspace( - ReplicatedWorkSpace::instance()); - DISTMATDTYPE* work_matrix = wspace.square_matrix(); - - int ierr = 0; - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.instancePE0()) - { - hid_t dset_id = H5Dopen2(file_id, "/Density_Matrix", H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) - << "H5Dopen failed for /Density_Matrix!!!" << std::endl; - } - else - { - ierr = 1; - herr_t status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) - << "H5Dread failed for /Density_Matrix!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - } - mmpi.bcast(&ierr, 1); - if (ierr >= 0) wspace.mpiBcastSquareMatrix(); - if (ierr >= 0) dm_->initMatrix(work_matrix); - - return ierr; + std::string name("/Density_Matrix"); + return dm_->read(h5f_file, name); } template diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 6725b827..9bc5e3c2 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -93,7 +93,14 @@ class ProjectedMatrices : public ProjectedMatricesInterface std::unique_ptr matHB_; std::unique_ptr matH_; + /*! + * Density Matrix + */ std::unique_ptr> dm_; + + /*! + * Gram matrix of orbitals overlaps + */ std::unique_ptr> gm_; // work matrix for tmp usage @@ -311,8 +318,8 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropy() override; double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; - int writeDM_hdf5(HDFrestart& h5f_file) override; - int read_dm_hdf5(hid_t file_id) override; + int writeDM(HDFrestart& h5f_file) override; + int readDM(HDFrestart& h5f_file) override; void printEigenvalues(std::ostream& os) const; void updateDM(const int iterative_index) override; void updateDMwithEigenstates(const int iterative_index); diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index 653b386d..e02e0b45 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -265,19 +265,19 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction exitWithErrorMessage("updateDMwithRelax"); } - virtual int read_dm_hdf5(hid_t file_id) + virtual int readDM(HDFrestart& h5f_file) { - (void)file_id; + (void)h5f_file; - exitWithErrorMessage("read_dm_hdf5"); + exitWithErrorMessage("readDM"); return 0; } - virtual int writeDM_hdf5(HDFrestart& h5f_file) + virtual int writeDM(HDFrestart& h5f_file) { (void)h5f_file; - exitWithErrorMessage("writeDM_hdf5"); + exitWithErrorMessage("writeDM"); return 0; } diff --git a/src/hdf_tools.cc b/src/hdf_tools.cc index 69a105dc..e56c0478 100644 --- a/src/hdf_tools.cc +++ b/src/hdf_tools.cc @@ -14,12 +14,16 @@ #include #include -using namespace std; +#define MGMOL_HDF5_FAIL(X) \ + { \ + std::cerr << "MGMOL_HDF5 failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } namespace mgmol_tools { -void write1d( - hid_t file_id, const string& datasetname, vector& data, size_t length) +void write1d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t length) { assert(file_id >= 0); @@ -31,7 +35,7 @@ void write1d( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "write1d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write1d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -40,7 +44,7 @@ void write1d( dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write1d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write1d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -49,24 +53,24 @@ void write1d( dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write1d(), H5Dwrite failed!!!" << endl; + std::cerr << "write1d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write1d(), H5Dclose failed!!!" << endl; + std::cerr << "write1d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d( - hid_t file_id, const string& datasetname, vector& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -175,7 +179,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -184,28 +188,28 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d(hid_t file_id, const string& datasetname, vector& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -224,14 +228,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -239,7 +244,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, tc.push_back(t); } - string attname("String_Length"); + std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; hid_t dataspaceA_id = H5Screate_simple(1, dimsA, nullptr); hid_t attribute_id = H5Acreate2(dataset_id, attname.c_str(), H5T_NATIVE_INT, @@ -248,15 +253,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, = H5Awrite(attribute_id, H5T_NATIVE_USHORT, &IonData_MaxStrLength); if (status < 0) { - cerr << "write2d(), Attribute: " << attname << " --- H5Awrite failed!!!" - << endl; + std::cerr << "write2d(), Attribute: " << attname + << " --- H5Awrite failed!!!" << std::endl; } status = H5Dwrite(dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tc[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -265,25 +270,25 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, status = H5Sclose(dataspaceA_id); if (status < 0) { - cerr << "write2d(), H5Sclose failed!!!" << endl; + std::cerr << "write2d(), H5Sclose failed!!!" << std::endl; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "write2d(), H5Aclose failed!!!" << endl; + std::cerr << "write2d(), H5Aclose failed!!!" << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; } } #ifdef MGMOL_USE_HDF5P -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -298,15 +303,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -317,8 +324,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -333,7 +340,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -345,7 +353,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -355,8 +363,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -371,15 +379,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -390,8 +400,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -406,7 +416,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -418,7 +429,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_USHORT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -428,8 +439,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -444,15 +455,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -463,8 +476,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -479,7 +492,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -491,7 +505,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -501,13 +515,13 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); @@ -521,15 +535,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -540,8 +556,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -556,7 +572,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -565,8 +582,9 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -576,7 +594,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -589,7 +607,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, #endif void addAttribute2Dataset( - hid_t dset_id, const char* attname, const vector& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -600,30 +618,30 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); - if (status < 0) cerr << "H5Sclose failed!!!" << endl; + if (status < 0) std::cerr << "H5Sclose failed!!!" << std::endl; //(*MPIdata::sout)<<"Write attribute "<& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -634,21 +652,21 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_INT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); if (status < 0) { - cerr << "H5Sclose failed!!!" << endl; + std::cerr << "H5Sclose failed!!!" << std::endl; return; } @@ -656,14 +674,14 @@ void addAttribute2Dataset( status = H5Awrite(attribute_id, H5T_NATIVE_INT, &attr_data[0]); if (status < 0) { - cerr << "H5Awrite failed!!!" << endl; + std::cerr << "H5Awrite failed!!!" << std::endl; return; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "H5Aclose failed!!!" << endl; + std::cerr << "H5Aclose failed!!!" << std::endl; } } @@ -676,7 +694,7 @@ int whatisopen(hid_t fid) if (cnt <= 0) return cnt; - if (cnt > 1) cout << "HDF5 file: " << cnt << " object(s) open\n"; + if (cnt > 1) std::cout << "HDF5 file: " << cnt << " object(s) open\n"; // objs = malloc(cnt * sizeof(hid_t)); hid_t* objs = new hid_t[cnt]; @@ -699,4 +717,80 @@ int whatisopen(hid_t fid) return howmany; } + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim) +{ + if (file_id < 0) return 0; + + hsize_t dims[2] = { (hsize_t)dim, (hsize_t)dim }; + + // filespace identifier + hid_t dataspace = H5Screate_simple(2, dims, nullptr); + + hid_t dset_id = H5Dcreate2(file_id, name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dcreate2 failed!!!"); + return -1; + } + + hid_t memspace = dataspace; + hid_t filespace = dataspace; + + herr_t status = H5Dwrite( + dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dwrite failed!!!"); + return -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + return -1; + } + status = H5Sclose(dataspace); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Sclose failed!!!"); + return -1; + } + + return 0; +} + +int read_matrix(hid_t file_id, std::string& name, double* matrix) +{ + int ierr = 0; + hid_t dset_id = H5Dopen2(file_id, name.c_str(), H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dopen failed!!"); + ierr = -1; + } + else + { + herr_t status = H5Dread( + dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dread failed!!"); + ierr = -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + ierr = -1; + } + } + + return ierr; } + +} // namespace diff --git a/src/hdf_tools.h b/src/hdf_tools.h index e1a0ed1f..7a64bdce 100644 --- a/src/hdf_tools.h +++ b/src/hdf_tools.h @@ -6,8 +6,8 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE - #include "hdf5.h" + #include #include #include @@ -39,4 +39,8 @@ void addAttribute2Dataset( void addAttribute2Dataset( hid_t dset_id, const char* attname, const std::vector& attr_data); int whatisopen(hid_t fid); + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim); +int read_matrix(hid_t file_id, std::string& name, double* matrix); } diff --git a/src/md.cc b/src/md.cc index 7c6e2aa8..b773c27c 100644 --- a/src/md.cc +++ b/src/md.cc @@ -226,7 +226,7 @@ void checkMaxForces(const std::vector& fion, } template -int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, +int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -245,9 +245,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); - OrbitalsType previous_orbitals("ForDumping", **orbitals, false); + OrbitalsType previous_orbitals("ForDumping", orbitals, false); if (!orbitals_extrapol_->getRestartData(previous_orbitals)) - previous_orbitals.assign(**orbitals); + previous_orbitals.assign(orbitals); + + // write all restart info in HDF5 file int ierr = write_hdf5(h5file, rho.rho_, ions, previous_orbitals, lrs_); mmpi.allreduce(&ierr, 1, MPI_MIN); @@ -259,12 +261,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, << std::endl; return ierr; } - // write_hdf5(h5file, rho.rho_, ions, *orbitals_minus1); - // stepper->write_hdf5(h5file); if (write_extrapolated_wf && ct.out_restart_info > 2) { - ierr = (*orbitals)->write_func_hdf5(h5file, "ExtrapolatedFunction"); + // write extra info needed for seamless MD restart + ierr = orbitals.write(h5file, "ExtrapolatedFunction"); mmpi.allreduce(&ierr, 1, MPI_MIN); if (ierr < 0) { @@ -354,16 +355,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.restart_info > 1) { - int flag_extrapolated_data = 0; - if (onpe0) - { + int flag_extrapolated_data + = h5f_file_->checkDataExists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0000"); - if (flag_extrapolated_data == 0) - flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0"); - } - MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + = h5f_file_->checkDataExists("ExtrapolatedFunction0"); if (ct.restart_info > 2) { @@ -377,7 +373,6 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) // need to reset a few things as we just read new orbitals (*orbitals)->computeGramAndInvS(); - dm_strategy_->update(*current_orbitals_); } DFTsolver::setItCountLarge(); @@ -615,7 +610,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) std::cout @@ -641,7 +636,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) diff --git a/src/restart.cc b/src/restart.cc index 944b1d8e..c12bb258 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -163,9 +163,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, // Write wavefunctions and old centers. if (ct.out_restart_info > 2) { - int ierr = orbitals.write_hdf5(h5f_file); + int ierr = orbitals.write(h5f_file); if (ierr < 0) return ierr; + if (!ct.fullyOccupied()) + { + int ierr = proj_matrices_->writeDM(h5f_file); + if (ierr < 0) return ierr; + } if (ct.isLocMode() && ct.WFExtrapolation() == WFExtrapolationType::Reversible) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 72378704..53793986 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -247,6 +247,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc + ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc + ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) @@ -552,7 +554,7 @@ target_include_directories(testConditionDistMatrix PRIVATE ${Boost_INCLUDE_DIRS} target_include_directories(testConditionDistMatrixPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) @@ -566,6 +568,7 @@ target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) +target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} From a227c04a43d32c932d9bb8e2b3a08695e9ab084f Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 2 Dec 2024 11:53:42 -0500 Subject: [PATCH 08/53] Write dm (#291) * Update use of DM in restart --- src/ProjectedMatrices.cc | 27 +++++++++++++++++++++++++++ src/ProjectedMatrices.h | 2 ++ src/ProjectedMatricesInterface.h | 16 ++++++++++++++++ src/md.cc | 18 ++++++++++++++++++ src/restart.cc | 2 +- 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e50e98bc..74ff01ed 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -745,6 +745,23 @@ int ProjectedMatrices::writeDM(HDFrestart& h5f_file) return dm_->write(h5f_file, name); } +template +int ProjectedMatrices::writeSavedDM(HDFrestart& h5f_file) +{ + std::string name("/Density_Matrix_WF"); + + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + + const MatrixType* matrix = mat_X_old_.get(); + wspace.initSquareMatrix(*matrix); + + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); +} + template int ProjectedMatrices::readDM(HDFrestart& h5f_file) { @@ -752,6 +769,16 @@ int ProjectedMatrices::readDM(HDFrestart& h5f_file) return dm_->read(h5f_file, name); } +template +int ProjectedMatrices::readWFDM(HDFrestart& h5f_file) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.barrier(); + if (mmpi.PE0()) std::cout << "ProjectedMatrices::readWFDM..." << std::endl; + std::string name("/Density_Matrix_WF"); + return dm_->read(h5f_file, name); +} + template void ProjectedMatrices::printEigenvalues(std::ostream& os) const { diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 9bc5e3c2..f2881763 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -319,7 +319,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; int writeDM(HDFrestart& h5f_file) override; + int writeSavedDM(HDFrestart& h5f_file); int readDM(HDFrestart& h5f_file) override; + int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; void updateDM(const int iterative_index) override; void updateDMwithEigenstates(const int iterative_index); diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index e02e0b45..c52595e1 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -273,6 +273,14 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } + virtual int readWFDM(HDFrestart& h5f_file) + { + (void)h5f_file; + + exitWithErrorMessage("readWFDM"); + + return 0; + } virtual int writeDM(HDFrestart& h5f_file) { (void)h5f_file; @@ -281,6 +289,14 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } + virtual int writeSavedDM(HDFrestart& h5f_file) + { + (void)h5f_file; + + exitWithErrorMessage("writeSavedDM"); + + return 0; + } virtual void updateDMwithChebApproximation(const int iterative_index) { (void)iterative_index; diff --git a/src/md.cc b/src/md.cc index b773c27c..b50618c8 100644 --- a/src/md.cc +++ b/src/md.cc @@ -275,6 +275,10 @@ int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, << std::endl; return ierr; } + + // write DM associated with non-extrapolated wavefunctions + // (last computed solution of KS equations) + proj_matrices_->writeSavedDM(h5file); } ierr = h5file.close(); @@ -579,6 +583,13 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) lrs_->clearOldCenters(); } + // save DM for possible restart write + // note: extrapolation is going to modify it! + if ((ct.out_restart_info > 2) + && (((md_iteration_ % ct.checkpoint) == 0) + || (mdstep == ct.num_MD_steps))) + proj_matrices_->saveDM(); + preWFextrapolation(); if (ct.dt > 0. @@ -656,6 +667,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) template void MGmol::loadRestartFile(const std::string filename) { + if (onpe0) std::cout << "loadRestartFile..." << std::endl; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); @@ -678,6 +690,12 @@ void MGmol::loadRestartFile(const std::string filename) global_exit(0); } + if (!ct.fullyOccupied()) + { + // overwrite DM with restart data in dataset Density_Matrix_WF + if (h5file.checkDataExists("Density_Matrix_WF")) + ierr = proj_matrices_->readWFDM(h5file); + } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); diff --git a/src/restart.cc b/src/restart.cc index c12bb258..2fa76ecb 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -168,7 +168,7 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, if (!ct.fullyOccupied()) { - int ierr = proj_matrices_->writeDM(h5f_file); + ierr = proj_matrices_->writeDM(h5f_file); if (ierr < 0) return ierr; } if (ct.isLocMode() From b263532e4f585267d8eab837abb86076bb044195 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 23 Dec 2024 15:04:37 -0500 Subject: [PATCH 09/53] Remove unused function in Control (#294) --- src/Control.cc | 72 -------------------------------------------------- src/Control.h | 1 - 2 files changed, 73 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index dd786678..2272d3cb 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1119,78 +1119,6 @@ void Control::printThermostatInfo(std::ostream& os) const } } -int Control::readOccupations(std::ifstream* tfile) -{ - int count = 0; - float nel = 0.; - do - { - float t1 = 0.; - int nst = 0; - if (mype_ == 0) - { -#ifdef DEBUG - (*MPIdata::sout) << " Occupations of states..." << std::endl; -#endif - (*tfile) >> nst; - if (nst <= 0) - { - (*MPIdata::sout) - << "Control::readOccupations: numst=" << numst << std::endl; - (*MPIdata::sout) << "Control::readOccupations: nst=" << nst - << ", count=" << count << std::endl; - (*MPIdata::sout) << "Control::readOccupations: Bad repeat " - "count for state occupations" - << std::endl; - return -1; - } - if ((count + nst) > numst) - { - (*MPIdata::sout) << "Control::readOccupations: Occupations " - "specified for too many states" - << std::endl; - return -1; - } - - (*tfile) >> t1; - if (t1 < 0.) - { - (*MPIdata::sout) - << "Control::readOccupations: occupation=" << t1 - << std::endl; - (*MPIdata::sout) << "Control::readOccupations: occupation " - "should be a positive number" - << std::endl; - return -1; - } - finishRead(*tfile); - } - int mpirc = MPI_Bcast(&nst, 1, MPI_INT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation numbers failed!!!" << std::endl; - return -1; - } - mpirc = MPI_Bcast(&t1, 1, MPI_FLOAT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation failed!!!" << std::endl; - return -1; - } - nel += nst * t1; - count += nst; - - } while (count < numst); - - nel_ = (int)nel; - - nempty_ = (2 * numst - (int)nel) / 2; - - return count; -} - void Control::setLocMode(const float radius, const float lx, const float ly, const float lz, const float mind_centers) { diff --git a/src/Control.h b/src/Control.h index 5134c718..2e781f79 100644 --- a/src/Control.h +++ b/src/Control.h @@ -294,7 +294,6 @@ class Control const float dtol, const short kim, const short itmax, const short lfil, const short maxfill, const short ilutype); void setSpreadRadius(); - int readOccupations(std::ifstream* tfile); bool checkTimeout() { return timeout_.check(); } bool occupationWidthIsZero() { return occ_width < 1.e-12; } From 3eb52d68f09e997f21b93de90ef7c5a1401e731c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 3 Jan 2025 09:07:27 -0500 Subject: [PATCH 10/53] Change symlink to restart in tests (#295) * enable tests when old link present --- tests/FIRE/test.py | 6 +++++- tests/LBFGS/test.py | 8 +++++++- tests/MD_D72/test.py | 6 +++++- tests/ShortSighted/test.py | 6 +++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/FIRE/test.py b/tests/FIRE/test.py index eaf380d9..6866e3ac 100755 --- a/tests/FIRE/test.py +++ b/tests/FIRE/test.py @@ -49,7 +49,11 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) print(command) diff --git a/tests/LBFGS/test.py b/tests/LBFGS/test.py index 2ff69c9a..324fd129 100755 --- a/tests/LBFGS/test.py +++ b/tests/LBFGS/test.py @@ -49,7 +49,12 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') + +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp2,coords,lrs) print(command) @@ -75,6 +80,7 @@ os.remove('wave.out') if force>tol: + print("Force larger than tol {}".format(tol)) sys.exit(1) sys.exit(0) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index a664b305..17b70de2 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 9b731f4c..27da7f7d 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) From c1d46bc129b8b0ba532676892a08a93a7f51f5b2 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 7 Jan 2025 13:53:37 -0500 Subject: [PATCH 11/53] Extract number empty orbitals from restart file (#296) --- src/Control.h | 3 + src/HDFrestart.cc | 15 ++ src/HDFrestart.h | 3 + src/Hamiltonian.cc | 7 +- src/MVPSolver.cc | 4 +- src/setup.cc | 17 +- tests/CMakeLists.txt | 12 + tests/RestartEnergyAndForces/h2o.xyz | 6 + tests/RestartEnergyAndForces/mgmol.cfg | 33 +++ tests/RestartEnergyAndForces/restart.cfg | 32 +++ tests/RestartEnergyAndForces/test.py | 88 ++++++++ .../testRestartEnergyAndForces.cc | 206 ++++++++++++++++++ 12 files changed, 418 insertions(+), 8 deletions(-) create mode 100644 tests/RestartEnergyAndForces/h2o.xyz create mode 100644 tests/RestartEnergyAndForces/mgmol.cfg create mode 100644 tests/RestartEnergyAndForces/restart.cfg create mode 100755 tests/RestartEnergyAndForces/test.py create mode 100644 tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc diff --git a/src/Control.h b/src/Control.h index 2e781f79..b3401c1d 100644 --- a/src/Control.h +++ b/src/Control.h @@ -242,6 +242,7 @@ class Control const float total_spin, std::string run_directory = "."); void setDefaultValues(); + bool withSpin() { return with_spin_; } bool globalColoring() const { return (coloring_algo_ / 10 == 0); } @@ -263,6 +264,8 @@ class Control float getSpin() const { return total_spin_; } + void setNempty(const int nempty) { nempty_ = nempty; } + short getMGlevels() { return mg_levels_; } bool withPreconditioner() const { return (mg_levels_ >= 0); } diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 08018d47..ef658955 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -2725,6 +2725,21 @@ void HDFrestart::gatherDataXdir(std::vector& data) } } +int HDFrestart::countFunctionObjects(std::string& name) const +{ + int count = 0; + int found = 0; + do + { + std::string datasetname(getDatasetName(name, count)); + // check if dataset exists... + found = checkDataExists(datasetname); + if (found) count++; + } while (found); // dataset exists + + return count; +} + template int HDFrestart::read_1func_hdf5(float*, const std::string&); template int HDFrestart::read_1func_hdf5(double*, const std::string&); diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 7ffadff7..72c5c1f5 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -173,6 +173,7 @@ class HDFrestart return checkDataExistsLocal(datasetname.c_str()); } + // Returns zero (false), a positive (true) or a negative (failure) value. herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; @@ -282,6 +283,8 @@ class HDFrestart int getMDstepFromFile() const; int getFromFile(const std::string& attname) const; + int countFunctionObjects(std::string& name) const; + hid_t createPlist() { hid_t plist_id = H5P_DEFAULT; diff --git a/src/Hamiltonian.cc b/src/Hamiltonian.cc index ba0aa6c4..5a5d2a53 100644 --- a/src/Hamiltonian.cc +++ b/src/Hamiltonian.cc @@ -59,9 +59,9 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) { (*MPIdata::sout) << "Hamiltonian::applyLocal(), new_index =" - << new_index << endl; + << new_index << std::endl; (*MPIdata::sout) << "Hamiltonian::applyLocal(), itindex_ =" - << itindex_ << endl; + << itindex_ << std::endl; } #endif if (force || new_index != itindex_) @@ -76,7 +76,8 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) (*MPIdata::sout) << "Hamiltonian::hlphi up to date, itindex_=" << itindex_ - << endl; + << ", Potential index=" << pot_->getIterativeIndex() + << std::endl; #endif } return *hlphi_; diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6115f933..7ab9f665 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -143,7 +143,6 @@ void MVPSolver::buildTarget_MVP( proj_mat_work_->setHB2H(); proj_mat_work_->updateDM(orbitals_index); - target = proj_mat_work_->dm(); if (ct.verbose > 2) @@ -286,7 +285,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) std::cout << "MVP --- Target energy..." << std::endl; proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) current_proj_mat->printOccupations(os_); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); const double nel = proj_mat_work_->getNel(); if (onpe0 && ct.verbose > 1) os_ << "MVP --- Number of electrons at beta=1 : " << nel @@ -346,7 +345,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "Number of electrons for interpolated DM = " << pnel << std::endl; } - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, *work_); } diff --git a/src/setup.cc b/src/setup.cc index f022678c..9963bfc5 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -50,6 +50,7 @@ int MGmol::setupFromInput(const std::string filename) if (ct.isLocMode()) mymesh->subdivGridx(ct.getMGlevels()); const pb::PEenv& myPEenv = mymesh->peenv(); + if (ct.restart_info > 0) h5f_file_.reset( new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type)); @@ -58,8 +59,20 @@ int MGmol::setupFromInput(const std::string filename) if (status == -1) return -1; const short myspin = mmpi.myspin(); - const int nval = ions_->getNValenceElectrons(); - ct.setNumst(myspin, nval); + const int nel = ions_->getNValenceElectrons(); + // for the case of extended wavefunctions, we can determine the number + // of empty states from the number of wavefunctions in restart file + if (ct.restart_info > 2 && !ct.short_sighted) + { + std::string name = "Function"; + int count = h5f_file_->countFunctionObjects(name); + std::cout << "found " << count << " functions in restart file..." + << std::endl; + int nempty = ct.withSpin() ? count - nel : count - int(0.5 * nel); + ct.setNempty(nempty); + } + ct.setNumst(myspin, nel); + ct.setTolEnergy(); ct.setSpreadRadius(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53793986..b29c1aad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -256,6 +256,8 @@ add_executable(testWFEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) add_executable(testDMandEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) +add_executable(testRestartEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -371,6 +373,15 @@ add_test(NAME testDMandEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRestartEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRestartEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -567,6 +578,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz new file mode 100644 index 00000000..cdc906f6 --- /dev/null +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -0,0 +1,6 @@ +3 +https://pubchem.ncbi.nlm.nih.gov/compound/Water +O 2.5369 -0.1550 0.0 +H 3.0739 0.1550 0.0 +H 2.0000 0.1550 0.0 + diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..e590f810 --- /dev/null +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -0,0 +1,33 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +nempty=2 +[Restart] +output_level=3 +output_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=1 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg new file mode 100644 index 00000000..99bc77d8 --- /dev/null +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -0,0 +1,32 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +input_level=3 +input_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=20 diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py new file mode 100755 index 00000000..349434e8 --- /dev/null +++ b/tests/RestartEnergyAndForces/test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test RestartEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-6] +test_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +ref_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + ref_energy=energy + break + +#sys.exit(0) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +test_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'Eks'): + print(line) + words=line.split() + print(words) + test_energy = words[2] + break + + +tol = 1.e-6 +diff=eval(test_energy)-eval(ref_energy) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(ref_energy,test_energy)) + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc new file mode 100644 index 00000000..e323afbc --- /dev/null +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -0,0 +1,206 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + HDFrestart h5file(ct.restart_file, myPEenv, ct.restart_file_type); + std::string name = "Function"; + int count = h5file.countFunctionObjects(name); + if (count != ct.numst) + { + std::cerr << "The number of functions in the restart file, " + << count << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + // read numst_ wavefunction + int nread = orbitals.read_func_hdf5(h5file, name); + if (nread != ct.numst) + { + std::cerr << "The number of functions read from the restart file, " + << nread << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + // set the iterative index to 1 to differentiate it from first instance + // in MGmol initial() function. This is not very clean and could be + // better designed, but works for now + orbitals.setIterativeIndex(1); + + // set initial DM with uniform occupations + projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->printDM(std::cout); + + // + // evaluate energy and forces with wavefunctions just read + // + std::vector forces; + double eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks : " << eks << std::endl; + std::cout << "Forces :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From eb539633b79a4db3386559408188469fb42163ca Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 16 Jan 2025 12:45:26 -0500 Subject: [PATCH 12/53] Clean up MD_IonicStepper restart data write (#297) --- src/Ion.h | 2 +- src/IonicStepper.cc | 83 ++++++++++++++------------- src/Ions.cc | 2 +- src/MD_IonicStepper.cc | 125 +++++------------------------------------ src/MD_IonicStepper.h | 13 +---- 5 files changed, 63 insertions(+), 162 deletions(-) diff --git a/src/Ion.h b/src/Ion.h index 336dee72..ae870820 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -176,7 +176,7 @@ class Ion } double position(const short i) const { return position_[i]; } - double old_position(const short i) const { return old_position_[i]; } + double getPreviousPosition(const short i) const { return old_position_[i]; } void setPosition(const double x, const double y, const double z) { old_position_[0] = position_[0]; diff --git a/src/IonicStepper.cc b/src/IonicStepper.cc index ca8d53cc..aa3a7f48 100644 --- a/src/IonicStepper.cc +++ b/src/IonicStepper.cc @@ -13,8 +13,6 @@ #include "MGmol_blas1.h" #include -using namespace std; - IonicStepper::IonicStepper(const double dt, const std::vector& atmove, std::vector& tau0, std::vector& taup) : atmove_(atmove), tau0_(tau0), taup_(taup) @@ -51,7 +49,8 @@ IonicStepper::IonicStepper(const double dt, const std::vector& atmove, } int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, - const vector& data, const string& name, const bool create) const + const std::vector& data, const std::string& name, + const bool create) const { hid_t file_id = h5f_file.file_id(); if (file_id < 0) return 0; @@ -62,7 +61,6 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (create) { - // Create the data space for new datasets hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; dataspace_id = H5Screate_simple(2, dims, nullptr); @@ -70,7 +68,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } @@ -81,7 +79,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dcreate2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -93,7 +91,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -103,7 +101,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeAtomicFields: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -111,14 +109,15 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeAtomicFields, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" << endl; + << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" + << std::endl; return -1; } @@ -129,7 +128,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -137,7 +136,8 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, return 0; } -int IonicStepper::writePositions(HDFrestart& h5f_file, const string& name) const +int IonicStepper::writePositions( + HDFrestart& h5f_file, const std::string& name) const { return writeAtomicFields(h5f_file, tau0_, name, false); } @@ -155,7 +155,8 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); if (dataspace_id < 0) { - (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" << endl; + (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" + << std::endl; return -1; } @@ -170,7 +171,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { std::cerr << "IonicStepper::writeVelocities, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -182,12 +183,12 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { (*MPIdata::serr) << "IonicStepper:: H5Dcreate2 /Ionic_velocities failed!!!" - << endl; + << std::endl; return -1; } } - vector data(taup_); + std::vector data(taup_); double minus = -1.; int n = (int)tau0_.size(), ione = 1; DAXPY(&n, &minus, &tau0_[0], &ione, &data[0], &ione); @@ -202,20 +203,20 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const if (status < 0) { (*MPIdata::serr) << "IonicStepper::H5Dwrite velocities failed!!!" - << endl; + << std::endl; return -1; } else { if (onpe0) (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } H5Sclose(dataspace_id); @@ -224,7 +225,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const } int IonicStepper::readAtomicFields( - HDFrestart& h5f_file, vector& data, const string& name) + HDFrestart& h5f_file, std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -239,7 +240,7 @@ int IonicStepper::readAtomicFields( if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper, H5Dopen2 failed for " << name - << " !!!" << endl; + << " !!!" << std::endl; return -1; } @@ -249,14 +250,15 @@ int IonicStepper::readAtomicFields( H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper, H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper, H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } @@ -265,13 +267,14 @@ int IonicStepper::readAtomicFields( return 0; } -int IonicStepper::readPositions_hdf5(HDFrestart& h5f_file, const string& name) +int IonicStepper::readPositions_hdf5( + HDFrestart& h5f_file, const std::string& name) { return readAtomicFields(h5f_file, tau0_, name); } int IonicStepper::writeRandomStates(HDFrestart& h5f_file, - vector& data, const string& name) const + std::vector& data, const std::string& name) const { hid_t file_id = h5f_file.file_id(); bool create = false; @@ -282,7 +285,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper:: H5Dopen2 " << name - << " failed!!! Creating new data file " << endl; + << " failed!!! Creating new data file " << std::endl; // Create the data space for the dataset hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; @@ -291,7 +294,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "Ions::writeRandomStates: H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } // Create the dataset @@ -300,7 +303,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) - << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << endl; + << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << std::endl; return -1; } create = true; @@ -312,7 +315,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeRandomStates: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -320,14 +323,15 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeRandomStates, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeRandomStates, H5Dclose failed!!!" << endl; + << "IonicStepper::writeRandomStates, H5Dclose failed!!!" + << std::endl; return -1; } @@ -338,7 +342,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -346,8 +350,8 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, return 0; } -int IonicStepper::readRandomStates( - HDFrestart& h5f_file, vector& data, const string& name) +int IonicStepper::readRandomStates(HDFrestart& h5f_file, + std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -360,10 +364,10 @@ int IonicStepper::readRandomStates( if (onpe0) { (*MPIdata::sout) - << "H5Dopen failed for /Ionic_RandomStates" << endl; + << "H5Dopen failed for /Ionic_RandomStates" << std::endl; (*MPIdata::sout) << "Set random states to default computed in Ion.cc" - << endl; + << std::endl; } } else @@ -373,19 +377,20 @@ int IonicStepper::readRandomStates( if (onpe0) (*MPIdata::sout) << "Read Ionic random states from " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; herr_t status = H5Dread(dataset_id, H5T_NATIVE_USHORT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper: H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper: H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } diff --git a/src/Ions.cc b/src/Ions.cc index 852151c7..8b956706 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -3209,7 +3209,7 @@ void Ions::initStepperData() for (short i = 0; i < 3; i++) { - taum_.push_back((*lion)->old_position(i)); + taum_.push_back((*lion)->getPreviousPosition(i)); tau0_.push_back((*lion)->position(i)); fion_.push_back((*lion)->force(i)); velocity_.push_back((*lion)->velocity(i)); diff --git a/src/MD_IonicStepper.cc b/src/MD_IonicStepper.cc index 7e797602..a1b4acb1 100644 --- a/src/MD_IonicStepper.cc +++ b/src/MD_IonicStepper.cc @@ -114,7 +114,8 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) (*MPIdata::sout) << "MD_IonicStepper::init() --- use positions " "from restart file with dt=" << dt_ << endl; - // taum_: velocities -> displacements = -dt*vel + // taum_ was initialized with velocities + // set taum_ to displacements = -dt*vel double alpha = -1. * dt_; DSCAL(&size_tau, &alpha, &taum_[0], &ione); @@ -123,117 +124,13 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) taup_ = tau0_; DAXPY(&size_tau, &minus_one, &taum_[0], &ione, &taup_[0], &ione); - // taum_ -> previous positions: tau0_ - dt*vel + // Now set taum_ to previous positions: tau0_ - dt*vel DAXPY(&size_tau, &one, &tau0_[0], &ione, &taum_[0], &ione); } return 0; } -int MD_IonicStepper::writeForces(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)tau0_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_forces", H5T_NATIVE_DOUBLE, - dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_forces failed!!!" << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &fion_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite forces failed!!!" - << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) - << "Ionic forces written into " << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - -int MD_IonicStepper::writeTaum(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)taum_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_velocities", - H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_velocities failed!!!" - << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &taum_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite taum failed!!!" << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) { hid_t file_id = h5f_file.file_id(); @@ -251,11 +148,18 @@ int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) if (status < 0) return status; if (dt_ > 0.) + { writeVelocities(h5f_file); + std::string datasetname("/Ionic_previous_positions"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } else - writeTaum(h5f_file); - - writeForces(h5f_file); + { + std::string datasetname("/Ionic_velocities"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } + std::string datasetname("/Ionic_forces"); + writeAtomicFields(h5f_file, fion_, datasetname, true); } // @@ -469,7 +373,8 @@ void MD_IonicStepper::updateTau() tau0_ = taup_; if (dt_ > 0.) { - // update taup to be able to compute velocity... + // update taup_ to be able to compute velocity: + // taup_ <- tau0_-taum_ int size_tau = (int)tau0_.size(); int ione = 1; double alpha = 1.; diff --git a/src/MD_IonicStepper.h b/src/MD_IonicStepper.h index c62b57a5..24705a1a 100644 --- a/src/MD_IonicStepper.h +++ b/src/MD_IonicStepper.h @@ -7,15 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -//////////////////////////////////////////////////////////////////////////////// -// -// MD_IonicStepper.h: -// -//////////////////////////////////////////////////////////////////////////////// -// $Id$ - -#ifndef MD_IONICSTEPPER_H -#define MD_IONICSTEPPER_H +#ifndef MGMOL_MD_IONICSTEPPER_H +#define MGMOL_MD_IONICSTEPPER_H #include "IonicStepper.h" #include @@ -72,8 +65,6 @@ class MD_IonicStepper : public IonicStepper void updateTau(); double etol(void) const override; int write_hdf5(HDFrestart&) override; - int writeForces(HDFrestart& h5f_file); - int writeTaum(HDFrestart& h5f_file); int init(HDFrestart&) override; void printVelocities(std::ostream& os) const; From 857673f8219c16724e39119685f5bfd9df86e9c4 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 16 Jan 2025 21:53:54 -0500 Subject: [PATCH 13/53] Clean up code based on compiler warnings (#299) --- scripts/build_ubuntu22_openmpi.sh | 2 +- src/Control.cc | 17 ----------------- src/DistributedIonicData.cc | 13 ++++++------- src/HDFrestart.cc | 2 +- src/HamiltonianMVP_DMStrategy.cc | 2 +- src/IonData.cc | 17 ++++++++--------- src/Ions.cc | 6 ------ src/MGmol.cc | 2 ++ src/MVP_DMStrategy.h | 2 +- src/Potentials.cc | 3 +-- src/Potentials.h | 2 +- src/hdf_tools.cc | 30 ++++++++++++++---------------- src/mgmol_run.cc | 1 - tests/testSetGhostValues.cc | 4 ---- 14 files changed, 36 insertions(+), 67 deletions(-) diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..474d61e5 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra -Wno-cast-function-type" \ .. # call make install diff --git a/src/Control.cc b/src/Control.cc index 2272d3cb..2c501aa9 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -31,23 +31,6 @@ float Control::total_spin_ = 0.; std::string Control::run_directory_("."); bool Control::with_spin_ = false; -static void finishRead(std::ifstream& tfile) -{ - // while( tfile.get()!='\n'); - // string str; - // getline(tfile,str); - char str[256]; - tfile.getline(str, 256); - - char cc = (char)tfile.peek(); - while (cc == ('#') || (cc == '\n') || cc == ' ') - { - while (tfile.get() != '\n') - ; - cc = (char)tfile.peek(); // look at next character - } -} - Control::Control() { assert(comm_global_ != MPI_COMM_NULL); diff --git a/src/DistributedIonicData.cc b/src/DistributedIonicData.cc index 6652c41a..05399cac 100644 --- a/src/DistributedIonicData.cc +++ b/src/DistributedIonicData.cc @@ -12,7 +12,6 @@ #include "tools.h" #include -using namespace std; DistributedIonicData::DistributedIonicData( const std::vector& local_names, @@ -31,16 +30,16 @@ DistributedIonicData::DistributedIonicData( int DistributedIonicData::pack(char* cbuff, double* dbuff) { - vector::iterator itf = data_.begin(); - double* dptr = dbuff; + std::vector::iterator itf = data_.begin(); + double* dptr = dbuff; int idx = 0; - for (vector::iterator it = ion_names_.begin(); + for (std::vector::iterator it = ion_names_.begin(); it != ion_names_.end(); ++it) { - string s(*it); + std::string s(*it); FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; @@ -58,7 +57,7 @@ void DistributedIonicData::unpack(char*& cptr, double*& dptr, const short ndata) for (short i = 0; i < ndata; i++) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_names_.push_back(name); cptr += IonData_MaxStrLength; diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index ef658955..ac856ff2 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -428,7 +428,7 @@ void HDFrestart::addReleaseNumber2File(const char* release) // if( onpe0 ) { HDF_FixedLengthString t; - strncpy(t.mystring, release, MyHDFStrLength); + strncpy(t.mystring, release, MyHDFStrLength - 1); herr_t status = H5Awrite(attribute_id, strtype, &t); if (status < 0) { diff --git a/src/HamiltonianMVP_DMStrategy.cc b/src/HamiltonianMVP_DMStrategy.cc index 2b5f8913..be62b894 100644 --- a/src/HamiltonianMVP_DMStrategy.cc +++ b/src/HamiltonianMVP_DMStrategy.cc @@ -57,7 +57,7 @@ HamiltonianMVP_DMStrategy void HamiltonianMVP_DMStrategy::initialize(OrbitalsType& orbitals) + OrbitalsType>::initialize(OrbitalsType&) { } diff --git a/src/IonData.cc b/src/IonData.cc index 8f62ddd8..89c1bd0c 100644 --- a/src/IonData.cc +++ b/src/IonData.cc @@ -10,12 +10,11 @@ #include "IonData.h" #include "tools.h" #include -using namespace std; void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_name = name; cptr += IonData_MaxStrLength; @@ -52,16 +51,16 @@ void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) // pack Ions data for communication void IonData::packIonData( - char* cbuff, int* ibuff, double* dbuff, vector& data) + char* cbuff, int* ibuff, double* dbuff, std::vector& data) { // pack ion_names buffer int idx = 0; - for (vector::iterator idata = data.begin(); idata != data.end(); - ++idata) + for (std::vector::iterator idata = data.begin(); + idata != data.end(); ++idata) { - string s = (*idata).ion_name; + std::string s = (*idata).ion_name; FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; } @@ -69,8 +68,8 @@ void IonData::packIonData( // pack integer datatypes int* iptr = &ibuff[0]; // first pack local data size - *(iptr++) = data.size(); - vector::iterator idata = data.begin(); + *(iptr++) = data.size(); + std::vector::iterator idata = data.begin(); while (idata != data.end()) { // pack atomic_num diff --git a/src/Ions.cc b/src/Ions.cc index 8b956706..11c42f64 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1608,7 +1608,6 @@ int Ions::readAtomsFromXYZ( const std::string& filename, const bool cell_relative) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); // set up list boundaries // get radius of projectors @@ -1698,9 +1697,6 @@ int Ions::readAtomsFromXYZ( int Ions::setAtoms( const std::vector& crds, const std::vector& spec) { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); - const int natoms = crds.size() / 3; double velocity[3] = { 0., 0., 0. }; @@ -1748,8 +1744,6 @@ int Ions::setAtoms( addIonToList(species_[isp], aname, &crds[3 * ia], velocity, locked); } - // std::cout<& data, std::vector& tc) +{ + tc.clear(); + for (auto& d : data) + { + FixedLengthString t; + strncpy(t.mystring, d.c_str(), IonData_MaxStrLength - 1); + tc.push_back(t); + } +} + void write1d(hid_t file_id, const std::string& datasetname, std::vector& data, size_t length) { @@ -235,14 +247,7 @@ void write2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; @@ -583,14 +588,7 @@ void parallelWrite2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index e746611c..a5de1ad5 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -91,7 +91,6 @@ int mgmol_check() Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (myPEenv.color() > 0) { diff --git a/tests/testSetGhostValues.cc b/tests/testSetGhostValues.cc index f8520055..c3680087 100644 --- a/tests/testSetGhostValues.cc +++ b/tests/testSetGhostValues.cc @@ -18,8 +18,6 @@ TEST_CASE("Set ghost values", "[set ghosts") const double ll = 1.; const double lattice[3] = { ll, ll, ll }; - const int nfunc = 10; - MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -101,8 +99,6 @@ TEST_CASE("Set ghost values", "[set ghosts") gf.assign(inner_data.data(), 'd'); gf.set_updated_boundaries(false); - double norm_before = gf.norm2(); - // fill ghost values const bool direction[3] = { true, true, true }; gf.defaultTrade_boundaries(); From f294d3c262801ff173e198113533413b5fa0e225 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 23 Jan 2025 05:19:12 -0800 Subject: [PATCH 14/53] Add getForces for evaluateDMandEnergyAndForces (#300) From 98a79dee2df4c5d653e088fde18e4ce691262b2b Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 24 Jan 2025 12:52:46 -0500 Subject: [PATCH 15/53] Add check for compatibility MVP/Mehrstellen (#301) --- src/Control.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Control.cc b/src/Control.cc index 2c501aa9..d266aa2e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1818,6 +1818,14 @@ int Control::checkOptions() return -1; } + if (DM_solver_ > 0 && lap_type == 0) + { + std::cerr << "DM_solver_ = " << DM_solver_ << std::endl; + std::cerr << "ERROR: Mehrstellen not compatible with MVP inner solvers!" + << std::endl; + return -1; + } + if (it_algo_type_ == 3 && lap_type == 0) { std::cerr From 8eda83e52d9132892337daf92383522bfee3726c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Feb 2025 11:14:17 -0500 Subject: [PATCH 16/53] Add support for Br atom (#302) --- potentials/pseudo.Br_ONCV_PBE_SG15 | 2529 ++++++++++++++++++++++++++++ src/Ions.cc | 14 +- 2 files changed, 2536 insertions(+), 7 deletions(-) create mode 100644 potentials/pseudo.Br_ONCV_PBE_SG15 diff --git a/potentials/pseudo.Br_ONCV_PBE_SG15 b/potentials/pseudo.Br_ONCV_PBE_SG15 new file mode 100644 index 00000000..c135617b --- /dev/null +++ b/potentials/pseudo.Br_ONCV_PBE_SG15 @@ -0,0 +1,2529 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# Br 35.00 6 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 6.00 +# 3 0 2.00 +# 3 1 6.00 +# 3 2 10.00 +# 4 0 2.00 +# 4 1 5.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 2 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 2.49793 -0.73993 5 8 5.10687 +# 1 2.49793 -0.28884 5 8 4.46675 +# 2 2.49793 0.09432 5 8 4.91674 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 2.24365 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.26029 +# 1 2 1.41324 +# 2 2 1.05278 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +Br_ONCV_PBE +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +35 +# Atomic mass +79.90399933 +# Number of valence electrons +7 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +4 +# l-value for state which is local, then type of potential format +3 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 -0.3626325499E+01 0.1750463805E+01 +1 2 0.4803824052E+02 0.7506916045E+00 +2 2 0.1514320363E+02 0.8114306630E-01 +# l= 0 +0.0 -3.0708154786 -2.4047930787 +0.01 -3.070772567 -2.4042133705 +0.02 -3.0706536405 -2.4024662354 +0.03 -3.0704488283 -2.3995597251 +0.04 -3.0701482591 -2.3955018914 +0.05 -3.0697381761 -2.3903039657 +0.06 -3.0692009986 -2.3839803178 +0.07 -3.0685154018 -2.376548404 +0.08 -3.0676564136 -2.3680287033 +0.09 -3.0665955275 -2.3584446428 +0.1 -3.0653008322 -2.3478225123 +0.11 -3.063737157 -2.3361913688 +0.12 -3.0618662322 -2.3235829307 +0.13 -3.0596468635 -2.3100314626 +0.14 -3.0570351213 -2.2955736506 +0.15 -3.053984542 -2.2802484691 +0.16 -3.0504463422 -2.2640970395 +0.17 -3.0463696444 -2.2471624809 +0.18 -3.0417017127 -2.2294897538 +0.19 -3.0363881987 -2.2111254979 +0.2 -3.030373396 -2.1921178621 +0.21 -3.0236005018 -2.1725163322 +0.22 -3.0160118859 -2.1523715511 +0.23 -3.0075493645 -2.1317351366 +0.24 -2.9981544792 -2.1106594957 +0.25 -2.987768779 -2.0891976363 +0.26 -2.976334104 -2.0674029769 +0.27 -2.9637928718 -2.0453291551 +0.28 -2.9500883622 -2.0230298363 +0.29 -2.9351650013 -2.0005585216 +0.3 -2.9189686442 -1.9779683581 +0.31 -2.9014468534 -1.9553119497 +0.32 -2.8825491704 -1.9326411716 +0.33 -2.8622273897 -1.910006986 +0.34 -2.840435816 -1.8874592637 +0.35 -2.8171315174 -1.8650466102 +0.36 -2.7922745751 -1.8428161943 +0.37 -2.7658283185 -1.820813584 +0.38 -2.7377595404 -1.7990825945 +0.39 -2.7080387236 -1.7776651304 +0.4 -2.6766402368 -1.7566010457 +0.41 -2.643542518 -1.7359280108 +0.42 -2.6087282508 -1.7156813851 +0.43 -2.5721845223 -1.6958940993 +0.44 -2.5339029669 -1.6765965483 +0.45 -2.4938798925 -1.6578164926 +0.46 -2.4521163913 -1.639578971 +0.47 -2.4086184283 -1.6219062255 +0.48 -2.3633969093 -1.6048176402 +0.49 -2.3164677785 -1.588329662 +0.5 -2.2678520179 -1.5724557793 +0.51 -2.2175756537 -1.5572064995 +0.52 -2.165669807 -1.5425892978 +0.53 -2.1121706776 -1.5286086079 +0.54 -2.0571194038 -1.5152658873 +0.55 -2.0005622031 -1.5025595131 +0.56 -1.9425500828 -1.4904849389 +0.57 -1.8831389418 -1.4790346138 +0.58 -1.8223892906 -1.4681981355 +0.59 -1.760366311 -1.4579621933 +0.6 -1.6971394766 -1.4483107855 +0.61 -1.6327826828 -1.439225116 +0.62 -1.5673737973 -1.4306838573 +0.63 -1.5009946281 -1.4226631485 +0.64 -1.4337307139 -1.415136705 +0.65 -1.365670995 -1.4080760077 +0.66 -1.2969076816 -1.4014503615 +0.67 -1.227536009 -1.3952270271 +0.68 -1.1576539096 -1.3893714093 +0.69 -1.087361756 -1.3838471989 +0.7 -1.0167621481 -1.3786164792 +0.71 -0.94595957813 -1.3736399185 +0.72 -0.87506012463 -1.3688769435 +0.73 -0.80417116303 -1.3642858995 +0.74 -0.7334010578 -1.3598242236 +0.75 -0.66285885089 -1.355448623 +0.76 -0.59265395182 -1.351115238 +0.77 -0.52289580245 -1.3467798445 +0.78 -0.45369356504 -1.3423980267 +0.79 -0.38515580233 -1.337925358 +0.8 -0.31739015855 -1.3333175826 +0.81 -0.25050304259 -1.3285307948 +0.82 -0.18459931451 -1.3235216181 +0.83 -0.11978197659 -1.3182473801 +0.84 -0.056151869857 -1.3126662853 +0.85 0.0061926226578 -1.3067375838 +0.86 0.067155864931 -1.3004217357 +0.87 0.12664524827 -1.2936805697 +0.88 0.18457146187 -1.2864774371 +0.89 0.2408487527 -1.2787773586 +0.9 0.29539517376 -1.2705471649 +0.91 0.34813281968 -1.2617556292 +0.92 0.39898804895 -1.2523735929 +0.93 0.44789169169 -1.2423740812 +0.94 0.49477924217 -1.2317324118 +0.95 0.53959097092 -1.2204262598 +0.96 0.58227215742 -1.2084357833 +0.97 0.6227733871 -1.1957437779 +0.98 0.66105043859 -1.1823356271 +0.99 0.6970644809 -1.1681994044 +1.0 0.73078216308 -1.1533259198 +1.01 0.76217568426 -1.1377087564 +1.02 0.79122266757 -1.1213442058 +1.03 0.81790653051 -1.1042314518 +1.04 0.84221661359 -1.0863726296 +1.05 0.86414774871 -1.0677726042 +1.06 0.88370040536 -1.048439037 +1.07 0.90088046461 -1.0283822657 +1.08 0.91569937292 -1.0076153718 +1.09 0.92817465654 -0.98615441727 +1.1 0.93832889415 -0.96401792656 +1.11 0.9461899162 -0.94122697008 +1.12 0.9517900713 -0.91780480034 +1.13 0.95516808541 -0.8937777179 +1.14 0.95636695802 -0.86917403945 +1.15 0.95543424502 -0.84402421656 +1.16 0.95242155537 -0.81836058881 +1.17 0.94738611915 -0.79221806224 +1.18 0.94038861073 -0.76563307802 +1.19 0.93149314032 -0.73864360126 +1.2 0.9207680708 -0.71128944543 +1.21 0.90828544129 -0.68361195316 +1.22 0.89411988748 -0.65565351703 +1.23 0.87834887992 -0.62745766744 +1.24 0.86105341064 -0.5990692311 +1.25 0.84231617223 -0.57053359643 +1.26 0.82222172706 -0.54189679743 +1.27 0.80085735674 -0.51320561099 +1.28 0.77831122726 -0.48450694828 +1.29 0.75467272985 -0.45584799729 +1.3 0.73003271242 -0.42727593895 +1.31 0.70448226061 -0.39883777816 +1.32 0.6781130858 -0.37058037334 +1.33 0.65101693162 -0.34254982352 +1.34 0.62328542089 -0.31479192232 +1.35 0.59500981379 -0.28735157546 +1.36 0.56628028474 -0.26027246962 +1.37 0.53718687691 -0.23359805009 +1.38 0.50781737223 -0.20736952858 +1.39 0.47825865123 -0.18162739142 +1.4 0.44859623165 -0.15641087651 +1.41 0.41891237101 -0.13175663087 +1.42 0.38928923477 -0.10770120711 +1.43 0.35980439328 -0.084277653647 +1.44 0.33053421696 -0.061518184098 +1.45 0.30155239027 -0.03945302065 +1.46 0.27292797429 -0.0181091649 +1.47 0.24473036233 0.002486186983 +1.48 0.2170211898 0.022311130325 +1.49 0.18986298266 0.041343142397 +1.5 0.16331164708 0.059564125665 +1.51 0.13742115 0.076957214701 +1.52 0.11224181642 0.093507941835 +1.53 0.087818451049 0.10920533268 +1.54 0.064195549675 0.12403864261 +1.55 0.041408563173 0.13800275831 +1.56 0.019495861764 0.15109077964 +1.57 -0.0015158417959 0.16330294016 +1.58 -0.021596235849 0.17463728157 +1.59 -0.040725475422 0.18509813619 +1.6 -0.058881960784 0.19468876976 +1.61 -0.076052534895 0.20341742456 +1.62 -0.092224070602 0.21129232349 +1.63 -0.10739003129 0.21832549573 +1.64 -0.12154529981 0.2245297272 +1.65 -0.13468985191 0.22992068159 +1.66 -0.14682597759 0.23451524024 +1.67 -0.15795999363 0.23833250484 +1.68 -0.16810089293 0.24139293702 +1.69 -0.17726108092 0.24371882479 +1.7 -0.18545548467 0.24533367189 +1.71 -0.19270225049 0.24626263687 +1.72 -0.19902144307 0.24653172825 +1.73 -0.20443649328 0.2461685876 +1.74 -0.20897181658 0.24520121549 +1.75 -0.21265555109 0.24365927067 +1.76 -0.21551572366 0.24157228058 +1.77 -0.21758447021 0.23897137241 +1.78 -0.21889277302 0.23588720289 +1.79 -0.21947591783 0.23235173626 +1.8 -0.21936729817 0.22839644157 +1.81 -0.21860435564 0.224053418 +1.82 -0.2172225127 0.21935470487 +1.83 -0.21526018396 0.21433189433 +1.84 -0.21275469141 0.20901710885 +1.85 -0.2097443342 0.2034411581 +1.86 -0.20626747958 0.19763549074 +1.87 -0.20236211696 0.19163011691 +1.88 -0.19806642786 0.18545493817 +1.89 -0.19341811175 0.17913927614 +1.9 -0.18845384334 0.17271059028 +1.91 -0.18321119098 0.16619768388 +1.92 -0.17772431694 0.15962506074 +1.93 -0.17202997839 0.15302021981 +1.94 -0.1661601512 0.14640550171 +1.95 -0.16014881031 0.13980526812 +1.96 -0.15402698692 0.13324102441 +1.97 -0.14782425499 0.12673280543 +1.98 -0.14157131194 0.12030174533 +1.99 -0.13529270013 0.11396333406 +2.0 -0.12901767217 0.10773726718 +2.01 -0.12276761168 0.10163636831 +2.02 -0.11656673377 0.095675877908 +2.03 -0.1104363775 0.089868634417 +2.04 -0.10439399399 0.084224202689 +2.05 -0.098461395649 0.0787557483 +2.06 -0.09265062254 0.07346847772 +2.07 -0.086979074138 0.068371912909 +2.08 -0.081458820035 0.06347135213 +2.09 -0.076100125576 0.058770653954 +2.1 -0.070916200757 0.054275981844 +2.11 -0.065911720591 0.049986904382 +2.12 -0.061096352253 0.04590680729 +2.13 -0.05647481096 0.04203534513 +2.14 -0.052050288108 0.038371018518 +2.15 -0.047828987027 0.034914562031 +2.16 -0.043809718782 0.031661293642 +2.17 -0.039995100139 0.028609286026 +2.18 -0.036385130803 0.025754761958 +2.19 -0.032977565996 0.023092357681 +2.2 -0.029772938118 0.020618609458 +2.21 -0.02676722353 0.018327015582 +2.22 -0.023957902849 0.016212045303 +2.23 -0.021342381035 0.014268136652 +2.24 -0.018915988096 0.01248852304 +2.25 -0.016675937928 0.010867451055 +2.26 -0.014617052811 0.0093979619195 +2.27 -0.012733401274 0.0080727169653 +2.28 -0.011021551686 0.0068855480404 +2.29 -0.0094727420429 0.0058278549281 +2.3 -0.0080800306285 0.004891808646 +2.31 -0.0068367888466 0.0040696905805 +2.32 -0.0057335326398 0.0033529863328 +2.33 -0.0047622051973 0.002733387986 +2.34 -0.0039142269282 0.0022027527767 +2.35 -0.003180287976 0.001753168643 +2.36 -0.0025514070016 0.0013760738931 +2.37 -0.002018701345 0.0010641189905 +2.38 -0.0015733073204 0.00081018747083 +2.39 -0.0012057416961 0.000605822272 +2.4 -0.00090807593031 0.00044534423372 +2.41 -0.00067211676854 0.00032260120986 +2.42 -0.00048836884014 0.00022964110509 +2.43 -0.00035064527094 0.00016295096584 +2.44 -0.00025146290618 0.00011730333122 +2.45 -0.00018232534191 8.6257107481e-05 +2.46 -0.00013859768358 6.798715492e-05 +2.47 -0.00011459867708 5.9010018177e-05 +2.48 -0.00010538902877 5.5962206051e-05 +2.49 -0.0001014873045 5.4745829591e-05 +2.5 -9.0886540089e-05 4.9286426798e-05 +2.51 -5.7587161462e-05 3.1411431787e-05 +2.52 -2.3406234748e-05 1.2820803441e-05 +2.53 2.0554012075e-06 -1.1129389439e-06 +2.54 6.3698356898e-06 -3.4490775717e-06 +2.55 4.1835875739e-06 -2.265288898e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 0.16768810909 -0.09197895777 +0.02 0.33490177013 -0.18377178817 +0.03 0.50116814078 -0.27519297668 +0.04 0.66601758415 -0.36605823207 +0.05 0.82898525636 -0.45618509197 +0.06 0.98961267609 -0.54539352126 +0.07 1.1474492702 -0.63350650089 +0.08 1.3020538895 -0.7203506047 +0.09 1.4529962891 -0.80575656223 +0.1 1.5998585678 -0.88955980496 +0.11 1.7422365611 -0.97160099425 +0.12 1.8797411823 -1.0517265285 +0.13 2.0119997077 -1.129789028 +0.14 2.138657 -1.2056477949 +0.15 2.2593766658 -1.2791692479 +0.16 2.3738421434 -1.3502273279 +0.17 2.4817577163 -1.4187038753 +0.18 2.5828494491 -1.4844889757 +0.19 2.6768660426 -1.5474812743 +0.2 2.7635796037 -1.6075882562 +0.21 2.8427863308 -1.6647264933 +0.22 2.9143071076 -1.7188218556 +0.23 2.9779880072 -1.7698096863 +0.24 3.0337007024 -1.8176349412 +0.25 3.0813427856 -1.862252292 +0.26 3.1208379864 -1.9036261898 +0.27 3.1521363034 -1.9417308947 +0.28 3.1752140319 -1.9765504665 +0.29 3.1900737018 -2.0080787184 +0.3 3.1967439186 -2.0363191351 +0.31 3.1952791115 -2.0612847542 +0.32 3.1857591886 -2.0829980127 +0.33 3.1682891025 -2.1014905583 +0.34 3.1429983262 -2.1168030277 +0.35 3.110040245 -2.128984793 +0.36 3.0695914617 -2.1380936747 +0.37 3.0218510224 -2.1441956261 +0.38 2.9670395749 -2.1473643925 +0.39 2.9053984349 -2.1476811361 +0.4 2.8371885957 -2.1452340407 +0.41 2.7626896752 -2.1401178945 +0.42 2.6821987951 -2.1324336474 +0.43 2.596029406 -2.1222879509 +0.44 2.5045100614 -2.1097926797 +0.45 2.4079831454 -2.0950644377 +0.46 2.3068035597 -2.0782240514 +0.47 2.2013373814 -2.0593960544 +0.48 2.0919605029 -2.0387081682 +0.49 1.979057148 -2.0162907368 +0.5 1.8630185217 -1.9922762184 +0.51 1.7442414153 -1.9667986605 +0.52 1.6231266566 -1.9399931205 +0.53 1.500077649 -1.9119951227 +0.54 1.3754991997 -1.88294023 +0.55 1.2497955863 -1.852963328 +0.56 1.123369708 -1.8221983257 +0.57 0.99662116191 -1.7907774522 +0.58 0.86994536282 -1.7588309508 +0.59 0.74373162879 -1.7264863891 +0.6 0.61836264621 -1.6938684873 +0.61 0.49421220571 -1.6610983091 +0.62 0.37164494723 -1.628293199 +0.63 0.25101461062 -1.5955661733 +0.64 0.1326628874 -1.5630255372 +0.65 0.016918843032 -1.5307747113 +0.66 -0.095902538203 -1.4989117459 +0.67 -0.20550134835 -1.4675290072 +0.68 -0.31159326886 -1.4367130384 +0.69 -0.4139103595 -1.4065443242 +0.7 -0.51220225666 -1.377096921 +0.71 -0.60623655903 -1.3484383751 +0.72 -0.6957993 -1.3206296129 +0.73 -0.78069554004 -1.2937247949 +0.74 -0.86074977919 -1.267771236 +0.75 -0.93580620366 -1.2428093839 +0.76 -1.0057293808 -1.218872661 +0.77 -1.0704039576 -1.1959876322 +0.78 -1.1297349296 -1.1741739926 +0.79 -1.1836476649 -1.1534446391 +0.8 -1.23208785 -1.1338057713 +0.81 -1.2750213562 -1.1152570226 +0.82 -1.3124340302 -1.0977916176 +0.83 -1.34433141 -1.0813965584 +0.84 -1.3707383679 -1.0660528348 +0.85 -1.3916986843 -1.0517356601 +0.86 -1.4072745528 -1.0384147296 +0.87 -1.4175460222 -1.0260545009 +0.88 -1.422610376 -1.0146144944 +0.89 -1.4225814539 -1.0040496119 +0.9 -1.4175889193 -0.99431047237 +0.91 -1.4077774754 -0.98534376271 +0.92 -1.393306035 -0.97709260167 +0.93 -1.3743468472 -0.96949691479 +0.94 -1.3510845875 -0.96249381864 +0.95 -1.3237154565 -0.95601798287 +0.96 -1.2924462012 -0.9500020467 +0.97 -1.2574929566 -0.94437710703 +0.98 -1.2190803772 -0.93907301456 +0.99 -1.1774405576 -0.9340188113 +1.0 -1.1328119871 -0.92914312957 +1.01 -1.0854385025 -0.92437458501 +1.02 -1.0355685243 -0.91964203242 +1.03 -0.98345355948 -0.9148751682 +1.04 -0.92934684605 -0.91000502329 +1.05 -0.87350300645 -0.90496400678 +1.06 -0.816176798 -0.89968636628 +1.07 -0.75762253462 -0.89410834934 +1.08 -0.6980927773 -0.88816867824 +1.09 -0.63783592919 -0.88180942862 +1.1 -0.57709733472 -0.87497547403 +1.11 -0.51611779897 -0.86761500666 +1.12 -0.45513452401 -0.85967911403 +1.13 -0.39437481773 -0.8511240992 +1.14 -0.33406039561 -0.84190970478 +1.15 -0.27440564306 -0.83199970915 +1.16 -0.2156182574 -0.82136164295 +1.17 -0.15789253305 -0.80996907161 +1.18 -0.10141514283 -0.79779940144 +1.19 -0.046364491037 -0.78483408344 +1.2 0.0070936861676 -0.77106002408 +1.21 0.058806100624 -0.75646931936 +1.22 0.10862841944 -0.74105803457 +1.23 0.15642703646 -0.72482675268 +1.24 0.20208507927 -0.70778222023 +1.25 0.24549451563 -0.68993480428 +1.26 0.28655709783 -0.67129881979 +1.27 0.32519346013 -0.65189485473 +1.28 0.3613324373 -0.63174663121 +1.29 0.39491300311 -0.61088161585 +1.3 0.42589280427 -0.58933283509 +1.31 0.45423664439 -0.56713591443 +1.32 0.47992110013 -0.54433022226 +1.33 0.50293883528 -0.52095932209 +1.34 0.52328858456 -0.49706895242 +1.35 0.54098326189 -0.47270841667 +1.36 0.55604726384 -0.44792957482 +1.37 0.56851053426 -0.42278632121 +1.38 0.5784190736 -0.3973351547 +1.39 0.58582409631 -0.37163397592 +1.4 0.59078619814 -0.34574254453 +1.41 0.59337646951 -0.3197213658 +1.42 0.59367015608 -0.29363258185 +1.43 0.59175335557 -0.26753813652 +1.44 0.58771624712 -0.24150074456 +1.45 0.58165595284 -0.21558336285 +1.46 0.57367486997 -0.18984748079 +1.47 0.56387989108 -0.16435632354 +1.48 0.55238206952 -0.13916913159 +1.49 0.53929613871 -0.11434699408 +1.5 0.52473924056 -0.089947318262 +1.51 0.50883077179 -0.066027001308 +1.52 0.49169241327 -0.042641156566 +1.53 0.47344482644 -0.019841140122 +1.54 0.45421261948 0.0023209806635 +1.55 0.43411462436 0.023800386214 +1.56 0.41327662113 0.044548868159 +1.57 0.39181369387 0.064528338728 +1.58 0.36984863239 0.083696839073 +1.59 0.34749294808 0.10202192594 +1.6 0.32486335081 0.11946916202 +1.61 0.30206634054 0.13601212183 +1.62 0.27921056742 0.15162430019 +1.63 0.25639594087 0.16628574014 +1.64 0.23372158101 0.17997799414 +1.65 0.21127927552 0.19268795322 +1.66 0.18915759431 0.20440513617 +1.67 0.16743876844 0.21512355823 +1.68 0.1462002202 0.22484043033 +1.69 0.12551333682 0.23355703571 +1.7 0.10544417695 0.24127785752 +1.71 0.08605219679 0.24801143198 +1.72 0.067392201978 0.25376898991 +1.73 0.049510885271 0.25856612142 +1.74 0.032452166492 0.26242022156 +1.75 0.016249456911 0.26535352678 +1.76 0.0009363277765 0.26738900334 +1.77 -0.01346722955 0.26855489619 +1.78 -0.026937386187 0.26887918915 +1.79 -0.039467262404 0.26839525362 +1.8 -0.051042203966 0.26713561709 +1.81 -0.061667545688 0.26513764513 +1.82 -0.071337628583 0.26243806272 +1.83 -0.08006745591 0.25907689464 +1.84 -0.087863054815 0.25509471266 +1.85 -0.094744724104 0.25053311675 +1.86 -0.10073011925 0.24543531389 +1.87 -0.10584404869 0.23984427835 +1.88 -0.11011403507 0.23380415827 +1.89 -0.11356907255 0.22735934704 +1.9 -0.11624417545 0.22055306109 +1.91 -0.11817265478 0.21343130087 +1.92 -0.11939406617 0.20603495539 +1.93 -0.11994678428 0.1984103598 +1.94 -0.1198720834 0.19059676551 +1.95 -0.11921240203 0.18263758346 +1.96 -0.11801032113 0.17457234326 +1.97 -0.1163093863 0.1664389658 +1.98 -0.11415448856 0.15827800515 +1.99 -0.11158739315 0.15012088135 +2.0 -0.10865483203 0.14200718854 +2.01 -0.10539681541 0.13396470695 +2.02 -0.10185745506 0.12602623509 +2.03 -0.098078174326 0.11822061558 +2.04 -0.094096518357 0.11057099092 +2.05 -0.089956724139 0.10310784468 +2.06 -0.085689851344 0.095846617351 +2.07 -0.081335739687 0.088811521611 +2.08 -0.07692635626 0.082018660843 +2.09 -0.072491428123 0.075481468962 +2.1 -0.068066323213 0.069218277483 +2.11 -0.063671603987 0.063233949849 +2.12 -0.059337660545 0.057541412683 +2.13 -0.055085519235 0.052145715315 +2.14 -0.050933564418 0.047049428304 +2.15 -0.046906930776 0.042260020905 +2.16 -0.043014644681 0.037772793136 +2.17 -0.039274785396 0.03358922043 +2.18 -0.035699058908 0.029706278232 +2.19 -0.032293597937 0.026116974923 +2.2 -0.029072509077 0.022818973327 +2.21 -0.026036085371 0.019801644234 +2.22 -0.023189532297 0.017056816878 +2.23 -0.020537674284 0.014575946706 +2.24 -0.018077146728 0.012346366913 +2.25 -0.015810412032 0.010357603026 +2.26 -0.01373392254 0.0085968309318 +2.27 -0.011842116593 0.0070502736984 +2.28 -0.010133845747 0.0057048888874 +2.29 -0.0085993951615 0.0045459798677 +2.3 -0.0072316775018 0.0035587606278 +2.31 -0.0060239032063 0.0027280843754 +2.32 -0.0049651302707 0.0020399454291 +2.33 -0.0040460304292 0.0014785514847 +2.34 -0.0032566099898 0.0010293505502 +2.35 -0.0025860125292 0.00067967964524 +2.36 -0.002022999236 0.00041341795836 +2.37 -0.0015572677849 0.00021897250312 +2.38 -0.0011785405003 8.5467448311e-05 +2.39 -0.00087447260941 -2.6406389151e-06 +2.4 -0.00063670211965 -5.2354589398e-05 +2.41 -0.00045599180494 -7.2310661646e-05 +2.42 -0.00031988217501 -7.6466931673e-05 +2.43 -0.00022336127653 -6.6408931605e-05 +2.44 -0.00015833856538 -4.8596801665e-05 +2.45 -0.00011438780266 -3.2809722084e-05 +2.46 -8.9293750204e-05 -1.6747173877e-05 +2.47 -7.7560959826e-05 -3.3623200012e-06 +2.48 -7.3416025735e-05 3.0092130857e-06 +2.49 -7.2127889582e-05 6.165594886e-06 +2.5 -6.5365569508e-05 6.6445226696e-06 +2.51 -4.1806181002e-05 4.9128311678e-06 +2.52 -1.7125758779e-05 2.2069704328e-06 +2.53 1.4939152703e-06 -1.4573388792e-07 +2.54 4.6480499279e-06 -4.5342490346e-07 +2.55 3.0647699647e-06 -2.98973343e-07 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 2 +0.0 0.0 0.0 +0.01 -0.0016302809637 0.00035794683369 +0.02 -0.0065163515418 0.0014296822768 +0.03 -0.014643908542 0.0032089002835 +0.04 -0.025989159095 0.005685121165 +0.05 -0.040518889195 0.0088437370407 +0.06 -0.058190559384 0.012666075205 +0.07 -0.07895242731 0.017129479149 +0.08 -0.10274369681 0.022207406906 +0.09 -0.12949469307 0.027869546304 +0.1 -0.15912706346 0.034081946665 +0.11 -0.1915540033 0.040807166393 +0.12 -0.22668050609 0.048004435845 +0.13 -0.26440363738 0.055629834819 +0.14 -0.30461283157 0.06363648392 +0.15 -0.34719021073 0.07197474901 +0.16 -0.39201092451 0.08059245791 +0.17 -0.43894351037 0.089435128454 +0.18 -0.48785027272 0.098446206939 +0.19 -0.53858768033 0.10756731604 +0.2 -0.59100678041 0.11673851104 +0.21 -0.64495362865 0.12589854357 +0.22 -0.70026973351 0.13498513144 +0.23 -0.75679251382 0.14393523369 +0.24 -0.81435576809 0.15268532967 +0.25 -0.87279015472 0.16117170107 +0.26 -0.93192368055 0.16933071533 +0.27 -0.99158219813 0.17709911015 +0.28 -1.0515899082 0.18441427678 +0.29 -1.1117698675 0.19121454188 +0.3 -1.1719445 0.19743944636 +0.31 -1.2319361091 0.20303002007 +0.32 -1.2915673908 0.20792905119 +0.33 -1.3506619462 0.21208134961 +0.34 -1.4090447891 0.21543400246 +0.35 -1.4665428517 0.21793662148 +0.36 -1.522985483 0.21954158083 +0.37 -1.5782049408 0.22020424436 +0.38 -1.632036874 0.21988318183 +0.39 -1.6843207957 0.21854037236 +0.4 -1.7349005444 0.21614139504 +0.41 -1.7836247317 0.21265560581 +0.42 -1.8303471759 0.20805629957 +0.43 -1.8749273204 0.20232085696 +0.44 -1.9172306348 0.19543087537 +0.45 -1.9571289972 0.18737228343 +0.46 -1.9945010581 0.17813543873 +0.47 -2.0292325838 0.16771520866 +0.48 -2.0612167791 0.15611103466 +0.49 -2.090354582 0.14332697147 +0.5 -2.1165549426 0.12937171885 +0.51 -2.1397350778 0.11425863393 +0.52 -2.1598206914 0.098005713701 +0.53 -2.176746171 0.080635565211 +0.54 -2.190454786 0.062175383691 +0.55 -2.2008987911 0.04265684497 +0.56 -2.2080395975 0.022116075258 +0.57 -2.211847815 0.00059350427024 +0.58 -2.2123033659 -0.02186620296 +0.59 -2.2093954581 -0.045214335243 +0.6 -2.203122684 -0.06939820269 +0.61 -2.1934928744 -0.09436140661 +0.62 -2.1805231747 -0.12004391211 +0.63 -2.1642398919 -0.14638230327 +0.64 -2.1446783791 -0.17330999434 +0.65 -2.1218829812 -0.20075738758 +0.66 -2.0959068035 -0.22865215777 +0.67 -2.066811508 -0.25691950291 +0.68 -2.0346671658 -0.28548234681 +0.69 -1.999552031 -0.31426159213 +0.7 -1.961552178 -0.34317645587 +0.71 -1.9207612656 -0.37214470648 +0.72 -1.8772802547 -0.40108292621 +0.73 -1.8312170584 -0.42990680852 +0.74 -1.7826861934 -0.45853144401 +0.75 -1.7318084432 -0.48687159166 +0.76 -1.6787103295 -0.51484205255 +0.77 -1.6235238527 -0.542357874 +0.78 -1.5663860216 -0.56933467063 +0.79 -1.5074384166 -0.59568891442 +0.8 -1.4468267401 -0.62133822183 +0.81 -1.3847003559 -0.64620163671 +0.82 -1.3212118192 -0.6701999082 +0.83 -1.2565163983 -0.69325576249 +0.84 -1.1907715897 -0.71529416746 +0.85 -1.1241366287 -0.73624258926 +0.86 -1.0567719967 -0.75603123994 +0.87 -0.98883892656 -0.77459331519 +0.88 -0.92049890866 -0.79186522139 +0.89 -0.85191319821 -0.80778679106 +0.9 -0.78324232642 -0.82230148608 +0.91 -0.71464561672 -0.83535658783 +0.92 -0.64628070784 -0.84690337375 +0.93 -0.57830308546 -0.85689727953 +0.94 -0.51086562408 -0.86529804664 +0.95 -0.4441181986 -0.8720698411 +0.96 -0.37820717285 -0.87718139491 +0.97 -0.31327485984 -0.88060612622 +0.98 -0.24945933646 -0.88232217166 +0.99 -0.18689397376 -0.88231247569 +1.0 -0.12570707039 -0.88056484165 +1.01 -0.066021505913 -0.87707196654 +1.02 -0.0079545268368 -0.871831461 +1.03 0.048382799934 -0.86484587151 +1.04 0.10288589917 -0.85612263048 +1.05 0.15545667375 -0.84567402721 +1.06 0.20600385579 -0.83351716716 +1.07 0.25444313475 -0.81967394231 +1.08 0.30069748155 -0.80417095395 +1.09 0.34469751688 -0.78703927055 +1.1 0.38638120612 -0.76831446513 +1.11 0.42569413044 -0.74803646205 +1.12 0.46258944515 -0.72624964296 +1.13 0.49702846809 -0.70300202094 +1.14 0.52898007647 -0.67834564529 +1.15 0.5584209087 -0.65233634601 +1.16 0.58533542208 -0.62503380899 +1.17 0.60971583811 -0.59650035482 +1.18 0.63156188005 -0.56680171908 +1.19 0.65088091394 -0.5360069337 +1.2 0.66768769801 -0.5041873814 +1.21 0.68200399062 -0.47141655935 +1.22 0.69385878215 -0.43777069314 +1.23 0.70328821897 -0.40332830942 +1.24 0.71033429279 -0.3681684753 +1.25 0.71504576268 -0.33237249494 +1.26 0.71747825821 -0.29602370728 +1.27 0.71769167806 -0.25920453432 +1.28 0.7157522691 -0.22199923787 +1.29 0.71173247372 -0.18449341844 +1.3 0.70570736789 -0.14677065047 +1.31 0.69775812741 -0.10891610311 +1.32 0.68797057633 -0.071014918121 +1.33 0.67643200525 -0.033149740959 +1.34 0.66323581364 0.0045952849381 +1.35 0.64847703464 0.042138299609 +1.36 0.63225221039 0.079399428431 +1.37 0.61466413529 0.1162974046 +1.38 0.59581231224 0.15275639672 +1.39 0.57580091001 0.18870011769 +1.4 0.55473560956 0.22405425572 +1.41 0.53271899406 0.25874913705 +1.42 0.50985999692 0.29271380992 +1.43 0.48626100505 0.32588396457 +1.44 0.462028045 0.35819554496 +1.45 0.43726595407 0.3895878196 +1.46 0.41207481124 0.42000499326 +1.47 0.38656057126 0.44939063297 +1.48 0.3608175272 0.47769672342 +1.49 0.33494686826 0.5048740466 +1.5 0.30904061714 0.53088029232 +1.51 0.28319158621 0.55567492895 +1.52 0.25748919177 0.57922141737 +1.53 0.2320165748 0.60148809531 +1.54 0.20685930393 0.62244468231 +1.55 0.18209023524 0.64206822004 +1.56 0.15778974576 0.66033516372 +1.57 0.13402102121 0.67723064065 +1.58 0.11085528354 0.69273914034 +1.59 0.088348538575 0.70685266968 +1.6 0.066561188568 0.71956401527 +1.61 0.045541575888 0.73087203775 +1.62 0.025339115934 0.74077747805 +1.63 0.0059938468033 0.74928603386 +1.64 -0.012455641987 0.75640594374 +1.65 -0.02997814 0.762149628 +1.66 -0.046545525399 0.76653231439 +1.67 -0.062135771965 0.76957291518 +1.68 -0.07673072274 0.77129311635 +1.69 -0.090317688661 0.77171796373 +1.7 -0.10288787787 0.77087507287 +1.71 -0.11443785977 0.76879513028 +1.72 -0.12496755668 0.76551117705 +1.73 -0.13448246389 0.76105890947 +1.74 -0.14299056152 0.75547633432 +1.75 -0.15050566682 0.74880338946 +1.76 -0.15704320029 0.74108264673 +1.77 -0.162624416 0.73235738719 +1.78 -0.167271634 0.72267441308 +1.79 -0.17101238302 0.71207933872 +1.8 -0.17387544024 0.70062292022 +1.81 -0.17589320453 0.68835205878 +1.82 -0.17710054196 0.67532110015 +1.83 -0.17753340761 0.66157787454 +1.84 -0.17723165939 0.64717812643 +1.85 -0.17623431712 0.63217172116 +1.86 -0.17458444823 0.61661402113 +1.87 -0.17232411255 0.60055706063 +1.88 -0.16949748901 0.58405433927 +1.89 -0.16614984949 0.56716024762 +1.9 -0.16232429048 0.54992499374 +1.91 -0.15806950473 0.53240543237 +1.92 -0.15342617165 0.51464805908 +1.93 -0.14844426834 0.49670968764 +1.94 -0.14316373324 0.47863572096 +1.95 -0.13763122805 0.46047840766 +1.96 -0.13188874746 0.44228507248 +1.97 -0.12597657543 0.42410101919 +1.98 -0.1199394249 0.40597590216 +1.99 -0.11381039894 0.38794793686 +2.0 -0.10763397714 0.37006616432 +2.01 -0.10144010235 0.35236584341 +2.02 -0.095265583264 0.33488859255 +2.03 -0.089142973563 0.31767206088 +2.04 -0.083098195882 0.30074770623 +2.05 -0.077166630832 0.28415568454 +2.06 -0.071365978984 0.26791978875 +2.07 -0.065724731008 0.25207376486 +2.08 -0.060262269553 0.23664279998 +2.09 -0.054995055424 0.22164931787 +2.1 -0.049945501387 0.2071213062 +2.11 -0.045120802665 0.19307206246 +2.12 -0.040537413162 0.17952387706 +2.13 -0.036203128407 0.16649020909 +2.14 -0.032122968947 0.15398166035 +2.15 -0.028307414412 0.1420144928 +2.16 -0.024753587768 0.13059030089 +2.17 -0.02146523645 0.11971812052 +2.18 -0.018441211133 0.10940089432 +2.19 -0.015675932017 0.099636041806 +2.2 -0.013168594079 0.090427492502 +2.21 -0.010909475495 0.081766015473 +2.22 -0.0088911546145 0.073646095504 +2.23 -0.00710559859 0.06606113076 +2.24 -0.0055403251186 0.058996127683 +2.25 -0.0041843627321 0.052439195792 +2.26 -0.00302493571 0.046375295857 +2.27 -0.002048258537 0.040787704768 +2.28 -0.0012396425611 0.03565823697 +2.29 -0.00058545857417 0.030970770784 +2.3 -7.0524374943e-05 0.026706982272 +2.31 0.00032128610825 0.022847262124 +2.32 0.00060267490947 0.019374509348 +2.33 0.00079004202243 0.016268357268 +2.34 0.00089784358158 0.013509078375 +2.35 0.00093745353246 0.011078294802 +2.36 0.00092553347338 0.0089539544753 +2.37 0.00087252062706 0.0071167327738 +2.38 0.00078791337712 0.0055471546788 +2.39 0.00068748885458 0.0042218314666 +2.4 0.00057611638219 0.0031221978739 +2.41 0.0004608042569 0.0022279088413 +2.42 0.00035518992628 0.0015146245011 +2.43 0.00025785924786 0.00096472501905 +2.44 0.00017329303283 0.00055694774566 +2.45 0.00010989139907 0.00026733139288 +2.46 6.1872491483e-05 7.829302357e-05 +2.47 2.953833815e-05 -3.334821152e-05 +2.48 1.4806544348e-05 -9.5806441567e-05 +2.49 9.6783992659e-06 -0.00011221658675 +2.5 8.5136479095e-06 -9.7376625643e-05 +2.51 4.2505513471e-06 -6.772287195e-05 +2.52 1.390094449e-06 -2.9382005001e-05 +2.53 -2.082365022e-07 2.20387025e-06 +2.54 -6.504514312e-07 6.8840503135e-06 +2.55 -4.3057462874e-07 4.5569849889e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -7.0231214206E+00 +0.01 -7.0453717502E+00 +0.02 -7.1143468216E+00 +0.03 -7.2090176842E+00 +0.04 -7.3083553876E+00 +0.05 -7.3952429292E+00 +0.06 -7.4597953536E+00 +0.07 -7.4991297585E+00 +0.08 -7.5152902525E+00 +0.09 -7.5128744992E+00 +0.1 -7.4971693665E+00 +0.11 -7.4730064339E+00 +0.12 -7.4442385526E+00 +0.13 -7.4136363893E+00 +0.14 -7.3830123504E+00 +0.15 -7.3534314963E+00 +0.16 -7.3254268122E+00 +0.17 -7.2991803384E+00 +0.18 -7.2746597190E+00 +0.19 -7.2517133907E+00 +0.2 -7.2301330081E+00 +0.21 -7.2096920714E+00 +0.22 -7.1901684172E+00 +0.23 -7.1713562496E+00 +0.24 -7.1530717169E+00 +0.25 -7.1351547228E+00 +0.26 -7.1174685568E+00 +0.27 -7.0998985249E+00 +0.28 -7.0823500273E+00 +0.29 -7.0647465397E+00 +0.3 -7.0470276188E+00 +0.31 -7.0291470239E+00 +0.32 -7.0110709744E+00 +0.33 -6.9927765879E+00 +0.34 -6.9742504049E+00 +0.35 -6.9554870639E+00 +0.36 -6.9364880834E+00 +0.37 -6.9172607312E+00 +0.38 -6.8978169707E+00 +0.39 -6.8781725207E+00 +0.4 -6.8583459681E+00 +0.41 -6.8383579698E+00 +0.42 -6.8182305369E+00 +0.43 -6.7979863986E+00 +0.44 -6.7776484445E+00 +0.45 -6.7572392483E+00 +0.46 -6.7367806674E+00 +0.47 -6.7162935188E+00 +0.48 -6.6957973262E+00 +0.49 -6.6753101150E+00 +0.5 -6.6548482902E+00 +0.51 -6.6344265604E+00 +0.52 -6.6140578820E+00 +0.53 -6.5937534575E+00 +0.54 -6.5735227996E+00 +0.55 -6.5533737172E+00 +0.56 -6.5333124572E+00 +0.57 -6.5133437238E+00 +0.58 -6.4934708208E+00 +0.59 -6.4736956904E+00 +0.6 -6.4540190739E+00 +0.61 -6.4344405251E+00 +0.62 -6.4149585733E+00 +0.63 -6.3955707664E+00 +0.64 -6.3762737537E+00 +0.65 -6.3570633874E+00 +0.66 -6.3379347681E+00 +0.67 -6.3188823126E+00 +0.68 -6.2998998303E+00 +0.69 -6.2809805806E+00 +0.7 -6.2621173131E+00 +0.71 -6.2433023331E+00 +0.72 -6.2245275562E+00 +0.73 -6.2057845580E+00 +0.74 -6.1870646259E+00 +0.75 -6.1683588117E+00 +0.76 -6.1496579790E+00 +0.77 -6.1309528635E+00 +0.78 -6.1122341237E+00 +0.79 -6.0934923948E+00 +0.8 -6.0747183433E+00 +0.81 -6.0559027212E+00 +0.82 -6.0370364200E+00 +0.83 -6.0181105239E+00 +0.84 -5.9991163614E+00 +0.85 -5.9800455551E+00 +0.86 -5.9608900694E+00 +0.87 -5.9416422543E+00 +0.88 -5.9222948867E+00 +0.89 -5.9028412073E+00 +0.9 -5.8832749527E+00 +0.91 -5.8635903839E+00 +0.92 -5.8437823081E+00 +0.93 -5.8238460972E+00 +0.94 -5.8037776995E+00 +0.95 -5.7835736400E+00 +0.96 -5.7632310315E+00 +0.97 -5.7427475878E+00 +0.98 -5.7221215886E+00 +0.99 -5.7013518781E+00 +1.0 -5.6804378489E+00 +1.01 -5.6593794226E+00 +1.02 -5.6381770133E+00 +1.03 -5.6168315332E+00 +1.04 -5.5953443695E+00 +1.05 -5.5737173207E+00 +1.06 -5.5519525829E+00 +1.07 -5.5300527090E+00 +1.08 -5.5080205953E+00 +1.09 -5.4858594768E+00 +1.1 -5.4635728450E+00 +1.11 -5.4411644391E+00 +1.12 -5.4186381993E+00 +1.13 -5.3959983139E+00 +1.14 -5.3732491142E+00 +1.15 -5.3503950752E+00 +1.16 -5.3274407901E+00 +1.17 -5.3043909888E+00 +1.18 -5.2812504690E+00 +1.19 -5.2580240944E+00 +1.2 -5.2347167970E+00 +1.21 -5.2113335516E+00 +1.22 -5.1878793578E+00 +1.23 -5.1643592404E+00 +1.24 -5.1407782392E+00 +1.25 -5.1171413949E+00 +1.26 -5.0934537514E+00 +1.27 -5.0697203419E+00 +1.28 -5.0459461891E+00 +1.29 -5.0221363068E+00 +1.3 -4.9982956801E+00 +1.31 -4.9744292806E+00 +1.32 -4.9505420602E+00 +1.33 -4.9266389339E+00 +1.34 -4.9027248049E+00 +1.35 -4.8788045416E+00 +1.36 -4.8548829774E+00 +1.37 -4.8309649370E+00 +1.38 -4.8070551869E+00 +1.39 -4.7831584769E+00 +1.4 -4.7592795265E+00 +1.41 -4.7354229970E+00 +1.42 -4.7115935473E+00 +1.43 -4.6877957587E+00 +1.44 -4.6640341935E+00 +1.45 -4.6403133701E+00 +1.46 -4.6166377312E+00 +1.47 -4.5930117259E+00 +1.48 -4.5694396733E+00 +1.49 -4.5459259073E+00 +1.5 -4.5224746470E+00 +1.51 -4.4990900739E+00 +1.52 -4.4757763045E+00 +1.53 -4.4525373437E+00 +1.54 -4.4293771944E+00 +1.55 -4.4062996642E+00 +1.56 -4.3833086284E+00 +1.57 -4.3604076923E+00 +1.58 -4.3376005463E+00 +1.59 -4.3148906194E+00 +1.6 -4.2922813822E+00 +1.61 -4.2697760768E+00 +1.62 -4.2473779371E+00 +1.63 -4.2250900034E+00 +1.64 -4.2029152612E+00 +1.65 -4.1808565335E+00 +1.66 -4.1589165503E+00 +1.67 -4.1370979017E+00 +1.68 -4.1154030601E+00 +1.69 -4.0938343674E+00 +1.7 -4.0723940409E+00 +1.71 -4.0510841603E+00 +1.72 -4.0299066951E+00 +1.73 -4.0088634503E+00 +1.74 -3.9879561588E+00 +1.75 -3.9671863413E+00 +1.76 -3.9465555084E+00 +1.77 -3.9260648902E+00 +1.78 -3.9057157915E+00 +1.79 -3.8855091504E+00 +1.8 -3.8654460818E+00 +1.81 -3.8455272395E+00 +1.82 -3.8257535560E+00 +1.83 -3.8061254478E+00 +1.84 -3.7866435899E+00 +1.85 -3.7673082659E+00 +1.86 -3.7481198716E+00 +1.87 -3.7290785831E+00 +1.88 -3.7101845281E+00 +1.89 -3.6914377938E+00 +1.9 -3.6728382611E+00 +1.91 -3.6543859374E+00 +1.92 -3.6360805050E+00 +1.93 -3.6179218406E+00 +1.94 -3.5999095315E+00 +1.95 -3.5820432604E+00 +1.96 -3.5643225918E+00 +1.97 -3.5467470213E+00 +1.98 -3.5293160912E+00 +1.99 -3.5120291557E+00 +2.0 -3.4948856938E+00 +2.01 -3.4778850017E+00 +2.02 -3.4610264332E+00 +2.03 -3.4443092916E+00 +2.04 -3.4277328079E+00 +2.05 -3.4112962860E+00 +2.06 -3.3949988785E+00 +2.07 -3.3788398185E+00 +2.08 -3.3628182624E+00 +2.09 -3.3469333416E+00 +2.1 -3.3311842302E+00 +2.11 -3.3155699746E+00 +2.12 -3.3000897024E+00 +2.13 -3.2847424519E+00 +2.14 -3.2695272359E+00 +2.15 -3.2544431327E+00 +2.16 -3.2394890382E+00 +2.17 -3.2246639597E+00 +2.18 -3.2099668049E+00 +2.19 -3.1953963961E+00 +2.2 -3.1809517062E+00 +2.21 -3.1666314047E+00 +2.22 -3.1524342799E+00 +2.23 -3.1383591039E+00 +2.24 -3.1244044055E+00 +2.25 -3.1105688087E+00 +2.26 -3.0968509426E+00 +2.27 -3.0832494382E+00 +2.28 -3.0697627913E+00 +2.29 -3.0563898779E+00 +2.3 -3.0431294386E+00 +2.31 -3.0299801887E+00 +2.32 -3.0169410939E+00 +2.33 -3.0040109831E+00 +2.34 -2.9911887500E+00 +2.35 -2.9784733778E+00 +2.36 -2.9658637789E+00 +2.37 -2.9533589232E+00 +2.38 -2.9409577917E+00 +2.39 -2.9286593571E+00 +2.4 -2.9164625797E+00 +2.41 -2.9043664216E+00 +2.42 -2.8923698868E+00 +2.43 -2.8804718654E+00 +2.44 -2.8686712935E+00 +2.45 -2.8569671495E+00 +2.46 -2.8453582499E+00 +2.47 -2.8338434784E+00 +2.48 -2.8224217322E+00 +2.49 -2.8110918278E+00 +2.5 -2.7998525918E+00 +2.51 -2.7887028310E+00 +2.52 -2.7776415409E+00 +2.53 -2.7666676118E+00 +2.54 -2.7557799085E+00 +2.55 -2.7449775402E+00 +2.56 -2.7342595084E+00 +2.57 -2.7236248015E+00 +2.58 -2.7130724883E+00 +2.59 -2.7026015965E+00 +2.6 -2.6922111078E+00 +2.61 -2.6819001800E+00 +2.62 -2.6716679021E+00 +2.63 -2.6615133021E+00 +2.64 -2.6514355718E+00 +2.65 -2.6414338420E+00 +2.66 -2.6315071900E+00 +2.67 -2.6216548251E+00 +2.68 -2.6118759290E+00 +2.69 -2.6021696307E+00 +2.7 -2.5925351489E+00 +2.71 -2.5829717156E+00 +2.72 -2.5734785124E+00 +2.73 -2.5640547609E+00 +2.74 -2.5546997426E+00 +2.75 -2.5454126928E+00 +2.76 -2.5361928300E+00 +2.77 -2.5270394846E+00 +2.78 -2.5179519390E+00 +2.79 -2.5089294248E+00 +2.8 -2.4999712991E+00 +2.81 -2.4910768846E+00 +2.82 -2.4822454645E+00 +2.83 -2.4734763795E+00 +2.84 -2.4647690035E+00 +2.85 -2.4561226740E+00 +2.86 -2.4475367100E+00 +2.87 -2.4390105350E+00 +2.88 -2.4305435276E+00 +2.89 -2.4221350247E+00 +2.9 -2.4137844549E+00 +2.91 -2.4054912392E+00 +2.92 -2.3972547682E+00 +2.93 -2.3890744373E+00 +2.94 -2.3809497189E+00 +2.95 -2.3728800481E+00 +2.96 -2.3648648185E+00 +2.97 -2.3569035173E+00 +2.98 -2.3489956165E+00 +2.99 -2.3411405637E+00 +3.0 -2.3333378034E+00 +3.01 -2.3255868594E+00 +3.02 -2.3178872183E+00 +3.03 -2.3102383329E+00 +3.04 -2.3026397232E+00 +3.05 -2.2950909160E+00 +3.06 -2.2875914196E+00 +3.07 -2.2801407030E+00 +3.08 -2.2727383444E+00 +3.09 -2.2653838770E+00 +3.1 -2.2580768156E+00 +3.11 -2.2508166903E+00 +3.12 -2.2436030858E+00 +3.13 -2.2364355573E+00 +3.14 -2.2293136286E+00 +3.15 -2.2222368833E+00 +3.16 -2.2152049132E+00 +3.17 -2.2082172941E+00 +3.18 -2.2012735614E+00 +3.19 -2.1943733441E+00 +3.2 -2.1875162423E+00 +3.21 -2.1807018469E+00 +3.22 -2.1739297205E+00 +3.23 -2.1671995163E+00 +3.24 -2.1605108486E+00 +3.25 -2.1538633207E+00 +3.26 -2.1472565296E+00 +3.27 -2.1406901400E+00 +3.28 -2.1341637841E+00 +3.29 -2.1276770788E+00 +3.3 -2.1212296497E+00 +3.31 -2.1148211738E+00 +3.32 -2.1084513002E+00 +3.33 -2.1021196610E+00 +3.34 -2.0958259044E+00 +3.35 -2.0895697208E+00 +3.36 -2.0833507757E+00 +3.37 -2.0771687174E+00 +3.38 -2.0710232112E+00 +3.39 -2.0649139620E+00 +3.4 -2.0588406508E+00 +3.41 -2.0528029436E+00 +3.42 -2.0468005172E+00 +3.43 -2.0408330924E+00 +3.44 -2.0349003651E+00 +3.45 -2.0290020196E+00 +3.46 -2.0231377399E+00 +3.47 -2.0173072634E+00 +3.48 -2.0115103000E+00 +3.49 -2.0057465539E+00 +3.5 -2.0000157114E+00 +3.51 -1.9943175274E+00 +3.52 -1.9886517255E+00 +3.53 -1.9830180291E+00 +3.54 -1.9774161269E+00 +3.55 -1.9718457881E+00 +3.56 -1.9663067504E+00 +3.57 -1.9607987500E+00 +3.58 -1.9553214969E+00 +3.59 -1.9498747526E+00 +3.6 -1.9444582756E+00 +3.61 -1.9390718144E+00 +3.62 -1.9337151017E+00 +3.63 -1.9283878872E+00 +3.64 -1.9230899509E+00 +3.65 -1.9178210530E+00 +3.66 -1.9125809500E+00 +3.67 -1.9073693767E+00 +3.68 -1.9021861346E+00 +3.69 -1.8970309951E+00 +3.7 -1.8919037296E+00 +3.71 -1.8868040854E+00 +3.72 -1.8817318554E+00 +3.73 -1.8766868316E+00 +3.74 -1.8716687958E+00 +3.75 -1.8666775210E+00 +3.76 -1.8617127770E+00 +3.77 -1.8567743797E+00 +3.78 -1.8518621212E+00 +3.79 -1.8469757937E+00 +3.8 -1.8421151646E+00 +3.81 -1.8372800505E+00 +3.82 -1.8324702607E+00 +3.83 -1.8276855969E+00 +3.84 -1.8229258539E+00 +3.85 -1.8181908181E+00 +3.86 -1.8134803248E+00 +3.87 -1.8087941851E+00 +3.88 -1.8041322098E+00 +3.89 -1.7994941909E+00 +3.9 -1.7948799518E+00 +3.91 -1.7902893242E+00 +3.92 -1.7857221279E+00 +3.93 -1.7811781826E+00 +3.94 -1.7766572800E+00 +3.95 -1.7721592746E+00 +3.96 -1.7676839959E+00 +3.97 -1.7632312719E+00 +3.98 -1.7588009238E+00 +3.99 -1.7543927664E+00 +4.0 -1.7500066580E+00 +4.01 -1.7456424349E+00 +4.02 -1.7412999332E+00 +4.03 -1.7369789755E+00 +4.04 -1.7326793994E+00 +4.05 -1.7284010643E+00 +4.06 -1.7241438138E+00 +4.07 -1.7199074918E+00 +4.08 -1.7156919241E+00 +4.09 -1.7114969666E+00 +4.1 -1.7073224806E+00 +4.11 -1.7031683172E+00 +4.12 -1.6990343275E+00 +4.13 -1.6949203421E+00 +4.14 -1.6908262304E+00 +4.15 -1.6867518576E+00 +4.16 -1.6826970815E+00 +4.17 -1.6786617602E+00 +4.18 -1.6746457304E+00 +4.19 -1.6706488710E+00 +4.2 -1.6666710522E+00 +4.21 -1.6627121385E+00 +4.22 -1.6587719945E+00 +4.23 -1.6548504641E+00 +4.24 -1.6509474321E+00 +4.25 -1.6470627748E+00 +4.26 -1.6431963631E+00 +4.27 -1.6393480680E+00 +4.28 -1.6355177416E+00 +4.29 -1.6317052709E+00 +4.3 -1.6279105396E+00 +4.31 -1.6241334248E+00 +4.32 -1.6203738033E+00 +4.33 -1.6166315370E+00 +4.34 -1.6129065112E+00 +4.35 -1.6091986186E+00 +4.36 -1.6055077418E+00 +4.37 -1.6018337635E+00 +4.38 -1.5981765561E+00 +4.39 -1.5945360003E+00 +4.4 -1.5909119986E+00 +4.41 -1.5873044391E+00 +4.42 -1.5837132098E+00 +4.43 -1.5801381951E+00 +4.44 -1.5765792679E+00 +4.45 -1.5730363413E+00 +4.46 -1.5695093087E+00 +4.47 -1.5659980634E+00 +4.48 -1.5625024987E+00 +4.49 -1.5590224884E+00 +4.5 -1.5555579458E+00 +4.51 -1.5521087730E+00 +4.52 -1.5486748682E+00 +4.53 -1.5452561298E+00 +4.54 -1.5418524452E+00 +4.55 -1.5384637135E+00 +4.56 -1.5350898499E+00 +4.57 -1.5317307575E+00 +4.58 -1.5283863393E+00 +4.59 -1.5250564978E+00 +4.6 -1.5217411153E+00 +4.61 -1.5184401207E+00 +4.62 -1.5151534216E+00 +4.63 -1.5118809255E+00 +4.64 -1.5086225400E+00 +4.65 -1.5053781613E+00 +4.66 -1.5021476999E+00 +4.67 -1.4989310782E+00 +4.68 -1.4957282082E+00 +4.69 -1.4925390016E+00 +4.7 -1.4893633704E+00 +4.71 -1.4862012066E+00 +4.72 -1.4830524446E+00 +4.73 -1.4799170014E+00 +4.74 -1.4767947930E+00 +4.75 -1.4736857355E+00 +4.76 -1.4705897384E+00 +4.77 -1.4675067108E+00 +4.78 -1.4644365871E+00 +4.79 -1.4613792870E+00 +4.8 -1.4583347306E+00 +4.81 -1.4553028376E+00 +4.82 -1.4522835159E+00 +4.83 -1.4492766914E+00 +4.84 -1.4462822960E+00 +4.85 -1.4433002532E+00 +4.86 -1.4403304868E+00 +4.87 -1.4373729203E+00 +4.88 -1.4344274613E+00 +4.89 -1.4314940487E+00 +4.9 -1.4285726134E+00 +4.91 -1.4256630825E+00 +4.92 -1.4227653832E+00 +4.93 -1.4198794427E+00 +4.94 -1.4170051697E+00 +4.95 -1.4141425126E+00 +4.96 -1.4112914024E+00 +4.97 -1.4084517699E+00 +4.98 -1.4056235455E+00 +4.99 -1.4028066588E+00 +5.0 -1.4000010240E+00 +5.01 -1.3972065925E+00 +5.02 -1.3944232980E+00 +5.03 -1.3916510745E+00 +5.04 -1.3888898557E+00 +5.05 -1.3861395741E+00 +5.06 -1.3834001481E+00 +5.07 -1.3806715315E+00 +5.08 -1.3779536612E+00 +5.09 -1.3752464741E+00 +5.1 -1.3725499072E+00 +5.11 -1.3698638970E+00 +5.12 -1.3671883633E+00 +5.13 -1.3645232633E+00 +5.14 -1.3618685368E+00 +5.15 -1.3592241238E+00 +5.16 -1.3565899640E+00 +5.17 -1.3539659973E+00 +5.18 -1.3513521482E+00 +5.19 -1.3487483721E+00 +5.2 -1.3461546139E+00 +5.21 -1.3435708162E+00 +5.22 -1.3409969216E+00 +5.23 -1.3384328728E+00 +5.24 -1.3358786008E+00 +5.25 -1.3333340559E+00 +5.26 -1.3307991892E+00 +5.27 -1.3282739460E+00 +5.28 -1.3257582715E+00 +5.29 -1.3232521111E+00 +5.3 -1.3207554034E+00 +5.31 -1.3182680912E+00 +5.32 -1.3157901326E+00 +5.33 -1.3133214753E+00 +5.34 -1.3108620674E+00 +5.35 -1.3084118565E+00 +5.36 -1.3059707901E+00 +5.37 -1.3035388010E+00 +5.38 -1.3011158551E+00 +5.39 -1.2987019027E+00 +5.4 -1.2962968941E+00 +5.41 -1.2939007796E+00 +5.42 -1.2915135095E+00 +5.43 -1.2891350244E+00 +5.44 -1.2867652793E+00 +5.45 -1.2844042331E+00 +5.46 -1.2820518385E+00 +5.47 -1.2797080480E+00 +5.48 -1.2773728144E+00 +5.49 -1.2750460886E+00 +5.5 -1.2727278110E+00 +5.51 -1.2704179504E+00 +5.52 -1.2681164617E+00 +5.53 -1.2658232996E+00 +5.54 -1.2635384189E+00 +5.55 -1.2612617746E+00 +5.56 -1.2589933135E+00 +5.57 -1.2567329919E+00 +5.58 -1.2544807741E+00 +5.59 -1.2522366172E+00 +5.6 -1.2500004780E+00 +5.61 -1.2477723135E+00 +5.62 -1.2455520805E+00 +5.63 -1.2433397235E+00 +5.64 -1.2411352113E+00 +5.65 -1.2389385051E+00 +5.66 -1.2367495638E+00 +5.67 -1.2345683465E+00 +5.68 -1.2323948119E+00 +5.69 -1.2302289177E+00 +5.7 -1.2280706113E+00 +5.71 -1.2259198665E+00 +5.72 -1.2237766443E+00 +5.73 -1.2216409055E+00 +5.74 -1.2195126109E+00 +5.75 -1.2173917213E+00 +5.76 -1.2152781938E+00 +5.77 -1.2131719831E+00 +5.78 -1.2110730624E+00 +5.79 -1.2089813942E+00 +5.8 -1.2068969412E+00 +5.81 -1.2048196662E+00 +5.82 -1.2027495316E+00 +5.83 -1.2006864953E+00 +5.84 -1.1986305166E+00 +5.85 -1.1965815689E+00 +5.86 -1.1945396164E+00 +5.87 -1.1925046237E+00 +5.88 -1.1904765552E+00 +5.89 -1.1884553752E+00 +5.9 -1.1864410430E+00 +5.91 -1.1844335204E+00 +5.92 -1.1824327817E+00 +5.93 -1.1804387931E+00 +5.94 -1.1784515207E+00 +5.95 -1.1764709306E+00 +5.96 -1.1744969887E+00 +5.97 -1.1725296572E+00 +5.98 -1.1705688974E+00 +5.99 -1.1686146860E+00 +6.0 -1.1666669908E+00 +6.01 -1.1647257794E+00 diff --git a/src/Ions.cc b/src/Ions.cc index 11c42f64..ea69a870 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -34,13 +34,13 @@ Timer ions_setup_tm("ions::setup"); const double ang2bohr = 1.8897269; // const double rmax = 8.0; -std::map Ions::map_species_ - = { { "H", 1 }, { "D", 1 }, { "Li", 3 }, { "Be", 4 }, { "B", 5 }, - { "C", 6 }, { "N", 7 }, { "O", 8 }, { "F", 9 }, { "Na", 11 }, - { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, { "P", 15 }, { "S", 16 }, - { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, { "Cr", 24 }, { "Mn", 25 }, - { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, { "Cu", 29 }, { "Zn", 30 }, - { "Ga", 31 }, { "Ge", 32 }, { "La", 57 }, { "Au", 79 } }; +std::map Ions::map_species_ = { { "H", 1 }, { "D", 1 }, + { "Li", 3 }, { "Be", 4 }, { "B", 5 }, { "C", 6 }, { "N", 7 }, { "O", 8 }, + { "F", 9 }, { "Na", 11 }, { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, + { "P", 15 }, { "S", 16 }, { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, + { "Cr", 24 }, { "Mn", 25 }, { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, + { "Cu", 29 }, { "Zn", 30 }, { "Ga", 31 }, { "Ge", 32 }, { "Br", 35 }, + { "La", 57 }, { "Au", 79 } }; int Ions::num_ions_ = -1; short Ions::max_num_proj_ = -1; From f298d43b5adbf4fcb88b5280af5e289180815034 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 11 Feb 2025 15:23:42 -0500 Subject: [PATCH 17/53] Clean up some HDFrestart functions (#303) --- src/HDFrestart.cc | 584 ++----------------------------------- src/HDFrestart.h | 13 +- src/Ions.cc | 27 +- src/LocalizationRegions.cc | 3 +- 4 files changed, 51 insertions(+), 576 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index ac856ff2..0322bfea 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1874,141 +1874,32 @@ void HDFrestart::printTimers(std::ostream& os) create_file_tm_.print(os); close_file_tm_.print(os); } -/* -int HDFrestart::writeRandomState(unsigned short int rand_state[3]) -{ - if( active_ ){ - // Create the data space for new datasets - hsize_t dims[1]={3}; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - if( dataspace_id<0 ){ - (*MPIdata::serr)<<"HDFrestart::writeRandomState(): H5Screate_simple -failed!!!"<& data) +int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - { - (*MPIdata::sout) << "HDFrestart::readAtomicNumbers()..." << std::endl; - } + (*MPIdata::sout) << "HDFrestart::readAtomicData()..." << std::endl; if (active_) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_numbers", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return 0; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_numbers", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -1; } int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- No numbers!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("No " + datasetname); return -1; } data.resize(dim); @@ -2017,17 +1908,13 @@ int HDFrestart::readAtomicNumbers(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -1; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dclose failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -1; } } // if active_ @@ -2037,126 +1924,8 @@ int HDFrestart::readAtomicNumbers(std::vector& data) return 0; } -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicIDs()..." << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- No IDs!!!" << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } - } - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicNLprojIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs()..." << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (!exists) return -1; - - hid_t dataset_id = H5Dopen2(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs() --- " - "H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- No IDs!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - return 0; -} - -int HDFrestart::readAtomicPositions(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { if (onpe0) (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; @@ -2165,26 +1934,21 @@ int HDFrestart::readAtomicPositions(std::vector& data) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Ionic_positions", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return -1; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Ionic_positions", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -2; } int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- No positions!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("readAtomicData() --- No " + datasetname); return -2; } data.resize(dim); @@ -2193,15 +1957,13 @@ int HDFrestart::readAtomicPositions(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -2; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -2; } } @@ -2218,59 +1980,12 @@ int HDFrestart::readOldCenterOnMesh(std::vector& data, int i) "points from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); + std::stringstream datasetstream; + datasetstream << "OldCenterOnMesh_" << i; - std::stringstream datasetstream; - datasetstream << "OldCenterOnMesh_" << i; + std::string datasetname = datasetstream.str(); - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; + return readAtomicData(datasetname, data); } int HDFrestart::readOldCenter(std::vector& data, int i) @@ -2279,256 +1994,16 @@ int HDFrestart::readOldCenter(std::vector& data, int i) (*MPIdata::sout) << "Read old localization centers from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); - - std::stringstream datasetstream; - datasetstream << "OldCenter_" << i; - - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readGidsList(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read list of gids from hdf5 file" << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - std::string datasetname = "GidsList"; - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(int); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- No GidsList!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readAtomicVelocities(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read atomic velocities from hdf5 file" - << std::endl; - - if (active_) - { - assert(file_id_ >= 0); + std::stringstream datasetstream; + datasetstream << "OldCenter_" << i; - htri_t exists = H5Lexists(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (exists) - { - - // Open an existing dataset - hid_t dataset_id - = H5Dopen2(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (dataset_id < 0) - { - std::cerr << "HDFrestart::readAtomicVelocities(), " - "H5Dopen failed->no velocities read" - << std::endl; - data.clear(); - return -1; - } - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, &data[0]); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dread failed!!!"); - return -2; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dclose failed!!!"); - return -2; - } - } - } + std::string datasetname = datasetstream.str(); - if (gather_data_x_) gatherDataXdir(data); - - return 0; + return readAtomicData(datasetname, data); } -int HDFrestart::readLockedAtomNames(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames()..." << std::endl; - - std::vector buffer; - short name_length = 7; // default, value used before February 2016 - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (!exists) return 0; - - hid_t dataset_id = H5Dopen2(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames(), " - "H5Dopen failed->no locked atoms read" - << std::endl; - return -1; - } - - std::string attname("String_Length"); - htri_t existsA = H5Aexists(dataset_id, attname.c_str()); - if (existsA) - { - hid_t attribute_id = H5Aopen_name(dataset_id, attname.c_str()); - herr_t status = H5Aread(attribute_id, H5T_NATIVE_INT, &name_length); - // check validity of data just read - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Aread failed!!!"); - return -1; - } - } - - int dim = (int)H5Dget_storage_size(dataset_id) / name_length; - - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), dataset size=" << dim - << std::endl; - - if (dim == 0) return 0; - - buffer.resize(dim * name_length); - - // create type for std::strings of length IonData_MaxStrLength - hid_t strtype = H5Tcopy(H5T_C_S1); - H5Tset_size(strtype, name_length); - herr_t status = H5Dread( - dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buffer[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), H5Dread failed!!!" - << std::endl; - return -1; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - - if (gather_data_x_) gatherDataXdir(buffer); - - data.clear(); - for (unsigned short i = 0; i < buffer.size(); i += name_length) - { - std::string t(&buffer[i], name_length); - assert(t.size() > 0); - - stripLeadingAndTrailingBlanks(t); - - assert(t.size() > 0); - data.push_back(t); - } - - return 0; -} - -int HDFrestart::readAtomicNames(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -2541,11 +2016,12 @@ int HDFrestart::readAtomicNames(std::vector& data) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_names", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (exists) { // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_names", H5P_DEFAULT); + hid_t dataset_id + = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { MGMOL_HDFRESTART_FAIL("H5Dopen2 failed!!!"); @@ -2577,7 +2053,7 @@ int HDFrestart::readAtomicNames(std::vector& data) int dim = (int)H5Dget_storage_size(dataset_id) / name_length; if (dim == 0) { - if (onpe0) MGMOL_HDFRESTART_FAIL("No names!!!"); + MGMOL_HDFRESTART_FAIL("No names!!!"); return -1; } diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 72c5c1f5..1ed2d080 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -260,19 +260,12 @@ class HDFrestart template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); - // int writeRandomState(unsigned short int rand_state[3]); - // int readRandomState(unsigned short* rand_state); - int readAtomicIDs(std::vector& data); - int readAtomicNLprojIDs(std::vector& data); - int readAtomicNumbers(std::vector& data); - int readAtomicNames(std::vector& data); - int readAtomicPositions(std::vector& data); - int readAtomicVelocities(std::vector& data); - int readLockedAtomNames(std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); int readRestartRandomStates(std::vector& data); int readOldCenter(std::vector& data, int i); int readOldCenterOnMesh(std::vector& data, int i); - int readGidsList(std::vector& data); void addDateToFilename(); void addMDTime2File(const float run_time); diff --git a/src/Ions.cc b/src/Ions.cc index ea69a870..913a0f55 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -700,13 +700,12 @@ void Ions::readLockedAtomNames(HDFrestart& h5f_file) if (dim == 0) return; std::vector data; - h5f_file.readLockedAtomNames(data); + std::string datasetname("/LockedAtomsNames"); + h5f_file.readAtomicData(datasetname, data); - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) + for (auto& i : data) { - lockAtom(*i); + lockAtom(i); } } @@ -874,13 +873,17 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) setupListIonsBoundaries(rmax); std::vector at_numbers; - h5_file.readAtomicNumbers(at_numbers); + std::string datasetname("/Atomic_numbers"); + h5_file.readAtomicData(datasetname, at_numbers); std::vector at_indexes; - int nidxs = h5_file.readAtomicIDs(at_indexes); + std::string datasetname_indexes("/Atomic_IDs"); + int nidxs = h5_file.readAtomicData(datasetname_indexes, at_indexes); std::vector at_nlprojIds; - int npids = h5_file.readAtomicNLprojIDs(at_nlprojIds); + std::string datasetname_nlprojIds("/AtomicNLproj_IDs"); + int npids = h5_file.readAtomicData(datasetname_nlprojIds, at_nlprojIds); std::vector at_names; - h5_file.readAtomicNames(at_names); + std::string datasetname_names("/Atomic_names"); + h5_file.readAtomicData(datasetname_names, at_names); if (onpe0 && ct.verbose > 2) { std::cout << "HDF file: at nb=" << at_numbers.size() << std::endl; @@ -967,7 +970,8 @@ void Ions::readRestartPositions(HDFrestart& h5_file) (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; std::vector data; - h5_file.readAtomicPositions(data); + std::string datasetname("/Ionic_positions"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) @@ -1142,7 +1146,8 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) << std::endl; std::vector data; - h5_file.readAtomicVelocities(data); + std::string datasetname("/Ionic_velocities"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 99cda54e..60f28301 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -1726,7 +1726,8 @@ void LocalizationRegions::writeOldCenter(HDFrestart& h5f_file, int i) void LocalizationRegions::setupOldCenters(HDFrestart& h5_file) { vector gids; - h5_file.readGidsList(gids); + std::string datasetname("GidsList"); + h5_file.readAtomicData(datasetname, gids); map gids_map; for (unsigned int i = 0; i < gids.size(); i++) From c690bc5bb8998a1202ebc22a16ebb3ffb80346a6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Feb 2025 07:19:17 -0500 Subject: [PATCH 18/53] Fix and test restart single hdf5 file (#305) --- src/Control.h | 2 + src/HDFrestart.cc | 28 ++++++++--- src/Ions.cc | 20 ++++++-- src/restart.cc | 2 +- tests/CMakeLists.txt | 8 +++ tests/DMandEnergyAndForces/test.py | 3 ++ tests/HDF5single/h2o.xyz | 6 +++ tests/HDF5single/md.cfg | 34 +++++++++++++ tests/HDF5single/mgmol.cfg | 31 ++++++++++++ tests/HDF5single/test.py | 75 ++++++++++++++++++++++++++++ tests/RestartEnergyAndForces/test.py | 5 +- tests/WFEnergyAndForces/test.py | 3 ++ 12 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 tests/HDF5single/h2o.xyz create mode 100644 tests/HDF5single/md.cfg create mode 100644 tests/HDF5single/mgmol.cfg create mode 100755 tests/HDF5single/test.py diff --git a/src/Control.h b/src/Control.h index b3401c1d..bd4a7d96 100644 --- a/src/Control.h +++ b/src/Control.h @@ -371,6 +371,8 @@ class Control // 10 or larger means CG, otherwise MG V-cycles bool MGPoissonSolver() { return (diel_flag_ / 10 == 0); } + bool LangevinThermostat() { return (thermostat_type == 1); } + // // data // diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 0322bfea..0fac518e 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -457,7 +457,8 @@ HDFrestart::HDFrestart(const std::string& filename, const pb::PEenv& pes, verbosity_ = 0; closed_ = false; - //(*MPIdata::sout)<<"HDFrestart::HDFrestart(), filename="<& data) // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), -1), data.end()); + } + return 0; } @@ -1928,7 +1934,7 @@ int HDFrestart::readAtomicData( std::string datasetname, std::vector& data) { if (onpe0) - (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + (*MPIdata::sout) << "Read atomic data from hdf5 file" << std::endl; if (active_) { @@ -1968,6 +1974,10 @@ int HDFrestart::readAtomicData( } } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); + } if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2094,6 +2104,11 @@ int HDFrestart::readAtomicData( data.push_back(t); } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), ""), data.end()); + } + return 0; } @@ -2128,7 +2143,6 @@ int HDFrestart::readRestartRandomStates(std::vector& data) dim = (int)H5Dget_storage_size(dataset_id) / sizeof(unsigned short); } - if (dim > 0) { data.resize(dim); @@ -2148,12 +2162,10 @@ int HDFrestart::readRestartRandomStates(std::vector& data) } if (!data.empty()) - if (data[0] != data[0]) + if (std::isnan(data[0])) { MGMOL_HDFRESTART_FAIL( - "ERROR: HDFrestart::readRestartRandomStates() " - "--- data[0]=" - << data[0]); + "readRestartRandomStates() is NaN"); return -2; } } diff --git a/src/Ions.cc b/src/Ions.cc index 913a0f55..e93d82ea 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -911,8 +911,10 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); - mmpi.allreduce(&num_ions_, 1, MPI_SUM); - + if (!h5_file.useHdf5p()) + { + mmpi.allreduce(&num_ions_, 1, MPI_SUM); + } if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ @@ -947,9 +949,21 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) } readRestartPositions(h5_file); readRestartVelocities(h5_file); - readRestartRandomStates(h5_file); + if (ct.LangevinThermostat()) readRestartRandomStates(h5_file); readLockedAtomNames(h5_file); + // remove atoms from local list if not local + for (std::vector::iterator it = local_ions_.begin(); + it != local_ions_.end();) + { + double p[3]; + (*it)->getPosition(p); + if (!inLocalIons(p[0], p[1], p[2])) + it = local_ions_.erase(it); + else + ++it; + } + // rescale all velocities by factor specified in input rescaleVelocities(ct.VelocityScalingFactor()); diff --git a/src/restart.cc b/src/restart.cc index 2fa76ecb..247cca06 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -117,7 +117,7 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicIDs(h5f_file); ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); - ions.writeRandomStates(h5f_file); + if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b29c1aad..baf1c451 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -504,6 +504,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/DMandEnergyAndForces/test.py b/tests/DMandEnergyAndForces/test.py index c36f0767..b21e6e72 100755 --- a/tests/DMandEnergyAndForces/test.py +++ b/tests/DMandEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test DMandEnergyAndForces...") @@ -33,6 +34,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: diff --git a/tests/HDF5single/h2o.xyz b/tests/HDF5single/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/HDF5single/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg new file mode 100644 index 00000000..1ff2adab --- /dev/null +++ b/tests/HDF5single/md.cfg @@ -0,0 +1,34 @@ +verbosity=3 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +input_type=single_file +output_level=4 +output_filename=WF_MD +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg new file mode 100644 index 00000000..4dba942a --- /dev/null +++ b/tests/HDF5single/mgmol.cfg @@ -0,0 +1,31 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py new file mode 100755 index 00000000..080ee0ba --- /dev/null +++ b/tests/HDF5single/test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +os.remove('WF') + +print("Check energy conservation...") +tol = 1.e-4 +energy = 0. +count = 0 +for line in lines: + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +if count<4: + print("ERROR needs 4 energy values for checking conservation!") + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index 349434e8..b62d39f8 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test RestartEnergyAndForces...") @@ -54,14 +55,14 @@ ref_energy=energy break -#sys.exit(0) - #run test command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + test_energy=1.e18 for line in lines: if line.count(b'%%'): diff --git a/tests/WFEnergyAndForces/test.py b/tests/WFEnergyAndForces/test.py index 45420ddf..6cac8c93 100755 --- a/tests/WFEnergyAndForces/test.py +++ b/tests/WFEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test WFEnergyAndForces...") @@ -37,6 +38,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: From 9f1fc20a63eb19e229a9f502b30d3271b264a3fc Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Feb 2025 07:19:37 -0500 Subject: [PATCH 19/53] Save Hartree potential for write in restart file (#306) --- src/Potentials.cc | 16 ++++++++++++---- src/Potentials.h | 23 +++++++++++++++++------ src/md.cc | 3 +++ src/restart.cc | 8 ++++++++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/Potentials.cc b/src/Potentials.cc index bed005db..abe277f1 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -75,10 +75,13 @@ Potentials::Potentials() dv_.resize(size_); - memset(&vepsilon_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&vh_rho_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&vxc_rho_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&v_ext_[0], 0, size_ * sizeof(POTDTYPE)); + vh_rho_backup_.resize(size_); + + memset(vepsilon_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vh_rho_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vxc_rho_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(v_ext_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vh_rho_backup_.data(), 0, size_ * sizeof(POTDTYPE)); #ifdef HAVE_TRICUBIC vext_tricubic_ = NULL; @@ -596,6 +599,11 @@ void Potentials::axpVcomp(POTDTYPE* v, const double alpha) LinearAlgebraUtils::MPaxpy(size_, alpha, &v_comp_[0], v); } +void Potentials::backupVh() +{ + memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); +} + void Potentials::initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp) { diff --git a/src/Potentials.h b/src/Potentials.h index 970cf95d..71819a75 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -76,6 +76,11 @@ class Potentials std::vector dv_; + /*! + * Backpup copy of Hartree potential to save previous state + */ + std::vector vh_rho_backup_; + int itindex_vxc_; int itindex_vh_; @@ -138,14 +143,15 @@ class Potentials double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } - POTDTYPE* vtot() { return &vtot_[0]; } - POTDTYPE* vh_rho() { return &vh_rho_[0]; } - RHODTYPE* rho_comp() { return &rho_comp_[0]; } + POTDTYPE* vtot() { return vtot_.data(); } + POTDTYPE* vh_rho() { return vh_rho_.data(); } + RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return &v_nuc_[0]; } - POTDTYPE* vext() { return &v_ext_[0]; } - POTDTYPE* vepsilon() { return &vepsilon_[0]; } + POTDTYPE* vnuc() { return v_nuc_.data(); } + POTDTYPE* vext() { return v_ext_.data(); } + POTDTYPE* vepsilon() { return vepsilon_.data(); } + POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -196,6 +202,11 @@ class Potentials void initBackground(Ions& ions); void addBackgroundToRhoComp(); + /*! + * Save current Hartree potential into backup array + */ + void backupVh(); + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/md.cc b/src/md.cc index b50618c8..dfb4dfe6 100644 --- a/src/md.cc +++ b/src/md.cc @@ -54,6 +54,9 @@ void MGmol::moveVnuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // save Hartree potential internally + pot.backupVh(); + // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); initNuc(ions); diff --git a/src/restart.cc b/src/restart.cc index 247cca06..30bdad73 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -141,6 +141,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, pot.vh_rho(), "Hartree", &ll[0], &origin[0]); if (ierr < 0) return ierr; + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + // Write if (ct.diel) { From 8a1cf0530f2a2246dc62fca1db5b145919330101 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 19 Feb 2025 17:11:48 -0500 Subject: [PATCH 20/53] Speed-up recently added tests (#309) * simply use a smaller domain/mesh --- tests/DMandEnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/EnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/RestartEnergyAndForces/h2o.xyz | 8 ++++---- tests/RestartEnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/RestartEnergyAndForces/restart.cfg | 18 +++++++++--------- 5 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tests/DMandEnergyAndForces/mgmol.cfg b/tests/DMandEnergyAndForces/mgmol.cfg index 5278f40b..e7712706 100644 --- a/tests/DMandEnergyAndForces/mgmol.cfg +++ b/tests/DMandEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=LDA FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/EnergyAndForces/mgmol.cfg b/tests/EnergyAndForces/mgmol.cfg index cf23889e..d1299db9 100644 --- a/tests/EnergyAndForces/mgmol.cfg +++ b/tests/EnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=1 xcFunctional=LDA FDtype=Mehrstellen [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz index cdc906f6..d5171c8b 100644 --- a/tests/RestartEnergyAndForces/h2o.xyz +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -1,6 +1,6 @@ 3 -https://pubchem.ncbi.nlm.nih.gov/compound/Water -O 2.5369 -0.1550 0.0 -H 3.0739 0.1550 0.0 -H 2.0000 0.1550 0.0 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg index e590f810..b96f3d9c 100644 --- a/tests/RestartEnergyAndForces/mgmol.cfg +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg index 99bc77d8..280778e9 100644 --- a/tests/RestartEnergyAndForces/restart.cfg +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 From 8a2687965d538179129ea9aed22d93129a7ac1c6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 11:23:03 -0500 Subject: [PATCH 21/53] Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing --- src/Electrostatic.cc | 4 +- src/Electrostatic.h | 2 +- src/Forces.cc | 3 +- src/MGmol.cc | 18 +--- src/Poisson.h | 5 +- src/Potentials.cc | 203 ++++++++++++++++++++++++++++++------------- src/Potentials.h | 25 +++--- src/restart.cc | 49 +---------- 8 files changed, 165 insertions(+), 144 deletions(-) diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 6ac4fef3..aba09d07 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -65,7 +65,7 @@ Electrostatic::~Electrostatic() if (grhoc_ != nullptr) delete grhoc_; } -void Electrostatic::setupInitialVh(const POTDTYPE* const vh_init) +void Electrostatic::setupInitialVh(const std::vector& vh_init) { poisson_solver_->set_vh(vh_init); @@ -153,7 +153,7 @@ void Electrostatic::setupPB( // initialize vh with last trial solution pb::GridFunc gf_vh(*pbGrid_, bc_[0], bc_[1], bc_[2]); - gf_vh.assign(pot.vh_rho()); + gf_vh.assign((pot.vh_rho()).data()); poisson_solver_->set_vh(gf_vh); } diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 072ce0cc..9beee899 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -58,7 +58,7 @@ class Electrostatic template void computeVh(const pb::GridFunc& vhinit, const Ions& ions, Rho& rho, Potentials& pot); - void setupInitialVh(const POTDTYPE* const); + void setupInitialVh(const std::vector&); void setupInitialVh(const pb::GridFunc&); template void computeVhRho(Rho& rho); diff --git a/src/Forces.cc b/src/Forces.cc index 9ab19f6a..04e6ffd7 100644 --- a/src/Forces.cc +++ b/src/Forces.cc @@ -272,9 +272,10 @@ void Forces::get_loc_proj(RHODTYPE* rho, const int numpt = mymesh->numpt(); Potentials& pot = hamiltonian_->potential(); + const std::vector& vh_rho(pot.vh_rho()); for (int idx = 0; idx < numpt; idx++) { - const double vhrho = pot.vh_rho(idx); + const double vhrho = vh_rho[idx]; for (short dir = 0; dir < 3; dir++) { double* lproj = &(loc_proj[dir * NPTS]); diff --git a/src/MGmol.cc b/src/MGmol.cc index 681efacc..649f2c39 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -319,8 +319,6 @@ int MGmol::initial() mmpi.barrier(); if (ct.verbose > 0) current_orbitals_->printChromaticNumber(os_); - pot.initBackground(*ions_); - // Random initialization of the wavefunctions if (ct.restart_info <= 2) { @@ -834,23 +832,9 @@ void MGmol::initNuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // initialize poentials based on ionic positions and their species pot.initialize(ions); - // Check compensating charges - double comp_rho = getCharge(pot.rho_comp()); - - if (onpe0 && ct.verbose > 1) - { - os_ << std::setprecision(8) << std::fixed - << " Charge of rhoc: " << comp_rho << std::endl; - } - -#if 1 - pot.rescaleRhoComp(); -#endif - - pot.addBackgroundToRhoComp(); - electrostat_->setupRhoc(pot.rho_comp()); if (onpe0 && ct.verbose > 3) os_ << " initNuc done" << std::endl; diff --git a/src/Poisson.h b/src/Poisson.h index 87107542..ba5daa39 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -71,7 +71,10 @@ class Poisson : public PoissonInterface virtual void set_rhod(pb::GridFunc* /*rhod*/){}; void set_vh(const pb::GridFunc& vh) { (*vh_) = vh; }; - void set_vh(const POTDTYPE* const vh) { vh_->assign(vh, 'd'); }; + void set_vh(const std::vector& vh) + { + vh_->assign(vh.data(), 'd'); + }; void resetVh() { vh_->resetData(); } void set_vepsilon(const POTDTYPE* const vepsilon) { diff --git a/src/Potentials.cc b/src/Potentials.cc index abe277f1..a0ff3f1a 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -25,7 +25,6 @@ #include "mputils.h" #include -using namespace std; // unit conversion factor Ha -> Ry const double ha2ry = 2.; @@ -92,7 +91,7 @@ void Potentials::initWithVnuc() { assert(size_ > 0); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::initWithVnuc()" << endl; + (*MPIdata::sout) << "Potentials::initWithVnuc()" << std::endl; itindex_vxc_ = 0; itindex_vh_ = 0; int ione = 1; @@ -122,7 +121,8 @@ double Potentials::min() const return vmin; } -void Potentials::evalNormDeltaVtotRho(const vector>& rho) +void Potentials::evalNormDeltaVtotRho( + const std::vector>& rho) { Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); @@ -141,14 +141,14 @@ void Potentials::evalNormDeltaVtotRho(const vector>& rho) mmpi.allreduce(&scf_dvrho_, 1, MPI_SUM); } -double Potentials::update(const vector>& rho) +double Potentials::update(const std::vector>& rho) { assert(itindex_vxc_ >= 0); assert(itindex_vh_ >= 0); assert(itindex_vxc_ == itindex_vh_); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::update(rho)" << endl; + (*MPIdata::sout) << "Potentials::update(rho)" << std::endl; int ione = 1; Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -185,7 +185,7 @@ double Potentials::update(const vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; + std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; MPI_Abort(myPEenv.comm(), 2); } dvdot = sum; @@ -202,7 +202,7 @@ void Potentials::update(const double mix) assert(itindex_vxc_ == itindex_vh_); #ifdef DEBUG - if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << endl; + if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << std::endl; #endif // int ione=1; double potmix = mix; @@ -210,13 +210,13 @@ void Potentials::update(const double mix) size_, potmix, &dv_[0], &vtot_[0]); } -double Potentials::delta_v(const vector>& rho) +double Potentials::delta_v(const std::vector>& rho) { assert(itindex_vxc_ == itindex_vh_); assert(size_ > 0); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::delta_v()" << endl; + (*MPIdata::sout) << "Potentials::delta_v()" << std::endl; int ione = 1; Mesh* mymesh = Mesh::instance(); @@ -253,7 +253,7 @@ double Potentials::delta_v(const vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; + std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; MPI_Abort(myPEenv.comm(), 2); } dvdot = sum; @@ -266,7 +266,7 @@ double Potentials::delta_v(const vector>& rho) } // in Ry -void Potentials::getVofRho(vector& vrho) const +void Potentials::getVofRho(std::vector& vrho) const { vrho.resize(size_); int ione = 1; @@ -283,7 +283,7 @@ void Potentials::getVofRho(vector& vrho) const // type: // 2->text // 3->binary -void Potentials::readExternalPot(const string filename, const char type) +void Potentials::readExternalPot(const std::string filename, const char type) { assert(type == 2 || type == 3); @@ -304,27 +304,27 @@ void Potentials::readExternalPot(const string filename, const char type) { from->seekg(0, ios::end); const int length = from->tellg(); - (*MPIdata::sout) << "Length file = " << length << endl; + (*MPIdata::sout) << "Length file = " << length << std::endl; from->seekg(0, ios::beg); if (length <= 0) { (*MPIdata::serr) - << "ERROR Potential: file length <=0!!!!" << endl; + << "ERROR Potential: file length <=0!!!!" << std::endl; mmpi.abort(); } } } if (!from) { - (*MPIdata::serr) << " Cannot open file " << filename << endl; + (*MPIdata::serr) << " Cannot open file " << filename << std::endl; mmpi.abort(); } if (onpe0) { (*MPIdata::sout) << "Potentials::read_ExternalPot(), filename=" - << filename << endl; - if (type == 2) (*MPIdata::sout) << "text file..." << endl; - if (type == 3) (*MPIdata::sout) << "binary file..." << endl; + << filename << std::endl; + if (type == 2) (*MPIdata::sout) << "text file..." << std::endl; + if (type == 3) (*MPIdata::sout) << "binary file..." << std::endl; } // read origin and end of cell (to check compatibility) @@ -348,29 +348,32 @@ void Potentials::readExternalPot(const string filename, const char type) if (onpe0) for (short d = 0; d < 3; d++) { - (*MPIdata::sout) << setprecision(8); + (*MPIdata::sout) << std::setprecision(8); if (fabs(origin[d] - mygrid.origin(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell origin in direction " - << d << endl; - (*MPIdata::serr) << "Potential origin=" << origin[d] << endl; - (*MPIdata::serr) << "MGmol origin=" << mygrid.origin(d) << endl; + << d << std::endl; + (*MPIdata::serr) + << "Potential origin=" << origin[d] << std::endl; + (*MPIdata::serr) + << "MGmol origin=" << mygrid.origin(d) << std::endl; (*MPIdata::serr) << "Difference=" << fabs(origin[d] - mygrid.origin(d)) - << endl; + << std::endl; mmpi.abort(); } if (fabs(ll[d] - mygrid.ll(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell " "dimension in direction " - << d << endl; - (*MPIdata::serr) << "Potential cell end=" << end[d] << endl; + << d << std::endl; + (*MPIdata::serr) + << "Potential cell end=" << end[d] << std::endl; (*MPIdata::serr) - << "Potential cell dimension=" << ll[d] << endl; + << "Potential cell dimension=" << ll[d] << std::endl; (*MPIdata::serr) - << "MGmol cell dimension=" << mygrid.ll(d) << endl; + << "MGmol cell dimension=" << mygrid.ll(d) << std::endl; mmpi.abort(); } } @@ -392,9 +395,9 @@ void Potentials::readExternalPot(const string filename, const char type) if (nxyz[i] != gdim_[i]) { (*MPIdata::serr) << "Potentials::read_ExternalPot(): dimension " - << i << " incompatible with Grid!!!" << endl; + << i << " incompatible with Grid!!!" << std::endl; (*MPIdata::serr) - << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << endl; + << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << std::endl; mmpi.abort(); } @@ -454,7 +457,7 @@ void Potentials::readExternalPot(const string filename, const char type) if (type == 3) { - vector tmp(dim_[2]); + std::vector tmp(dim_[2]); for (int i = 0; i < dim_[0]; i++) { // advance (start-file_index) positions @@ -512,7 +515,8 @@ void Potentials::getGradVext(const double r[3], double dfdr[3]) const vext_tricubic_->getGradient(r, dfdr, comm); } -void Potentials::getValVext(const vector& r, vector& val) const +void Potentials::getValVext( + const std::vector& r, std::vector& val) const { assert(vext_tricubic_ != NULL); @@ -522,21 +526,22 @@ void Potentials::getValVext(const vector& r, vector& val) const } #endif -void Potentials::readAll(vector& sp) +void Potentials::readAll(std::vector& sp) { assert(sp.size() <= pot_filenames_.size()); if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::readAll() for " << pot_types_.size() - << " potentials" << endl; + << " potentials" << std::endl; Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); double hmin = mygrid.hmin(); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "hmin= " << hmin << endl; + (*MPIdata::sout) << "hmin= " << hmin << std::endl; - vector::const_iterator it_filename = pot_filenames_.begin(); - int isp = 0; + std::vector::const_iterator it_filename + = pot_filenames_.begin(); + int isp = 0; while (it_filename != pot_filenames_.end()) { if (pot_types_[isp] == 'n' || pot_types_[isp] == 's' @@ -556,7 +561,7 @@ void Potentials::readAll(vector& sp) #else (*MPIdata::sout) << "ERROR: cannot read external potential " - << " -> need to compile with Tricubic library" << endl; + << " -> need to compile with Tricubic library" << std::endl; #endif } it_filename++; @@ -594,11 +599,6 @@ void Potentials::axpVcompToVh(const double alpha) size_, alpha, &v_comp_[0], &vh_rho_[0]); } -void Potentials::axpVcomp(POTDTYPE* v, const double alpha) -{ - LinearAlgebraUtils::MPaxpy(size_, alpha, &v_comp_[0], v); -} - void Potentials::backupVh() { memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); @@ -792,9 +792,12 @@ void Potentials::initialize(Ions& ions) const pb::Grid& mygrid = mymesh->grid(); const int numpt = mygrid.size(); - memset(&v_comp_[0], 0, numpt * sizeof(POTDTYPE)); - memset(&rho_comp_[0], 0, numpt * sizeof(RHODTYPE)); - memset(&v_nuc_[0], 0, numpt * sizeof(RHODTYPE)); + memset(v_comp_.data(), 0, numpt * sizeof(POTDTYPE)); + memset(rho_comp_.data(), 0, numpt * sizeof(RHODTYPE)); + memset(v_nuc_.data(), 0, numpt * sizeof(RHODTYPE)); + + // Count up the total ionic charge + ionic_charge_ = ions.computeIonicCharge(); char flag_filter = pot_type(0); @@ -805,6 +808,7 @@ void Potentials::initialize(Ions& ions) Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + // initialize rho_comp_, v_comp_, v_nuc_ if (flag_filter == 's') { const int sampleRate = 3; @@ -820,6 +824,13 @@ void Potentials::initialize(Ions& ions) initializeRadialDataOnMesh(position, sp); } } + + // rescale rho_comp_ due to finite mesh effects + rescaleRhoComp(); + + initBackground(); + + addBackgroundToRhoComp(); } void Potentials::rescaleRhoComp() @@ -831,24 +842,33 @@ void Potentials::rescaleRhoComp() const pb::Grid& mygrid = mymesh->grid(); // Check compensating charges - double comp_rho = getCharge(&rho_comp_[0]); + double comp_rho = getCharge(rho_comp_.data()); + if (onpe0 && ct.verbose > 1) + { + std::cout << std::setprecision(8) << std::fixed + << " Charge of rhoc: " << comp_rho << std::endl; + } if (onpe0 && ct.verbose > 1) { - cout << " Rescaling rhoc" << endl; + std::cout << " Rescaling rhoc" << std::endl; } + + // rescale rho_comp_ (initialized by sampling on mesh) + // so that its integral exactly matches ionic_charge_ if (ionic_charge_ > 0.) { const int numpt = mygrid.size(); double t = ionic_charge_ / comp_rho; - LinearAlgebraUtils::MPscal(numpt, t, &rho_comp_[0]); + LinearAlgebraUtils::MPscal( + numpt, t, rho_comp_.data()); // Check new compensating charges - comp_rho = getCharge(&rho_comp_[0]); + comp_rho = getCharge(rho_comp_.data()); } if (onpe0 && ct.verbose > 1) - cout << " Rescaled compensating charges: " << setprecision(8) << fixed - << comp_rho << endl; + std::cout << " Rescaled compensating charges: " << std::setprecision(8) + << std::fixed << comp_rho << std::endl; if (comp_rho < 0.) mmpi.abort(); } @@ -868,26 +888,23 @@ void Potentials::addBackgroundToRhoComp() { if (onpe0) { - cout << setprecision(12) << scientific - << "Add background charge " << background << " to rhoc " - << endl; + std::cout << std::setprecision(12) << std::scientific + << "Add background charge " << background + << " to rhoc " << std::endl; } for (int i = 0; i < numpt; i++) rho_comp_[i] += background; // Check new compensating charges - getCharge(&rho_comp_[0]); + getCharge(rho_comp_.data()); } } } -void Potentials::initBackground(Ions& ions) +void Potentials::initBackground() { Control& ct = *(Control::instance()); - // Count up the total ionic charge - ionic_charge_ = ions.computeIonicCharge(); - // calculation the compensating background charge // for charged supercell calculations background_charge_ = 0.; @@ -898,14 +915,76 @@ void Potentials::initBackground(Ions& ions) } if (onpe0 && ct.verbose > 0) { - cout << "N electrons= " << ct.getNel() << endl; - cout << "ionic charge= " << ionic_charge_ << endl; - cout << "background charge=" << background_charge_ << endl; + std::cout << "N electrons= " << ct.getNel() << std::endl; + std::cout << "ionic charge= " << ionic_charge_ << std::endl; + std::cout << "background charge=" << background_charge_ << std::endl; } if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; } +int Potentials::read(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + // Read total potential + h5f_file.read_1func_hdf5(vtot_.data(), "Vtotal"); + + // Read the hartree potential + h5f_file.read_1func_hdf5(vh_rho_.data(), "Hartree"); + + // Read dielectric potential + if (ct.diel) + { + h5f_file.read_1func_hdf5(vepsilon_.data(), "VDielectric"); + } + + return 0; +} + +int Potentials::write(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double ll[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + double origin[3] = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + + // Write total potential + int ierr + = h5f_file.write_1func_hdf5(vtot_.data(), "Vtotal", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + // Write the hartree potential + ierr = h5f_file.write_1func_hdf5( + vh_rho_.data(), "Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + vh_rho_backup_.data(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + + // Write + if (ct.diel) + { + ierr = h5f_file.write_1func_hdf5( + vepsilon_.data(), "VDielectric", &ll[0], &origin[0]); + } + if (ierr < 0) return ierr; + + // Write external potential + ierr = h5f_file.write_1func_hdf5(v_ext_.data(), "Vext", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + return ierr; +} + template void Potentials::setVxc( const double* const vxc, const int iterativeIndex); template void Potentials::setVxc( diff --git a/src/Potentials.h b/src/Potentials.h index 71819a75..cb0f8e05 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -10,6 +10,7 @@ #ifndef MGMOL_POTENTIALS_H #define MGMOL_POTENTIALS_H +#include "HDFrestart.h" #include "Rho.h" #include "TriCubic.h" @@ -100,6 +101,12 @@ class Potentials void initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp); + void rescaleRhoComp(); + + void addBackgroundToRhoComp(); + + void initBackground(); + public: Potentials(); @@ -139,27 +146,22 @@ class Potentials void turnOnDiel() { diel_ = true; } + int write(HDFrestart& h5f_file); + int read(HDFrestart& h5f_file); + int size() const { return size_; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return vtot_.data(); } - POTDTYPE* vh_rho() { return vh_rho_.data(); } RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return v_nuc_.data(); } - POTDTYPE* vext() { return v_ext_.data(); } + const std::vector& vh_rho() const { return vh_rho_; } + POTDTYPE* vepsilon() { return vepsilon_.data(); } - POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); - void axpVcomp(POTDTYPE* v, const double alpha); - - POTDTYPE vtot(const int i) { return vtot_[i]; } - POTDTYPE vh_rho(const int i) { return vh_rho_[i]; } - POTDTYPE vxc_rho(const int i) { return vxc_rho_[i]; } - POTDTYPE vepsilon(const int i) { return vepsilon_[i]; } bool diel() const { return diel_; } @@ -198,9 +200,6 @@ class Potentials void setVh(const pb::GridFunc& vh, const int iterativeIndex); void initialize(Ions& ions); - void rescaleRhoComp(); - void initBackground(Ions& ions); - void addBackgroundToRhoComp(); /*! * Save current Hartree potential into backup array diff --git a/src/restart.cc b/src/restart.cc index 30bdad73..ed7f478c 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -43,27 +43,11 @@ int MGmol::read_rho_and_pot_hdf5( os_ << "Try to read density and potentials" << std::endl; Potentials& pot = hamiltonian_->potential(); + pot.read(file); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - POTDTYPE* tmp = new POTDTYPE[mygrid.size()]; - - // Read total potential - file.read_1func_hdf5(pot.vtot(), "Vtotal"); - - // Read the hartree potential - file.read_1func_hdf5(tmp, "Hartree"); - pot.setVh(tmp, 0); - - // Read dielectric potential - if (ct.diel) - { - file.read_1func_hdf5(pot.vepsilon(), "VDielectric"); - } // Read the Density rho.readRestart(file); - delete[] tmp; return 0; } @@ -131,41 +115,12 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, if (ct.out_restart_info > 1) { - // Write total potential - int ierr = h5f_file.write_1func_hdf5( - pot.vtot(), "Vtotal", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - // Write the hartree potential - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho(), "Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - if (ct.AtomsDynamic() == AtomsDynamicType::MD) - { - // Write hartree potential before extrapolation - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - } - - // Write - if (ct.diel) - { - ierr = h5f_file.write_1func_hdf5( - pot.vepsilon(), "VDielectric", &ll[0], &origin[0]); - } - if (ierr < 0) return ierr; + int ierr = pot.write(h5f_file); // Write the Density ierr = h5f_file.write_1func_hdf5( &rho[0][0], "Density", &ll[0], &origin[0]); if (ierr < 0) return ierr; - - // Write external potential - ierr - = h5f_file.write_1func_hdf5(pot.vext(), "Vext", &ll[0], &origin[0]); - if (ierr < 0) return ierr; } // Write wavefunctions and old centers. From 59b79e75b3f538c1f2ec6a76dc1806fa4d4df059 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 12:54:11 -0500 Subject: [PATCH 22/53] Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry --- src/MGmol.cc | 1 - src/md.cc | 9 ++++++--- tests/MD_D72/test.py | 2 +- tests/MD_MVP/md.cfg | 2 +- tests/MD_MVP/test.py | 8 ++++---- tests/ShortSighted/test.py | 2 +- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 649f2c39..439f4fac 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1096,7 +1096,6 @@ void MGmol::dumpRestart() // create restart file std::string filename(std::string(ct.out_restart_file)); - if (ct.out_restart_file_naming_strategy) filename += "0"; HDFrestart h5restartfile( filename, myPEenv, gdim, ct.out_restart_file_type); diff --git a/src/md.cc b/src/md.cc index dfb4dfe6..dc2857eb 100644 --- a/src/md.cc +++ b/src/md.cc @@ -242,9 +242,12 @@ int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, std::string filename(std::string(ct.out_restart_file)); // add an integer corresponding to attempt number/count // to allow several attempts at creating and writing file - std::stringstream s; - s << count; - filename += s.str(); + if (count > 0) + { + std::stringstream s; + s << count; + filename += s.str(); + } HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index 17b70de2..3f98b2bd 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg index 83d5aa4c..e30a8807 100644 --- a/tests/MD_MVP/md.cfg +++ b/tests/MD_MVP/md.cfg @@ -34,6 +34,6 @@ solver=MVP nb_inner_it=1 mixing=1. [Restart] -input_filename=wave.out +input_filename=snapshotMVP input_level=3 output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py index 489c9c8c..47b5462d 100755 --- a/tests/MD_MVP/test.py +++ b/tests/MD_MVP/test.py @@ -42,20 +42,20 @@ #run MD for i in range(2): - command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) - os.rename(restart_file, 'wave.out') + os.rename(restart_file, 'snapshotMVP') #run MGmol command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) #remove used restart files - shutil.rmtree('wave.out') + shutil.rmtree('snapshotMVP') #analyse mgmol standard output lines=output2.split(b'\n') @@ -81,7 +81,7 @@ sys.exit(1) #remove last restart files -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') shutil.rmtree(restart_file) diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 27da7f7d..68a617aa 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') From 16a818a43bae7906ff9b08f1765898c6b0569ae8 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 18:48:12 -0500 Subject: [PATCH 23/53] Add functionalities for extra info in restart file (#312) --- src/HDFrestart.cc | 22 ++++++------ src/HDFrestart.h | 8 ++--- src/Ion.cc | 8 ++++- src/Ion.h | 16 +++++---- src/Ions.cc | 87 +++++++++++++++++++++++++++++++++++++++++++++-- src/Ions.h | 5 +++ src/restart.cc | 2 ++ 7 files changed, 124 insertions(+), 24 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 0fac518e..87982290 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1440,8 +1440,8 @@ int HDFrestart::read_1func_hdf5(T* vv, const std::string& datasetname) } template -int HDFrestart::write_1func_hdf5( - T* vv, const std::string& datasetname, double* ll, double* cell_origin) +int HDFrestart::write_1func_hdf5(const T* const vv, + const std::string& datasetname, double* ll, double* cell_origin) { assert(ll != nullptr); assert(cell_origin != nullptr); @@ -1652,7 +1652,7 @@ int HDFrestart::readData( } template -int HDFrestart::writeData(T* data, hid_t space_id, hid_t memspace, +int HDFrestart::writeData(const T* const data, hid_t space_id, hid_t memspace, hid_t dset_id, const short precision) { if (precision == 1) @@ -2017,7 +2017,8 @@ int HDFrestart::readAtomicData( { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNames()..." << std::endl; + (*MPIdata::sout) << "HDFrestart::readAtomicData(), dataset = " + << datasetname << std::endl; std::vector buffer; short name_length = 7; // default, value used before February 2016 @@ -2095,10 +2096,9 @@ int HDFrestart::readAtomicData( { std::string t(&buffer[i], name_length); assert(t.size() > 0); - // cout<<"name="< - int writeData(T* vv, hid_t filespace, hid_t memspace, hid_t dset_id, - const short precision); + int writeData(const T* const vv, hid_t filespace, hid_t memspace, + hid_t dset_id, const short precision); template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); diff --git a/src/Ion.cc b/src/Ion.cc index 33465635..90350477 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -294,12 +294,18 @@ void Ion::getIonData(IonData& idata) const } } +void Ion::resetPositionsToPrevious() +{ + for (int pos = 0; pos < 3; pos++) + position_[pos] = old_position_[pos]; +} + void Ion::setFromIonData(const IonData& data) { // random state setRandomState(data.rand_state[0], data.rand_state[1], data.rand_state[2]); // previous position - setOldPosition( + setPreviousPosition( data.old_position[0], data.old_position[1], data.old_position[2]); // velocity setVelocity(data.velocity[0], data.velocity[1], data.velocity[2]); diff --git a/src/Ion.h b/src/Ion.h index ae870820..871205a3 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -74,12 +74,6 @@ class Ion position_[i] += shift; } - void setOldPosition(const double x, const double y, const double z) - { - old_position_[0] = x; - old_position_[1] = y; - old_position_[2] = z; - } public: Ion(const Species& species, const std::string& name, const double crds[3], @@ -96,6 +90,13 @@ class Ion void init(const double crds[3], const double velocity[3], const bool lock); void setup(); + void setPreviousPosition(const double x, const double y, const double z) + { + old_position_[0] = x; + old_position_[1] = y; + old_position_[2] = z; + } + std::shared_ptr kbproj() { return kbproj_; } const std::shared_ptr kbproj() const { return kbproj_; } @@ -189,6 +190,9 @@ class Ion kbproj_->clear(); } + + void resetPositionsToPrevious(); + void shiftPositionXLBOMDTest(Vector3D shift) { for (short dir = 0; dir < 3; dir++) diff --git a/src/Ions.cc b/src/Ions.cc index e93d82ea..09b97d2e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -853,6 +853,42 @@ void Ions::writePositions(HDFrestart& h5f_file) } } +void Ions::writePreviousPositions(HDFrestart& h5f_file) +{ + Control& ct(*(Control::instance())); + + if (onpe0 && ct.verbose > 1) + { + (*MPIdata::sout) << "Ions::writePositions" << std::endl; + } + + std::vector data; + if (h5f_file.gatherDataX()) + { + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MPI_Comm comm = myPEenv.comm_x(); + + gatherPreviousPositions(data, 0, comm); + } + else + { + for (auto& ion : local_ions_) + { + data.push_back(ion->getPreviousPosition(0)); + data.push_back(ion->getPreviousPosition(1)); + data.push_back(ion->getPreviousPosition(2)); + } + } + + hid_t file_id = h5f_file.file_id(); + if (file_id >= 0) + { + std::string datasetname("/Ionic_previous_positions"); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); + } +} + void Ions::initFromRestartFile(HDFrestart& h5_file) { assert(list_ions_.empty()); @@ -995,14 +1031,39 @@ void Ions::readRestartPositions(HDFrestart& h5_file) } } +void Ions::readRestartPreviousPositions(HDFrestart& h5_file) +{ + Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 0) + (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + + std::vector data; + std::string datasetname("/Ionic_previous_positions"); + h5_file.readAtomicData(datasetname, data); + assert(data.size() == 3 * local_ions_.size()); + + int i = 0; + for (auto& ion : local_ions_) + { + ion->setPreviousPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); + i++; + } +} + +void Ions::resetPositionsToPrevious() +{ + for (auto& ion : local_ions_) + { + ion->resetPositionsToPrevious(); + } +} + void Ions::writeVelocities(HDFrestart& h5f_file) { Control& ct(*(Control::instance())); if (onpe0 && ct.verbose > 1) - { (*MPIdata::sout) << "Ions::writeVelocities" << std::endl; - } std::vector data; if (h5f_file.gatherDataX()) @@ -2535,6 +2596,28 @@ void Ions::gatherPositions( if (mype == root) positions = data; } +void Ions::gatherPreviousPositions( + std::vector& positions, const int root, const MPI_Comm comm) const +{ + std::vector local_positions; + + for (auto& ion : local_ions_) + { + local_positions.push_back(ion->getPreviousPosition(0)); + local_positions.push_back(ion->getPreviousPosition(1)); + local_positions.push_back(ion->getPreviousPosition(2)); + } + + // gather data to PE root + std::vector data; + mgmol_tools::gatherV(local_positions, data, root, comm); + + int mype = 0; + MPI_Comm_rank(comm, &mype); + positions.clear(); + if (mype == root) positions = data; +} + void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { diff --git a/src/Ions.h b/src/Ions.h index ae4f6adb..b03972d1 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -147,6 +147,8 @@ class Ions const MPI_Comm comm) const; void gatherPositions(std::vector& positions, const int root, const MPI_Comm comm) const; + void gatherPreviousPositions(std::vector& positions, const int root, + const MPI_Comm comm) const; void gatherLockedNames(std::vector& names, const int root, const MPI_Comm comm) const; void gatherIndexes( @@ -198,6 +200,7 @@ class Ions ion++; } } + void resetPositionsToPrevious(); void removeMassCenterMotion(); bool hasNLprojectors() @@ -237,6 +240,7 @@ class Ions double kinetic_E(void) const; void writePositions(HDFrestart& h5f_file); + void writePreviousPositions(HDFrestart& h5f_file); void writeVelocities(HDFrestart& h5f_file); void writeRandomStates(HDFrestart& h5f_file); void writeForces(HDFrestart& h5f_file); @@ -350,6 +354,7 @@ class Ions void addIonToList(const Species& sp, const std::string& name, const double crds[3], const double velocity[3], const bool lock); + void readRestartPreviousPositions(HDFrestart& h5_file); // void checkUnicityLocalIons(); }; diff --git a/src/restart.cc b/src/restart.cc index ed7f478c..532043af 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -102,6 +102,8 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + ions.writePreviousPositions(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); From 77bd87ef256623ee5d95866a75f601417e559ca7 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 21 Feb 2025 12:51:24 -0500 Subject: [PATCH 24/53] MPI abort (#313) * Fix error code used by MPI_Abort --- src/BlockVector.cc | 2 +- src/Control.cc | 18 ++++++++--------- src/Control.h | 2 +- src/DistMatrix/BlacsContext.cc | 10 +++++----- src/DistMatrix/DistMatrix.h | 2 +- src/DistanceConstraint.cc | 2 +- src/ExtendedGridOrbitals.cc | 2 +- src/IonicAlgorithm.cc | 2 +- src/Ions.cc | 3 +-- src/LDAFunctional.cc | 3 +-- src/LocalizationRegions.cc | 20 +++++-------------- src/LocalizationRegions.h | 6 +++--- src/MDfiles.cc | 11 +++++----- src/MGmol.cc | 15 ++++++-------- src/MGmol.h | 1 - src/NonOrthoDMStrategy.cc | 2 +- src/PBEFunctional.cc | 3 +-- src/Potentials.cc | 8 ++++---- src/ProjectedMatricesSparse.cc | 2 +- src/main.cc | 2 +- src/md.cc | 7 +++---- src/mgmol_memory.cc | 2 +- src/mgmol_run.cc | 2 +- src/pb/Lap.h | 2 +- src/pb/PEenv.cc | 12 +++++------ src/pb/PEenv.h | 6 +++--- src/radial/RadialMeshFunction.cc | 4 ++-- src/setup.cc | 4 ++-- src/sparse_linear_algebra/DataDistribution.cc | 6 +++--- src/tools.cc | 6 +++--- src/tools/MGmol_MPI.cc | 6 +++--- src/tools/MGmol_MPI.h | 2 +- src/tools/Vector3D.cc | 2 +- 33 files changed, 80 insertions(+), 97 deletions(-) diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 40271750..9a411ebe 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -209,7 +209,7 @@ void BlockVector::allocate_storage() std::cerr << "ERROR BlockVector: trying to use allocation " << size_storage_ << " bigger than initialy preallocated " << allocated_size_storage_ << "!!!" << std::endl; - ct.global_exit(0); + ct.global_exit(); } storage_ = class_storage_[my_allocation_]; assert(class_storage_.size() > 0); diff --git a/src/Control.cc b/src/Control.cc index d266aa2e..33fd93c4 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -495,7 +495,7 @@ void Control::sync(void) if (mpirc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } }; @@ -1152,7 +1152,7 @@ void Control::setTolEigenvalueGram(const float tol) << threshold_eigenvalue_gram_ << std::endl; } -void Control::global_exit(int i) { MPI_Abort(comm_global_, i); } +void Control::global_exit() { MPI_Abort(comm_global_, EXIT_FAILURE); } void Control::setSpecies(Potentials& pot) { @@ -1342,7 +1342,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::serr) << "ERROR in Control::setOptions: Invalid restart dump type" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } (*MPIdata::sout) << "Output restart file: " << out_restart_file @@ -1549,14 +1549,14 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else { std::cerr << "ERROR: Spread Penalty needs a type" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (spread_penalty_target_ <= 0.) { (*MPIdata::sout) << "Invalid value for Spread Penalty target: " << spread_penalty_target_ << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } @@ -1599,7 +1599,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) { (*MPIdata::sout) << "Invalid value for Thermostat.type: " << thermostat_type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } tkel = vm["Thermostat.temperature"].as(); @@ -1608,7 +1608,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.temperature: " << tkel << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } thtime = vm["Thermostat.relax_time"].as(); if (thtime < 0.) @@ -1616,7 +1616,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.relax_time: " << thtime << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (str.compare("SCALING") == 0) @@ -1627,7 +1627,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.width: " << thwidth << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } } diff --git a/src/Control.h b/src/Control.h index bd4a7d96..9481cc24 100644 --- a/src/Control.h +++ b/src/Control.h @@ -346,7 +346,7 @@ class Control return pair_mlwf_distance_threshold_; } - void global_exit(int i); + void global_exit(); bool Mehrstellen() const { return (lap_type == 0 || lap_type == 10); } diff --git a/src/DistMatrix/BlacsContext.cc b/src/DistMatrix/BlacsContext.cc index e87cb7dd..7610a3cb 100644 --- a/src/DistMatrix/BlacsContext.cc +++ b/src/DistMatrix/BlacsContext.cc @@ -191,7 +191,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: type = " << type << " is an incorrect parameter" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } size_ = nprow_ * npcol_; @@ -222,7 +222,7 @@ BlacsContext::BlacsContext( { std::cerr << " nprocs_=" << nprocs_ << std::endl; std::cerr << " BlacsContext nprow*npcol > nprocs_" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); @@ -252,7 +252,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << " in " << __FILE__ << ":" << __LINE__ << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -296,7 +296,7 @@ BlacsContext::BlacsContext(BlacsContext& bc, const int irow, const int icol, { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -350,7 +350,7 @@ BlacsContext::BlacsContext(const BlacsContext& bc, const char type) std::cerr << " BlacsContext::BlacsContext: row/col incorrect parameter: " << type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); diff --git a/src/DistMatrix/DistMatrix.h b/src/DistMatrix/DistMatrix.h index f56df2b4..175fbd9b 100644 --- a/src/DistMatrix/DistMatrix.h +++ b/src/DistMatrix/DistMatrix.h @@ -28,7 +28,7 @@ std::cerr << "ERROR in file " << __FILE__ << " at line " << __LINE__ \ << std::endl; \ std::cerr << "Error Message: " << X << std::endl; \ - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); #endif diff --git a/src/DistanceConstraint.cc b/src/DistanceConstraint.cc index d5175d7e..6d046f3b 100644 --- a/src/DistanceConstraint.cc +++ b/src/DistanceConstraint.cc @@ -99,7 +99,7 @@ bool DistanceConstraint::enforce(void) { cerr << "mype=" << mype << ", tau1p_[0]=" << tau1p_[0] << endl; cerr << "mype=" << mype << ", tau2p_[0]=" << tau2p_[0] << endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } if (locally_owned_) (*MPIdata::sout) << setprecision(8) << "DistanceConstraint, d=" << d diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 19c0d26b..07face7c 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -1239,7 +1239,7 @@ double ExtendedGridOrbitals::dotProduct( "dot product type" << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(2); + ct.global_exit(); } dot_product_tm_.stop(); diff --git a/src/IonicAlgorithm.cc b/src/IonicAlgorithm.cc index 37a077e4..e804077a 100644 --- a/src/IonicAlgorithm.cc +++ b/src/IonicAlgorithm.cc @@ -57,7 +57,7 @@ void IonicAlgorithm::init(HDFrestart* h5f_file) if (ct.restart_info > 0) { int status = stepper_->init(*h5f_file); - if (status < 0) ct.global_exit(2); + if (status < 0) ct.global_exit(); // if restart data for lbfgs found if (status == 0) diff --git a/src/Ions.cc b/src/Ions.cc index 09b97d2e..9af0a64e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1961,7 +1961,6 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) double velocity[3] = { 0., 0., 0. }; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); short movable = 0; std::string query; @@ -2005,7 +2004,7 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) { std::cerr << "ERROR: Invalid name read in input file: " << name_read << std::endl; - ct.global_exit(2); + mmpi.abort(); } short dummy; ss >> dummy; // not used anymore (was species index) diff --git a/src/LDAFunctional.cc b/src/LDAFunctional.cc index 4b60b51e..89f5ee5f 100644 --- a/src/LDAFunctional.cc +++ b/src/LDAFunctional.cc @@ -120,8 +120,7 @@ double LDAFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = (POTDTYPE)sum; return exc; diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 60f28301..f0f96412 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -492,7 +492,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'nglobal_'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (nglobal_ == 0) return; @@ -522,7 +522,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'centers'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } vector::iterator it = all_regions_.begin(); int i = 0; @@ -554,7 +554,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'radius'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } it = all_regions_.begin(); i = 0; @@ -642,7 +642,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'volume'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (ct.verbose > 0) printWithTimeStamp("LocalizationRegions::bcast() done...", cout); @@ -915,19 +915,9 @@ void LocalizationRegions::setupLocalRegionsFromOverlapRegions() cerr << "ERROR in distribution of localization centers: count=" << count << endl; cerr << "global number of regions=" << nglobal_ << endl; - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } -#if 0 - //if(onpe0) - if( count!=nglobal_ ) - { - cerr<<"ERROR in distribution of localization centers: count="<::write_header() if (ct.isLocMode() && ct.verbose > 3) lrs_->printAllRegions(os_); } -template -void MGmol::global_exit(int i) -{ - MPI_Abort(comm_, i); -} - template void MGmol::check_anisotropy() { @@ -748,7 +742,8 @@ void MGmol::check_anisotropy() << ", hmin=" << mygrid.hmin() << std::endl; (*MPIdata::serr) << "init: Anisotropy too large: " << mygrid.anisotropy() << std::endl; - global_exit(2); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.abort(); } } @@ -1046,7 +1041,8 @@ void MGmol::setup() total_tm_.start(); setup_tm_.start(); - Control& ct = *(Control::instance()); + Control& ct = *(Control::instance()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (ct.verbose > 0) printWithTimeStamp("MGmol::setup()...", os_); @@ -1064,7 +1060,8 @@ void MGmol::setup() #else int ierr = initial(); #endif - if (ierr < 0) global_exit(0); + + if (ierr < 0) mmpi.abort(); // Write header to stdout write_header(); diff --git a/src/MGmol.h b/src/MGmol.h index 4d82dd64..bccbc7af 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -221,7 +221,6 @@ class MGmol : public MGmolInterface void initNuc(Ions& ions); void initKBR(); - void global_exit(int i); void printEigAndOcc(); int readCoordinates(std::ifstream* tfile, const bool cell_relative); diff --git a/src/NonOrthoDMStrategy.cc b/src/NonOrthoDMStrategy.cc index 5452592f..3835b039 100644 --- a/src/NonOrthoDMStrategy.cc +++ b/src/NonOrthoDMStrategy.cc @@ -46,7 +46,7 @@ int NonOrthoDMStrategy::update(OrbitalsType& orbitals) { std::cerr << "NonOrthoDMStrategy, Invalid mixing value: " << mix_ << std::endl; - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); } if (mmpi.PE0() && ct.verbose > 2) diff --git a/src/PBEFunctional.cc b/src/PBEFunctional.cc index 4a878c3c..29151a14 100644 --- a/src/PBEFunctional.cc +++ b/src/PBEFunctional.cc @@ -167,8 +167,7 @@ double PBEFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << std::endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = sum; return exc; diff --git a/src/Potentials.cc b/src/Potentials.cc index a0ff3f1a..ebe42320 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -185,8 +185,8 @@ double Potentials::update(const std::vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + MPI_Abort(myPEenv.comm(), EXIT_FAILURE); } dvdot = sum; @@ -253,8 +253,8 @@ double Potentials::delta_v(const std::vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + MPI_Abort(myPEenv.comm(), EXIT_FAILURE); } dvdot = sum; diff --git a/src/ProjectedMatricesSparse.cc b/src/ProjectedMatricesSparse.cc index 8b328b1c..15ad6d6c 100644 --- a/src/ProjectedMatricesSparse.cc +++ b/src/ProjectedMatricesSparse.cc @@ -354,7 +354,7 @@ double ProjectedMatricesSparse::dotProductSimple( << std::endl; MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); return -1.; } diff --git a/src/main.cc b/src/main.cc index 3fcbf791..3c6830a0 100644 --- a/src/main.cc +++ b/src/main.cc @@ -45,7 +45,7 @@ int main(int argc, char** argv) if (mpirc != MPI_SUCCESS) { std::cerr << "MPI Initialization failed!!!" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } MPI_Comm comm = MPI_COMM_WORLD; diff --git a/src/md.cc b/src/md.cc index dc2857eb..8ec20074 100644 --- a/src/md.cc +++ b/src/md.cc @@ -690,11 +690,10 @@ void MGmol::loadRestartFile(const std::string filename) if (ierr < 0) { if (onpe0) - (*MPIdata::serr) - << "loadRestartFile: failed to read the restart file." - << std::endl; + std::cerr << "loadRestartFile: failed to read the restart file." + << std::endl; - global_exit(0); + mmpi.abort(); } if (!ct.fullyOccupied()) { diff --git a/src/mgmol_memory.cc b/src/mgmol_memory.cc index a1360c99..b1be12d6 100644 --- a/src/mgmol_memory.cc +++ b/src/mgmol_memory.cc @@ -62,7 +62,7 @@ void addTrack(long addr, long asize) if (nPos > MAXNUMALLOCATIONS) { printf("ERROR: Not enough memory slots!!!"); - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } // if(onpe0)printf("nPos=%d, addr=%ld, size=%ld\n",nPos,addr,asize); } diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index a5de1ad5..468d61bd 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -96,7 +96,7 @@ int mgmol_check() { std::cerr << "Code should be called with " << myPEenv.n_mpi_tasks() << " MPI tasks only" << std::endl; - ct.global_exit(2); + ct.global_exit(); } assert(ct.getMGlevels() >= -1); diff --git a/src/pb/Lap.h b/src/pb/Lap.h index 33d90706..460e36c1 100644 --- a/src/pb/Lap.h +++ b/src/pb/Lap.h @@ -35,7 +35,7 @@ class Lap : public FDoper virtual void applyWithPot(GridFunc&, const double* const, T*) { std::cerr << "ERROR: Lap::applyWithPot() not implemented" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } std::string name() const { return name_; } diff --git a/src/pb/PEenv.cc b/src/pb/PEenv.cc index d5ff78f3..3e2fb777 100644 --- a/src/pb/PEenv.cc +++ b/src/pb/PEenv.cc @@ -197,7 +197,7 @@ PEenv::~PEenv() if (mpirc != MPI_SUCCESS) { std::cerr << "MPI_Comm_free failed!" << std::endl; - MPI_Abort(comm_, 2); + MPI_Abort(comm_, EXIT_FAILURE); } } if (cart_comm_ != MPI_COMM_NULL) MPI_Comm_free(&cart_comm_); @@ -277,7 +277,7 @@ void PEenv::task2xyz() if (rc != MPI_SUCCESS) { std::cerr << " error in MPI_Cart_coords()!!!" << std::endl; - MPI_Abort(comm_, 1); + MPI_Abort(comm_, EXIT_FAILURE); } #else mytask_dir_[2] = mytask_ % n_mpi_tasks_dir_[2]; @@ -650,7 +650,7 @@ void PEenv::split_comm(const int nx, const int ny, const int nz, const int bias) { std::cerr << "MPI_Comm_split failed!, my color_=" << color_ << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } MPI_Comm_size(comm_active_, &n_mpi_tasks_); #ifndef NDEBUG @@ -696,7 +696,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Recv() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } else if (ip == mytask_) @@ -708,7 +708,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Send() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } if (mytask_ == 0) @@ -717,7 +717,7 @@ void PEenv::printPEnames(std::ostream& os) const if (mytask_ == 0) os << std::endl; } -void PEenv::globalExit() const { MPI_Abort(comm_, 2); } +void PEenv::globalExit() const { MPI_Abort(comm_, EXIT_FAILURE); } void PEenv::bcast(int* val, const int n) const { diff --git a/src/pb/PEenv.h b/src/pb/PEenv.h index bc4751a5..9782ddb9 100644 --- a/src/pb/PEenv.h +++ b/src/pb/PEenv.h @@ -166,7 +166,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxXdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -181,7 +181,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxYdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -196,7 +196,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxZdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } diff --git a/src/radial/RadialMeshFunction.cc b/src/radial/RadialMeshFunction.cc index 499e482a..79b32518 100644 --- a/src/radial/RadialMeshFunction.cc +++ b/src/radial/RadialMeshFunction.cc @@ -98,7 +98,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int i = 0; i < nn[1]; i++) @@ -110,7 +110,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } } } diff --git a/src/setup.cc b/src/setup.cc index 9963bfc5..2ff6d79e 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -140,7 +140,7 @@ int MGmol::setupLRsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { @@ -173,7 +173,7 @@ int MGmol::setupConstraintsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { diff --git a/src/sparse_linear_algebra/DataDistribution.cc b/src/sparse_linear_algebra/DataDistribution.cc index d4b26d93..67bb1319 100644 --- a/src/sparse_linear_algebra/DataDistribution.cc +++ b/src/sparse_linear_algebra/DataDistribution.cc @@ -288,7 +288,7 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, std::cout << "ERROR: " << name_ << ", dir=" << dir << ", remote_size=" << remote_size << ", bsiz=" << bsiz << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } // string stamp="DataDistribution ("+name_+"), buffer size checked..."; // printWithTimeStamp(stamp,cout); @@ -302,14 +302,14 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, if (mpircv != MPI_SUCCESS) { std::cout << "ERROR in MPI_Irecv, code=" << mpircv << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } int mpisnd = MPI_Isend(packed_buffer.sendBuffer(), siz, MPI_CHAR, dest, 0, cart_comm_, &request[1]); if (mpisnd != MPI_SUCCESS) { std::cout << "ERROR in MPI_Isend, code=" << mpisnd << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } /* wait to complete communication */ MPI_Waitall(2, request, MPI_STATUSES_IGNORE); diff --git a/src/tools.cc b/src/tools.cc index 95790c59..aff34933 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -29,7 +29,7 @@ void noMoreMemory() std::cerr << "Unable to satisfy request for memory for MPI task " << mype << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(3); + ct.global_exit(); } // an atom name should start with a capital letter and end with a number @@ -229,12 +229,12 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) if( mpierr!=MPI_SUCCESS ) { cerr << " Error in MPI!!!" << std::endl; - MPI_Abort(mmpi.commGlobal(),1); + MPI_Abort(mmpi.commGlobal(),EXIT_FAILURE); } if( r!=mmpi.size()*s && onpe0 ) { cerr << " Error in barrier: "<& vv, MPI_Comm comm) std::cerr << "ERROR!!!! bcastvv3d(), Failure in MPI_Bcast of 'radii_'!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int j = 0; j < n; j++) for (short i = 0; i < 3; i++) From 8b84de49791659a4d874c617f71a920fb08e9138 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 24 Feb 2025 15:03:24 -0500 Subject: [PATCH 25/53] Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung --- src/Electrostatic.h | 3 + src/MGmol.h | 4 +- src/Potentials.cc | 6 + src/Potentials.h | 2 + src/md.cc | 13 ++ tests/CMakeLists.txt | 15 ++ tests/RhoVhRestart/h2o.xyz | 6 + tests/RhoVhRestart/md.cfg | 30 ++++ tests/RhoVhRestart/mgmol.cfg | 28 ++++ tests/RhoVhRestart/restart.cfg | 25 +++ tests/RhoVhRestart/test.py | 79 +++++++++ tests/RhoVhRestart/testRhoVhRestart.cc | 220 +++++++++++++++++++++++++ 12 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 tests/RhoVhRestart/h2o.xyz create mode 100644 tests/RhoVhRestart/md.cfg create mode 100644 tests/RhoVhRestart/mgmol.cfg create mode 100644 tests/RhoVhRestart/restart.cfg create mode 100755 tests/RhoVhRestart/test.py create mode 100644 tests/RhoVhRestart/testRhoVhRestart.cc diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 9beee899..06a9e17c 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -47,6 +47,9 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + pb::GridFunc* getRhoc() { return grhoc_; } + Poisson* getPoissonSolver() { return poisson_solver_; } + void setup(const short max_sweeps); void setupPB(const double e0, const double rho0, const double drho0, Potentials& pot); diff --git a/src/MGmol.h b/src/MGmol.h index bccbc7af..b18465a1 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -150,7 +150,6 @@ class MGmol : public MGmolInterface void initialMasks(); int setupLRsFromInput(const std::string filename); - void setup(); int setupLRs(const std::string input_file) override; int setupFromInput(const std::string input_file) override; int setupConstraintsFromInput(const std::string input_file) override; @@ -177,12 +176,15 @@ class MGmol : public MGmolInterface ~MGmol() override; + void setup(); + /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/Potentials.cc b/src/Potentials.cc index ebe42320..96375c6d 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -939,6 +939,12 @@ int Potentials::read(HDFrestart& h5f_file) h5f_file.read_1func_hdf5(vepsilon_.data(), "VDielectric"); } + std::string datasetname("Preceding_Hartree"); + if (h5f_file.checkDataExists(datasetname)) + { + h5f_file.read_1func_hdf5(vh_rho_backup_.data(), datasetname); + } + return 0; } diff --git a/src/Potentials.h b/src/Potentials.h index cb0f8e05..8e9b763c 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -206,6 +206,8 @@ class Potentials */ void backupVh(); + void resetVhRho2Backup() { vh_rho_ = vh_rho_backup_; } + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/md.cc b/src/md.cc index 8ec20074..b1ffb619 100644 --- a/src/md.cc +++ b/src/md.cc @@ -702,6 +702,19 @@ void MGmol::loadRestartFile(const std::string filename) ierr = proj_matrices_->readWFDM(h5file); } + if (h5file.checkDataExists("Preceding_Hartree")) + { + ions_->readRestartPreviousPositions(h5file); + ions_->resetPositionsToPrevious(); + ions_->setup(); + + Potentials& pot = hamiltonian_->potential(); + pot.initialize(*ions_); + if (onpe0) std::cout << "Reset VhRho to backup..." << std::endl; + pot.resetVhRho2Backup(); + electrostat_->setupRhoc(pot.rho_comp()); + } + ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); if (ierr < 0) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index baf1c451..838ce6a4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,6 +250,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +add_executable(testRhoVhRestart + ${CMAKE_SOURCE_DIR}/tests/RhoVhRestart/testRhoVhRestart.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) add_executable(testWFEnergyAndForces @@ -382,6 +384,17 @@ add_test(NAME testRestartEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRhoVhRestart + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRhoVhRestart + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) + if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -577,6 +590,7 @@ target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) +target_include_directories(testRhoVhRestart PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -589,6 +603,7 @@ target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) +target_link_libraries(testRhoVhRestart mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/RhoVhRestart/h2o.xyz b/tests/RhoVhRestart/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/RhoVhRestart/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/RhoVhRestart/md.cfg b/tests/RhoVhRestart/md.cfg new file mode 100644 index 00000000..2b8a378b --- /dev/null +++ b/tests/RhoVhRestart/md.cfg @@ -0,0 +1,30 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +output_level=4 +output_filename=WF_MD diff --git a/tests/RhoVhRestart/mgmol.cfg b/tests/RhoVhRestart/mgmol.cfg new file mode 100644 index 00000000..eee7f11c --- /dev/null +++ b/tests/RhoVhRestart/mgmol.cfg @@ -0,0 +1,28 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF diff --git a/tests/RhoVhRestart/restart.cfg b/tests/RhoVhRestart/restart.cfg new file mode 100644 index 00000000..20f0293a --- /dev/null +++ b/tests/RhoVhRestart/restart.cfg @@ -0,0 +1,25 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF_MD diff --git a/tests/RhoVhRestart/test.py b/tests/RhoVhRestart/test.py new file mode 100755 index 00000000..a34b962f --- /dev/null +++ b/tests/RhoVhRestart/test.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-7] +test_exe = sys.argv[nargs-6] +input1 = sys.argv[nargs-5] +input2 = sys.argv[nargs-4] +input3 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("Initial quench failed to complete!") + sys.exit(1) + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("MD failed to complete!") + sys.exit(1) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input3,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') +for line in lines: + print(line) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc new file mode 100644 index 00000000..09da3320 --- /dev/null +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -0,0 +1,220 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "Electrostatic.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Poisson.h" +#include "Potentials.h" +#include "mgmol_run.h" + +#include +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +template +int testRhoRestart(MGmolInterface* mgmol_) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + std::shared_ptr> rho = mgmol->getRho(); + + /* save density from the restart file to elsewhere */ + std::vector rho0(rho->rho_[0].size()); + rho0 = rho->rho_[0]; + + /* recompute rho from the orbital */ + rho->update(*mgmol->getOrbitals()); + + /* check if the recomputed density is the same */ + for (int d = 0; d < (int)rho0.size(); d++) + { + double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); + if (error > 1e-10) + { + printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, + rho->rho_[0][d], d, rho0[d]); + std::cerr << "Density is inconsistent!!!" << std::endl; + return -1; + } + } + if (rank == 0) std::cout << "Density is consistent..." << std::endl; + + return 0; +} + +template +int testPotRestart(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + Potentials& pot = mgmol->getHamiltonian()->potential(); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + std::shared_ptr> rho = mgmol->getRho(); + + /* GridFunc initialization inputs */ + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = ct.bcPoisson[d]; + + /* save potential from the restart file to elsewhere */ + pb::GridFunc vh0_gf(mygrid, bc[0], bc[1], bc[2]); + vh0_gf.assign((pot.vh_rho()).data(), 'd'); + double n = vh0_gf.norm2(); + std::cout << "Norm2 of vh = " << n << std::endl; + + std::vector vh0(pot.size()); + const std::vector& d_vhrho(pot.vh_rho()); + for (int d = 0; d < (int)vh0.size(); d++) + vh0[d] = d_vhrho[d]; + + /* recompute potential */ + pb::GridFunc grho(mygrid, bc[0], bc[1], bc[2]); + grho.assign(&rho->rho_[0][0]); + pb::GridFunc* grhoc = mgmol->electrostat_->getRhoc(); + + poisson->solve(grho, *grhoc); + const pb::GridFunc& vh(poisson->vh()); + + pb::GridFunc error_gf(vh0_gf); + error_gf -= vh; + + double rel_error = error_gf.norm2() / vh0_gf.norm2(); + if (rank == 0) + { + printf("FOM potential relative error: %.3e\n", rel_error); + } + if (rel_error > 1e-9) + { + if (rank == 0) + { + std::cerr << "Potential is inconsistent!!!" << std::endl; + } + return -1; + } + if (rank == 0) std::cout << "Potential is consistent..." << std::endl; + + return 0; +} + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + int status = 0; + + // Enter main scope + { + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + mgmol->setup(); + + /* load a restart file */ + MGmol* mgmol_ext + = dynamic_cast*>(mgmol); + mgmol_ext->loadRestartFile(ct.restart_file); + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testRhoRestart..." << std::endl; + status = testRhoRestart(mgmol); + if (status < 0) return status; + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testPotRestart..." << std::endl; + status = testPotRestart(mgmol); + if (status < 0) return status; + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From 8003d70f5758f3cef4d74724f2b7d3a04afe7964 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Mar 2025 21:51:29 -0500 Subject: [PATCH 26/53] Strenghten testIons (#315) --- tests/testIons.cc | 76 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/tests/testIons.cc b/tests/testIons.cc index 30b2ad0c..6bfdd74a 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -41,7 +41,7 @@ int main(int argc, char** argv) // read species info from pseudopotential file std::string file_path = argv[1]; std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); - std::cout << "Potential = " << filename << std::endl; + if (myrank == 0) std::cout << "Potential = " << filename << std::endl; sp.read_1species(filename); sp.set_dim_nl(h[0]); @@ -80,13 +80,71 @@ int main(int argc, char** argv) ions.setup(); - std::vector& new_local_ions(ions.local_ions()); + // verify sum of local ions adds up to total number of ions + { + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + } + MPI_Barrier(MPI_COMM_WORLD); - int nlocal = new_local_ions.size(); - std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + // verify some functionalities of class Ions + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + ions.getAtomicNumbers(anumbers); + if (myrank == 0) + { + int i = 0; + for (auto& position : positions) + { + std::cout << position; + if (i % 3 == 2) + std::cout << std::endl; + else + std::cout << " "; + i++; + } + } + MPI_Barrier(MPI_COMM_WORLD); + + // swap x and z + for (size_t i = 0; i < positions.size() - 2; i++) + { + double x = positions[i]; + double z = positions[i + 2]; + positions[i] = z; + positions[i + 2] = x; + } + + ions.setPositions(positions, anumbers); + } - int ntotal = 0; - MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + MPI_Barrier(MPI_COMM_WORLD); + { + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + } mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { @@ -94,11 +152,5 @@ int main(int argc, char** argv) return 1; } - if (ntotal != na) - { - std::cout << "ntotal = " << ntotal << std::endl; - return 1; - } - return 0; } From 4efb34c5e2e2283043faa4f8b612e195570806bd Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Mar 2025 21:52:11 -0500 Subject: [PATCH 27/53] More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes --- src/MGmol.cc | 9 +++--- src/Potentials.cc | 70 ++++++++++++++--------------------------------- src/Potentials.h | 15 +++------- 3 files changed, 29 insertions(+), 65 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 8d87f1c1..c18e58c7 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -400,7 +400,6 @@ int MGmol::initial() current_orbitals_->setDataWithGhosts(); current_orbitals_->trade_boundaries(); - // if(ct.restart_info <= 1)pot.initWithVnuc(); // initialize matrices S and invB if (ct.numst > 0) @@ -1379,14 +1378,14 @@ void MGmol::update_pot(const Ions& ions) const bool flag_mixing = (fabs(ct.mix_pot - 1.) > 1.e-3); - // evaluate potential correction + // update total potential if (flag_mixing) { - pot.delta_v(rho_->rho_); - pot.update(ct.mix_pot); + pot.computeDeltaV(rho_->rho_); + pot.updateVtot(ct.mix_pot); } else - pot.update(rho_->rho_); + pot.updateVtot(rho_->rho_); } template diff --git a/src/Potentials.cc b/src/Potentials.cc index 96375c6d..4da9ab9a 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -87,37 +87,19 @@ Potentials::Potentials() #endif } -void Potentials::initWithVnuc() -{ - assert(size_ > 0); - if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::initWithVnuc()" << std::endl; - itindex_vxc_ = 0; - itindex_vh_ = 0; - int ione = 1; - Tcopy(&size_, &v_nuc_[0], &ione, &vtot_[0], &ione); - double one = 1.; - LinearAlgebraUtils::MPaxpy( - size_, one, &v_ext_[0], &vtot_[0]); - // factor ha2ry to get total potential in [Ry] for calculations - LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); -} - double Potentials::max() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmax = (*max_element(vtot_.begin(), vtot_.end())); - vmax = myPEenv.double_max_all(vmax); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmax = (*max_element(vtot_.begin(), vtot_.end())); + vmax = mmpi.allreduce(&vmax, 1, MPI_MAX); return vmax; } double Potentials::min() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmin = -(*min_element(vtot_.begin(), vtot_.end())); - vmin = -myPEenv.double_max_all(vmin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmin = -(*min_element(vtot_.begin(), vtot_.end())); + vmin = -mmpi.allreduce(&vmin, 1, MPI_MAX); return vmin; } @@ -141,7 +123,7 @@ void Potentials::evalNormDeltaVtotRho( mmpi.allreduce(&scf_dvrho_, 1, MPI_SUM); } -double Potentials::update(const std::vector>& rho) +double Potentials::updateVtot(const std::vector>& rho) { assert(itindex_vxc_ >= 0); assert(itindex_vh_ >= 0); @@ -149,9 +131,8 @@ double Potentials::update(const std::vector>& rho) if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::update(rho)" << std::endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -181,23 +162,22 @@ double Potentials::update(const std::vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), EXIT_FAILURE); + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; } -void Potentials::update(const double mix) +void Potentials::updateVtot(const double mix) { assert(itindex_vxc_ == itindex_vh_); @@ -210,7 +190,7 @@ void Potentials::update(const double mix) size_, potmix, &dv_[0], &vtot_[0]); } -double Potentials::delta_v(const std::vector>& rho) +double Potentials::computeDeltaV(const std::vector>& rho) { assert(itindex_vxc_ == itindex_vh_); assert(size_ > 0); @@ -218,9 +198,8 @@ double Potentials::delta_v(const std::vector>& rho) if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::delta_v()" << std::endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -249,17 +228,16 @@ double Potentials::delta_v(const std::vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), EXIT_FAILURE); + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; @@ -437,7 +415,6 @@ void Potentials::readExternalPot(const std::string filename, const char type) { assert(index < size_); (*from) >> v_ext_[index]; - //(*MPIdata::sout)<& vrho) const; /*! * evaluate potential correction associated with a new rho */ - double delta_v(const std::vector>& rho); + double computeDeltaV(const std::vector>& rho); /*! - * update potentials based on argument rho + * update total potential with updated components */ - double update(const std::vector>& rho); + double updateVtot(const std::vector>& rho); /*! * update potentials based on potential correction delta v and mixing * parameter */ - void update(const double mix); + void updateVtot(const double mix); double max() const; double min() const; @@ -196,7 +190,6 @@ class Potentials template void setVxc(const T* const vxc, const int iterativeIndex); - void setVh(const POTDTYPE* const vh, const int iterativeIndex); void setVh(const pb::GridFunc& vh, const int iterativeIndex); void initialize(Ions& ions); From 2f511b775505a127869107d8c90a30cb212ad591 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 14 Mar 2025 17:41:39 -0400 Subject: [PATCH 28/53] Misc code fixes (#317) * exposed by porting on new environment --- src/HDFrestart.cc | 6 ++++++ src/Ions.cc | 2 ++ src/tools/SymmetricMatrix.h | 1 + 3 files changed, 9 insertions(+) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 87982290..e2bfa1ab 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1922,10 +1922,12 @@ int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), -1), data.end()); } +#endif return 0; } @@ -1974,10 +1976,12 @@ int HDFrestart::readAtomicData( } } +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); } +#endif if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2104,10 +2108,12 @@ int HDFrestart::readAtomicData( data.push_back(t); } +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), ""), data.end()); } +#endif return 0; } diff --git a/src/Ions.cc b/src/Ions.cc index 9af0a64e..bc30417b 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -947,10 +947,12 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); +#ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } +#endif if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ diff --git a/src/tools/SymmetricMatrix.h b/src/tools/SymmetricMatrix.h index bd887fb3..b7aece1e 100644 --- a/src/tools/SymmetricMatrix.h +++ b/src/tools/SymmetricMatrix.h @@ -14,6 +14,7 @@ #include #include #include +#include #include // full lower triangular part of symmetric matrix with compact storage From 45a2c80cd6199da0643adf787d04831650e57d07 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 14 Mar 2025 17:42:00 -0400 Subject: [PATCH 29/53] Use unique restart filenames in test HDF5single (#318) --- tests/HDF5single/md.cfg | 4 ++-- tests/HDF5single/mgmol.cfg | 2 +- tests/HDF5single/test.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg index 1ff2adab..8d6b21e2 100644 --- a/tests/HDF5single/md.cfg +++ b/tests/HDF5single/md.cfg @@ -25,10 +25,10 @@ max_steps=24 atol=1.e-8 [Restart] input_level=4 -input_filename=WF +input_filename=wf.h5 input_type=single_file output_level=4 -output_filename=WF_MD +output_filename=wf_md.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg index 4dba942a..e8f01a29 100644 --- a/tests/HDF5single/mgmol.cfg +++ b/tests/HDF5single/mgmol.cfg @@ -25,7 +25,7 @@ initial_type=Random initial_width=1.5 [Restart] output_level=4 -output_filename=WF +output_filename=wf.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py index 080ee0ba..e2f0596a 100755 --- a/tests/HDF5single/test.py +++ b/tests/HDF5single/test.py @@ -47,7 +47,7 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') -os.remove('WF') +os.remove('wf.h5') print("Check energy conservation...") tol = 1.e-4 @@ -71,5 +71,7 @@ print("ERROR needs 4 energy values for checking conservation!") sys.exit(1) +os.remove('wf_md.h5') + print("Test SUCCESSFUL!") sys.exit(0) From b6cf162eac41663473659d44bd04c087d56e5fdc Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Sat, 15 Mar 2025 09:12:08 -0400 Subject: [PATCH 30/53] Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces --- src/MGmol.h | 5 +++ src/MGmolInterface.h | 3 ++ tests/RestartEnergyAndForces/test.py | 6 ++++ .../testRestartEnergyAndForces.cc | 32 +++++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/src/MGmol.h b/src/MGmol.h index b18465a1..350e8511 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -341,6 +341,11 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + void setPositions(const std::vector& positions, + const std::vector& atnumbers) + { + ions_->setPositions(positions, atnumbers); + } /* * simply dump current state diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9a9bf8a6..dc54a8e7 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -40,6 +40,9 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual void setPositions(const std::vector& positions, + const std::vector& atnumbers) + = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index b62d39f8..585722f0 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -64,7 +64,13 @@ shutil.rmtree('WF') test_energy=1.e18 +l=-1 for line in lines: + if line.count(b'Positions'): + l=0 + if l>=0 and l<4: + print(line) + l=l+1 if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index e323afbc..3027ab55 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,6 +124,8 @@ int main(int argc, char** argv) } } + mgmol->setPositions(positions, anumbers); + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -165,6 +167,36 @@ int main(int argc, char** argv) projmatrices->setDMuniform(ct.getNelSpin(), 0); projmatrices->printDM(std::cout); + // swap H and O to make sure order of atoms in list does not matter + double x = positions[0]; + double y = positions[1]; + double z = positions[2]; + positions[0] = positions[3]; + positions[1] = positions[4]; + positions[2] = positions[5]; + positions[3] = x; + positions[4] = y; + positions[5] = z; + short tmp = anumbers[0]; + anumbers[0] = anumbers[1]; + anumbers[1] = tmp; + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + mgmol->setPositions(positions, anumbers); + // // evaluate energy and forces with wavefunctions just read // From 7fbfe133c1ebd4f0932c9464e4e512ef8228d8d3 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Mar 2025 13:44:30 -0400 Subject: [PATCH 31/53] Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() --- src/HDFrestart.cc | 1 - src/KBPsiMatrixSparse.cc | 45 +++++++++++++++++++--------------------- src/KBPsiMatrixSparse.h | 15 ++++++-------- src/MGmol.cc | 18 ++++++++-------- src/MGmol.h | 2 +- src/md.cc | 2 +- src/quench.cc | 1 - 7 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index e2bfa1ab..1b2c7fb7 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -2104,7 +2104,6 @@ int HDFrestart::readAtomicData( stripLeadingAndTrailingBlanks(t); // std::cout<<"stripped name="< 0); data.push_back(t); } diff --git a/src/KBPsiMatrixSparse.cc b/src/KBPsiMatrixSparse.cc index 604e2f8e..8f5b3d49 100644 --- a/src/KBPsiMatrixSparse.cc +++ b/src/KBPsiMatrixSparse.cc @@ -11,16 +11,15 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" -#include "Ions.h" #include "LocGridOrbitals.h" #include "MGmol_MPI.h" #include "Mesh.h" #include "ProjectedMatrices.h" -#include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include + #define Ry2Ha 0.5; Timer KBPsiMatrixSparse::global_sum_tm_("KBPsiMatrixSparse::global_sum"); @@ -133,7 +132,7 @@ void KBPsiMatrixSparse::globalSumKBpsi() // Loop over the ions with projectors overlapping with local subdomain // and evaluate for some state. template -void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, T& orbitals, const int first_color, const int nb_colors, const bool flag) { assert(first_color >= 0); @@ -189,7 +188,7 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, // Loop over the ions if (gid != -1) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement( *ion, gid, iloc, ppsi + color * ldsize, flag); @@ -211,8 +210,8 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, compute_kbpsi_tm_.stop(); } -void KBPsiMatrixSparse::computeKBpsi( - Ions& ions, pb::GridFunc* phi, const int istate, const bool flag) +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, + pb::GridFunc* phi, const int istate, const bool flag) { assert(lapop_ != nullptr); compute_kbpsi_tm_.start(); @@ -238,7 +237,7 @@ void KBPsiMatrixSparse::computeKBpsi( for (int iloc = 0; iloc < subdivx; iloc++) { // Loop over the ions - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement(*ion, istate, iloc, ppsi, flag); } @@ -251,7 +250,7 @@ void KBPsiMatrixSparse::computeKBpsi( void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -266,8 +265,8 @@ void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) // loop over states to multiply kbpsi_[st][gid] and kbBpsi_[st][gid] // by coeff - (*kbpsimat_).scaleRow(gid, coeff); - if (lapop_) (*kbBpsimat_).scaleRow(gid, coeff); + kbpsimat_->scaleRow(gid, coeff); + if (lapop_) kbBpsimat_->scaleRow(gid, coeff); } } } @@ -454,7 +453,7 @@ SquareSubMatrix KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, Aij); } @@ -473,7 +472,7 @@ void KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, mat); } @@ -538,14 +537,14 @@ void KBPsiMatrixSparse::getPsiKBPsiSym( { // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { getPsiKBPsiSym(*ion, sm); } } template -void KBPsiMatrixSparse::computeAll(Ions& ions, T& orbitals) +void KBPsiMatrixSparse::computeAll(const Ions& ions, T& orbitals) { assert(count_proj_subdomain_ == ions.countProjectorsSubdomain()); @@ -603,7 +602,7 @@ double KBPsiMatrixSparse::getEvnl( double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -618,10 +617,9 @@ double KBPsiMatrixSparse::getEvnl( /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -637,7 +635,7 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -652,10 +650,9 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -736,12 +733,12 @@ double KBPsiMatrixSparse::getTraceDM( return trace; } -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, LocGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, LocGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, LocGridOrbitals&); -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, ExtendedGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, ExtendedGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, ExtendedGridOrbitals&); diff --git a/src/KBPsiMatrixSparse.h b/src/KBPsiMatrixSparse.h index 91f6c583..073fbddc 100644 --- a/src/KBPsiMatrixSparse.h +++ b/src/KBPsiMatrixSparse.h @@ -12,7 +12,9 @@ #include "DataDistribution.h" #include "DensityMatrixSparse.h" +#include "Ions.h" #include "KBPsiMatrixInterface.h" +#include "ProjectedMatricesSparse.h" #include "SquareSubMatrix.h" #include "VariableSizeMatrix.h" @@ -23,11 +25,6 @@ #include #include -class Ions; -class Ion; -class ProjectedMatricesInterface; -class ProjectedMatricesSparse; - class KBPsiMatrixSparse : public KBPsiMatrixInterface { static Timer global_sum_tm_; @@ -79,8 +76,8 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface void getPsiKBPsiSym(const Ions& ions, VariableSizeMatrix& sm); void getPsiKBPsiSym(const Ion& ion, VariableSizeMatrix& sm); template - void computeKBpsi(Ions& ions, OrbitalsType& orbitals, const int first_color, - const int nb_colors, const bool flag); + void computeKBpsi(const Ions& ions, OrbitalsType& orbitals, + const int first_color, const int nb_colors, const bool flag); void clearData(); public: @@ -105,7 +102,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface double getEvnl( const Ions& ions, ProjectedMatrices* proj_matrices); void computeKBpsi( - Ions& ions, pb::GridFunc*, const int, const bool flag); + const Ions& ions, pb::GridFunc*, const int, const bool flag); double getValIonState(const int gid, const int st) const { return (*kbpsimat_).get_value(gid, st); @@ -127,7 +124,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface const Ions& ions, ProjectedMatricesInterface* proj_matrices) const; template - void computeAll(Ions& ions, T& orbitals); + void computeAll(const Ions& ions, T& orbitals); void setup(const Ions& ions); double getTraceDM( diff --git a/src/MGmol.cc b/src/MGmol.cc index c18e58c7..8d516b24 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -358,8 +358,8 @@ int MGmol::initial() } // Initialize the nuclear local potential and the compensating charges - if (ct.verbose > 0) printWithTimeStamp("initNuc()...", os_); - initNuc(*ions_); + if (ct.verbose > 0) printWithTimeStamp("setupPotentials()...", os_); + setupPotentials(*ions_); // initialize Rho if (ct.verbose > 0) printWithTimeStamp("Initialize Rho...", os_); @@ -415,9 +415,6 @@ int MGmol::initial() current_orbitals_->checkCond(100000., ct.AtomsMove()); } - if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); - if (ct.restart_info == 0) { if (ct.verbose > 0) printWithTimeStamp("update_pot...", os_); @@ -817,7 +814,7 @@ double get_trilinval(const double xc, const double yc, const double zc, #endif template -void MGmol::initNuc(Ions& ions) +void MGmol::setupPotentials(Ions& ions) { init_nuc_tm_.start(); @@ -829,9 +826,12 @@ void MGmol::initNuc(Ions& ions) // initialize poentials based on ionic positions and their species pot.initialize(ions); + if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); + g_kbpsi_->setup(*ions_); + electrostat_->setupRhoc(pot.rho_comp()); - if (onpe0 && ct.verbose > 3) os_ << " initNuc done" << std::endl; + if (onpe0 && ct.verbose > 3) os_ << " setupPotentials done" << std::endl; init_nuc_tm_.stop(); } @@ -1428,7 +1428,7 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); @@ -1450,7 +1450,7 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); // initialize electronic density rho_->update(*dorbitals); diff --git a/src/MGmol.h b/src/MGmol.h index 350e8511..12197dfd 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -220,7 +220,7 @@ class MGmol : public MGmolInterface void getAtomicNumbers(std::vector& an); - void initNuc(Ions& ions); + void setupPotentials(Ions& ions); void initKBR(); void printEigAndOcc(); diff --git a/src/md.cc b/src/md.cc index b1ffb619..edccdb17 100644 --- a/src/md.cc +++ b/src/md.cc @@ -59,7 +59,7 @@ void MGmol::moveVnuc(Ions& ions) // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); - initNuc(ions); + setupPotentials(ions); pot.axpVcompToVh(-1.); proj_matrices_->setHiterativeIndex(-1, -1); diff --git a/src/quench.cc b/src/quench.cc index 59a16d50..073798f6 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -545,7 +545,6 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, // get actual indexes of stored functions const std::vector>& gids(orbitals.getOverlappingGids()); - g_kbpsi_->setup(*ions_); electrostat_->setup(ct.vh_its); rho_->setup(ct.getOrthoType(), gids); From 14eb43ff48c5f0eaf6ccad0e819536dcc3e08ee6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 21 Mar 2025 07:54:45 -0400 Subject: [PATCH 32/53] Added functionalities to set local forces (#321) --- src/Ions.cc | 39 +++++++++++++++++++++++++++++++++++++++ src/Ions.h | 9 +++++++++ tests/testIons.cc | 45 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index bc30417b..10baad10 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1288,6 +1288,29 @@ void Ions::writeForces(HDFrestart& h5f_file) } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& names) +{ + assert(forces.size() == 3 * names.size()); + + // loop over global list of forces and atom names + std::vector::const_iterator s = names.begin(); + for (auto it = forces.begin(); it != forces.end(); it += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + if (ion->compareName(*s)) + { + ion->set_force(0, *it); + ion->set_force(1, *(it + 1)); + ion->set_force(2, *(it + 2)); + } + } + s++; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { @@ -2187,6 +2210,22 @@ void Ions::getLocalPositions(std::vector& tau) const } } +void Ions::getLocalNames(std::vector& names) const +{ + for (auto& ion : local_ions_) + { + names.push_back(ion->name()); + } +} + +void Ions::getNames(std::vector& names) const +{ + for (auto& ion : list_ions_) + { + names.push_back(ion->name()); + } +} + void Ions::getPositions(std::vector& tau) { std::vector tau_local(3 * local_ions_.size()); diff --git a/src/Ions.h b/src/Ions.h index b03972d1..33b8982b 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -285,11 +285,20 @@ class Ions const std::vector& tau, const std::vector& anumbers); void getLocalPositions(std::vector& tau) const; + void getLocalNames(std::vector& names) const; + void getNames(std::vector& names) const; void getPositions(std::vector& tau); void getAtomicNumbers(std::vector& atnumbers); void getForces(std::vector& forces); void getLocalForces(std::vector& tau) const; + + /*! + * set forces for ions in local_ions_ based on names matching + */ + void setLocalForces(const std::vector& forces, + const std::vector& names); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/tests/testIons.cc b/tests/testIons.cc index 6bfdd74a..e23e0e24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -8,6 +8,8 @@ int main(int argc, char** argv) { + int status = 0; + int mpirc = MPI_Init(&argc, &argv); MPI_Comm comm = MPI_COMM_WORLD; @@ -92,7 +94,7 @@ int main(int argc, char** argv) if (ntotal != na) { std::cout << "ntotal = " << ntotal << std::endl; - return 1; + status = 1; } } MPI_Barrier(MPI_COMM_WORLD); @@ -141,16 +143,49 @@ int main(int argc, char** argv) MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); if (ntotal != na) { - std::cout << "ntotal = " << ntotal << std::endl; - return 1; + std::cerr << "ntotal = " << ntotal << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + + // get the names of all the ions + std::vector names; + ions.getNames(names); + if (myrank == 0) + for (auto& name : names) + std::cout << "name = " << name << std::endl; + if (names.size() != na) + { + std::cerr << "Incorrect count of names..." << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + MPI_Barrier(MPI_COMM_WORLD); + + std::vector forces(3 * na); + // arbitrary value + const double fval = 1.12; + for (auto& f : forces) + f = fval; + ions.setLocalForces(forces, names); + + int nlocal = ions.getNumLocIons(); + std::vector lforces(3 * nlocal); + ions.getLocalForces(lforces); + for (auto& f : lforces) + { + if (std::abs(f - fval) > 1.e-14) + { + std::cerr << "f = " << f << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { std::cerr << "MPI Finalize failed!!!" << std::endl; - return 1; + status = 1; } - return 0; + return status; } From c61b48d2dabcdbe291b1545eeba338c56889cc40 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 24 Mar 2025 13:35:03 -0400 Subject: [PATCH 33/53] Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code --- CMakeLists.txt | 15 +- scripts/build_ubuntu22_mixedp.sh | 32 ++ src/BlockVector.cc | 4 +- src/ExtendedGridOrbitals.cc | 29 +- src/LocGridOrbitals.cc | 14 +- src/MGmol.cc | 2 + src/Rho.cc | 2 +- src/global.h | 2 +- src/linear_algebra/CMakeLists.txt | 2 +- src/linear_algebra/MGmol_blas1.h | 14 +- src/linear_algebra/gemm_impl.cc | 251 ++++++++++++++ src/linear_algebra/gemm_impl.h | 14 + src/linear_algebra/mputils.cc | 550 ++++++++---------------------- src/linear_algebra/mputils.h | 23 +- src/linear_algebra/syrk_impl.cc | 227 ++++++++++++ src/linear_algebra/syrk_impl.h | 13 + src/mgmol_run.cc | 8 + tests/CMakeLists.txt | 30 ++ 18 files changed, 768 insertions(+), 464 deletions(-) create mode 100755 scripts/build_ubuntu22_mixedp.sh create mode 100644 src/linear_algebra/gemm_impl.cc create mode 100644 src/linear_algebra/gemm_impl.h create mode 100644 src/linear_algebra/syrk_impl.cc create mode 100644 src/linear_algebra/syrk_impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 753a455d..e931b5cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,12 @@ else() find_package(OpenMP) endif() +# Use mixed precision +set(MGMOL_WITH_MP FALSE CACHE BOOL "Compile with mixed precision") +if(MGMOL_WITH_MP) + add_definitions(-DMGMOL_USE_MIXEDP) +endif() + # HDF5 (required) set(MGMOL_USE_HDF5P TRUE CACHE BOOL "Use HDF5 parallel capability") if(HDF5_LIBRARIES) #user sets libraries explicitly @@ -100,9 +106,16 @@ if(NOT BLAS_LIBRARIES) message(FATAL_ERROR "Required blas library not found.") endif(NOT ${BLAS_FOUND}) else(NOT BLAS_LIBRARIES) - message(STATUS "BLAS_LIBARIES: ${BLAS_LIBRARIES}") + message(STATUS "BLAS_LIBARIES set to ${BLAS_LIBRARIES}") endif(NOT BLAS_LIBRARIES) +set(MGMOL_WITH_BLIS FALSE CACHE BOOL "Use BLIS library") +if(MGMOL_WITH_BLIS) + add_definitions(-DMGMOL_USE_BLIS) + message(STATUS "BLIS_LIBRARIES include: ${BLIS_INCLUDE_DIRS}") + include_directories("${BLIS_INCLUDE_DIRS}") +endif() + if(NOT LAPACK_LIBRARIES) find_package(LAPACK REQUIRED) message(STATUS "LAPACK_LIBARIES: ${LAPACK_LIBRARIES}") diff --git a/scripts/build_ubuntu22_mixedp.sh b/scripts/build_ubuntu22_mixedp.sh new file mode 100755 index 00000000..d85ab9a5 --- /dev/null +++ b/scripts/build_ubuntu22_mixedp.sh @@ -0,0 +1,32 @@ +#/bin/bash +MGMOL_ROOT=`pwd` + +INSTALL_DIR=${MGMOL_ROOT}/mgmol_install +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_COMPILER=mpiCC.openmpi \ + -DMGMOL_WITH_BLIS=ON \ + -DBLAS_LIBRARIES=/home/q8j/blis/lib/libblis.so \ + -DBLIS_INCLUDE_DIRS="/home/q8j/blis//include" \ + -DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/openblas-pthread/liblapack.so \ + -DCMAKE_Fortran_COMPILER=mpif77.openmpi \ + -DSCALAPACK_LIBRARY=/usr/lib/x86_64-linux-gnu/libscalapack-openmpi.so.2.1 \ + -DMPIEXEC_EXECUTABLE=/usr/bin/mpirun \ + -DMPIEXEC_NUMPROC_FLAG="-np" \ + -DMPIEXEC_PREFLAGS="--oversubscribe" \ + -DMGMOL_WITH_CLANG_FORMAT=ON \ + -DMGMOL_WITH_MP=ON \ + -DCMAKE_PREFIX_PATH=${HOME}/bin \ + -D CMAKE_CXX_FLAGS="-Wall" \ + .. + +# call make install +make -j 4 +make install diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 9a411ebe..671ba90d 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -556,7 +556,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); @@ -586,7 +586,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 07face7c..3e887dad 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -926,10 +926,10 @@ void ExtendedGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(numst_, nf, loc_numpt_, 1., - orbitals_psi_host_view + iloc * loc_numpt_, lda_, - work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * numst_, - numst_); + LinearAlgebraUtils::MPgemmTN(numst_, nf, + loc_numpt_, 1., orbitals_psi_host_view + iloc * loc_numpt_, + lda_, work + iloc * loc_numpt_, lda_, 0., + ssiloc + icolor * numst_, numst_); } } @@ -970,7 +970,7 @@ void ExtendedGridOrbitals::getLocalOverlap( if (numst_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else ORBDTYPE* psi = block_vector_.vect(0); @@ -1041,21 +1041,12 @@ void ExtendedGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, const int lda = transpose ? ld : lda_; const int ldb = transpose ? lda_ : ld; -#ifdef USE_MP - // use temporary float data for matrix ss - LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); -#else - LocalMatrices& ssf(ss); -#endif for (short iloc = 0; iloc < subdivx_; iloc++) { - LinearAlgebraUtils::MPgemm('T', 'N', numst_, numst_, + LinearAlgebraUtils::MPgemmTN(numst_, numst_, loc_numpt_, 1., a + iloc * loc_numpt_, lda, b + +iloc * loc_numpt_, - ldb, 0., ssf.getRawPtr(iloc), ssf.m()); + ldb, 0., ss.getRawPtr(iloc), ss.m()); } -#ifdef USE_MP - ss.copy(ssf); -#endif ss.scal(grid_.vel()); } @@ -1709,9 +1700,9 @@ void ExtendedGridOrbitals::addDotWithNcol2Matrix( Apsi.getPsi(0, iloc), phi_size, phi_host_view); // TODO this can be done on the GPU - MPgemmTN(numst_, numst_, loc_numpt_, vel, - block_vector_host_view + iloc * loc_numpt_, lda_, phi_host_view, - lda_, 1., work.data(), numst_); + LinearAlgebraUtils::MPgemmTN(numst_, numst_, + loc_numpt_, vel, block_vector_host_view + iloc * loc_numpt_, lda_, + phi_host_view, lda_, 1., work.data(), numst_); MemorySpace::Memory::free_host_view( phi_host_view); diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index 4a1b9442..3d14e051 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -1448,8 +1448,8 @@ void LocGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(chromatic_number_, nf, loc_numpt_, 1., - orbitals_psi + iloc * loc_numpt_, lda_, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + nf, loc_numpt_, 1., orbitals_psi + iloc * loc_numpt_, lda_, work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * chromatic_number_, chromatic_number_); } @@ -1490,7 +1490,7 @@ void LocGridOrbitals::getLocalOverlap( if (chromatic_number_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else const ORBDTYPE* const psi = block_vector_.vect(0); @@ -1560,7 +1560,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, MemorySpace::Memory::copy_view_to_host( const_cast(b), b_size, b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP // use temporary float data for matrix ss LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); #else @@ -1575,7 +1575,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, a_host_view); MemorySpace::Memory::free_host_view( b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP ss.copy(ssf); #endif @@ -2495,8 +2495,8 @@ void LocGridOrbitals::addDotWithNcol2Matrix( { MATDTYPE* ssiloc = ss.getRawPtr(iloc); - // TODO - MPgemmTN(chromatic_number_, chromatic_number_, loc_numpt_, vel, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + chromatic_number_, loc_numpt_, vel, block_vector_.vect(0) + iloc * loc_numpt_, lda_, Apsi.getPsi(0, iloc), lda_, 0., ssiloc, chromatic_number_); } diff --git a/src/MGmol.cc b/src/MGmol.cc index 8d516b24..4d69387d 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -87,6 +87,7 @@ extern Timer sgemm_tm; extern Timer dgemm_tm; extern Timer mpgemm_tm; extern Timer tttgemm_tm; +extern Timer bligemm_tm; extern Timer dsyrk_tm; extern Timer ssyrk_tm; extern Timer mpsyrk_tm; @@ -864,6 +865,7 @@ void MGmol::printTimers() dgemm_tm.print(os_); mpgemm_tm.print(os_); tttgemm_tm.print(os_); + bligemm_tm.print(os_); ssyrk_tm.print(os_); dsyrk_tm.print(os_); diff --git a/src/Rho.cc b/src/Rho.cc index 45a41f3e..e80e74fe 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -588,7 +588,7 @@ Rho::computeRho>( ExtendedGridOrbitals&, const dist_matrix::DistMatrix&); template void Rho::computeRho>( LocGridOrbitals&, const dist_matrix::DistMatrix&); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template double Rho::dotWithRho( const float* const func) const; #endif diff --git a/src/global.h b/src/global.h index de6c8559..a23e4e3b 100644 --- a/src/global.h +++ b/src/global.h @@ -15,7 +15,7 @@ //#include "mgmol_memory.h" -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP typedef float ORBDTYPE; #else typedef double ORBDTYPE; diff --git a/src/linear_algebra/CMakeLists.txt b/src/linear_algebra/CMakeLists.txt index 8bd5ef2f..0178f539 100644 --- a/src/linear_algebra/CMakeLists.txt +++ b/src/linear_algebra/CMakeLists.txt @@ -1,4 +1,4 @@ -set(SOURCES mputils.cc) +set(SOURCES mputils.cc gemm_impl.cc syrk_impl.cc) add_library(mgmol_linear_algebra ${SOURCES}) target_link_libraries(mgmol_linear_algebra PUBLIC MPI::MPI_CXX) if(${MGMOL_WITH_MAGMA}) diff --git a/src/linear_algebra/MGmol_blas1.h b/src/linear_algebra/MGmol_blas1.h index 41204226..32431468 100644 --- a/src/linear_algebra/MGmol_blas1.h +++ b/src/linear_algebra/MGmol_blas1.h @@ -11,6 +11,9 @@ #define MGMOL_MYBLAS1_H #include "fc_mangle.h" +#ifdef MGMOL_USE_BLIS +#include +#endif #include #include @@ -18,6 +21,8 @@ #define MY_VERSION 0 #define EPSILON 1.e-12 +#ifndef MGMOL_USE_BLIS + #ifdef __cplusplus extern "C" { @@ -42,12 +47,17 @@ extern "C" float SNRM2(const int* const, const float* const, const int* const); int IDAMAX(const int* const, const double* const, const int* const); int ISAMAX(const int* const, const float* const, const int* const); - void DROT(int*, double*, int*, double*, int*, double*, double*); - void SROT(int*, float*, int*, float*, int*, float*, float*); + void DROT(const int*, double*, const int*, double*, const int*, + const double*, const double*); + void SROT(const int*, float*, const int*, float*, const int*, const float*, + const float*); + #ifdef __cplusplus } #endif +#endif // MGMOL_USE_BLIS + inline void my_daxpy( const int n, const double alpha, const double* const a, double* b) { diff --git a/src/linear_algebra/gemm_impl.cc b/src/linear_algebra/gemm_impl.cc new file mode 100644 index 00000000..3500410c --- /dev/null +++ b/src/linear_algebra/gemm_impl.cc @@ -0,0 +1,251 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "gemm_impl.h" +#include "Timer.h" +#include "mputils.h" + +#include +#include + +Timer mpgemm_tm("mpgemm"); +Timer tttgemm_tm("tttgemm"); + +using LAU_H = LinearAlgebraUtils; + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) +{ + tttgemm_tm.start(); + // std::cout<<"template MPgemm..."< buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const T2* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = static_cast(c[pos]) * beta; + const T1* __restrict__ ai = a + lda * i; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const T1* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (T3)sum; + } + } + } + } + + tttgemm_tm.stop(); +} + +// input/output in float, computation in double +template <> +void gemm_impl(const char transa, const char transb, + const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const float* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + mpgemm_tm.start(); + + if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; + + /* case transb == 'N' and transa == 'N' */ + if (transb == 'N' || transb == 'n') + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = (double)c[pos] * beta; + const float* __restrict__ ai = a + lda * i; + c[pos] = (float)(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const float* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (float)sum; + } + } + } + } + + mpgemm_tm.stop(); +} + +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, + const int ldc); diff --git a/src/linear_algebra/gemm_impl.h b/src/linear_algebra/gemm_impl.h new file mode 100644 index 00000000..5140b143 --- /dev/null +++ b/src/linear_algebra/gemm_impl.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc); diff --git a/src/linear_algebra/mputils.cc b/src/linear_algebra/mputils.cc index 77b08689..487d1367 100644 --- a/src/linear_algebra/mputils.cc +++ b/src/linear_algebra/mputils.cc @@ -16,19 +16,27 @@ #include #endif +#include "MGmol_blas1.h" +#include "gemm_impl.h" +#include "syrk_impl.h" + +#ifdef MGMOL_USE_BLIS +#include +#else +#include "blas2_c.h" +#include "blas3_c.h" +#endif + #include #include #include Timer dgemm_tm("dgemm"); Timer sgemm_tm("sgemm"); -Timer mpgemm_tm("mpgemm"); -Timer tttgemm_tm("tttgemm"); +Timer bligemm_tm("bligemm"); Timer dsyrk_tm("dsyrk"); Timer ssyrk_tm("ssyrk"); -Timer mpsyrk_tm("mpsyrk"); -Timer tttsyrk_tm("tttsyrk"); Timer mpdot_tm("mpdot"); Timer ttdot_tm("ttdot"); @@ -304,106 +312,6 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, dsyrk_tm.stop(); } -template <> -void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, - const double alpha, const float* const a, const int lda, const double beta, - float* c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(c); - - mpsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - mpsyrk_tm.stop(); -} - template <> template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, @@ -413,97 +321,7 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, MemorySpace::assert_is_host_ptr(a); MemorySpace::assert_is_host_ptr(c); - tttsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - - tttsyrk_tm.stop(); + syrk_impl(uplo, trans, n, k, alpha, a, lda, beta, c, ldc); } // MemorySpace::Device @@ -725,103 +543,7 @@ void LAU_H::MPgemm(const char transa, const char transb, const int m, MemorySpace::assert_is_host_ptr(b); MemorySpace::assert_is_host_ptr(c); - tttgemm_tm.start(); - // if(onpe0)cout<<"template MPgemm..."< buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const T2* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = static_cast(c[pos]) * beta; - const T1* __restrict__ ai = a + lda * i; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const T1* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (T3)sum; - } - } - } - } - - tttgemm_tm.stop(); + gemm_impl(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } // input/output in double, computation in double @@ -842,111 +564,6 @@ void LAU_H::MPgemm(const char transa, const char transb, dgemm_tm.stop(); } -// input/output in float, computation in double -template <> -template <> -void LAU_H::MPgemm(const char transa, const char transb, - const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(b); - MemorySpace::assert_is_host_ptr(c); - - mpgemm_tm.start(); - - if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; - - /* case transb == 'N' and transa == 'N' */ - if (transb == 'N' || transb == 'n') - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = (double)c[pos] * beta; - const float* __restrict__ ai = a + lda * i; - c[pos] = (float)(alpha * MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const float* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (float)sum; - } - } - } - } - - mpgemm_tm.stop(); -} - // MemorySpace::Device #ifdef HAVE_MAGMA template <> @@ -1014,10 +631,6 @@ void LAU_D::MPgemm(const char transa, const char transb, const int m, } #endif -/////////////////////////////// -// MPgemmNN // -/////////////////////////////// - template template void LinearAlgebraUtils::MPgemmNN(const int m, const int n, @@ -1031,19 +644,122 @@ void LinearAlgebraUtils::MPgemmNN(const int m, const int n, transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } +template <> +template <> +void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const double* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, k, const_cast(a), 1, lda, &A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, k, n, const_cast(b), 1, ldb, &B); + + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else + char transa = 'n'; + char transb = 'n'; + + LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif +} + // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc) { + // std::cout << "LAU_H::MPgemmTN" << std::endl; + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, m, const_cast(a), 1, lda, &A); + bli_obj_toggle_trans(&A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, n, const_cast(b), 1, ldb, &B); + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + // dafault: precision of C + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else char transa = 't'; char transb = 'n'; LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif } // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, float* const c, const int ldc) { @@ -1053,12 +769,12 @@ void MPgemmTN(const int m, const int n, const int k, const double alpha, LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } -/////// additional calls ... may be removed later if unused - +template template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc) +void LinearAlgebraUtils::MPgemmTN(const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) { // if(onpe0)cout<<"template MPgemmNN..."<(const char transa, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc); -template void LAU_H::MPgemmNN(const int m, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double* const b, const int ldb, const double beta, float* const c, +template void LAU_H::MPgemm(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, const int ldc); + +// template void LAU_H::MPgemmNN(const int m, const int n, +// const int k, const double alpha, const float* const a, const int lda, +// const double* const b, const int ldb, const double beta, float* const c, +// const int ldc); template void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, @@ -1127,14 +849,18 @@ template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, double* __restrict__ yptr); template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, float* __restrict__ yptr); + template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, float* c, const int ldc); template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const float* const a, const int lda, const double beta, double* c, const int ldc); +template void LAU_H::MPsyrk(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, float* c, const int ldc); -template void MPgemmTN(const int m, const int n, +template void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, const int ldc); diff --git a/src/linear_algebra/mputils.h b/src/linear_algebra/mputils.h index 17bd0b4b..fa838f17 100644 --- a/src/linear_algebra/mputils.h +++ b/src/linear_algebra/mputils.h @@ -11,8 +11,6 @@ #define MGMOL_MPUTILS_H #include "MGmol_blas1.h" -#include "blas2_c.h" -#include "blas3_c.h" #include "memory_space.h" /* scal */ @@ -100,6 +98,11 @@ struct LinearAlgebraUtils const double alpha, const T1* const a, const int lda, const T2* const b, const int ldb, const double beta, T3* const c, const int ldc); + template + static void MPgemmTN(const int m, const int n, const int k, + const double alpha, const T1* const a, const int lda, const T2* const b, + const int ldb, const double beta, T3* const c, const int ldc); + /* mixed-precision scalar times vector. Accumulates results * in double precision and stores as single precision. */ @@ -125,28 +128,12 @@ struct LinearAlgebraUtils static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, double* c, const int ldc); - static void MPsyrk(const char uplo, const char trans, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double beta, float* c, const int ldc); template static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const T1* const a, const int lda, const double beta, T2* c, const int ldc); }; -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, double* const c, const int ldc); - -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc); - -template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc); - /* trsm */ void Ttrsm(const char, const char, const char, const char, const int, const int, const double, const double* const, const int, double* const, const int); diff --git a/src/linear_algebra/syrk_impl.cc b/src/linear_algebra/syrk_impl.cc new file mode 100644 index 00000000..e1a4833f --- /dev/null +++ b/src/linear_algebra/syrk_impl.cc @@ -0,0 +1,227 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Timer.h" +#include "mputils.h" + +#include "MGmol_blas1.h" +#include "syrk_impl.h" + +#include "blas3_c.h" + +#include +#include +#include + +Timer mpsyrk_tm("mpsyrk"); +Timer tttsyrk_tm("tttsyrk"); + +using LAU_H = LinearAlgebraUtils; + +template <> +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const float* const a, const int lda, const double beta, + float* c, const int ldc) +{ + mpsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + mpsyrk_tm.stop(); +} + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc) +{ + tttsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + + tttsyrk_tm.stop(); +} + +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const double* const a, + const int lda, const double beta, float* c, const int ldc); +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, double* c, const int ldc); diff --git a/src/linear_algebra/syrk_impl.h b/src/linear_algebra/syrk_impl.h new file mode 100644 index 00000000..ab3786f4 --- /dev/null +++ b/src/linear_algebra/syrk_impl.h @@ -0,0 +1,13 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc); diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index 468d61bd..92390e83 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -20,6 +20,10 @@ #include #include +#ifdef MGMOL_USE_BLIS +#include +#endif + #ifdef _OPENMP #include #endif @@ -39,6 +43,10 @@ int mgmol_init(MPI_Comm comm) assert(mype > -1); MPIdata::onpe0 = (MPIdata::mype == 0); +#ifdef MGMOL_USE_BLIS + bli_init(); +#endif + #ifdef HAVE_MAGMA magma_int_t magmalog = magma_init(); if (magmalog == MAGMA_SUCCESS) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 838ce6a4..df774171 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,8 @@ add_executable(testDistVector ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -32,6 +34,8 @@ add_executable(testReplicated2DistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -43,6 +47,8 @@ add_executable(testDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -54,6 +60,8 @@ add_executable(testConditionDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -68,6 +76,8 @@ add_executable(testConditionDistMatrixPower ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -79,6 +89,8 @@ add_executable(testPower ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -93,6 +105,8 @@ add_executable(testPowerDistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -123,6 +137,8 @@ add_executable(testVariableSizeMatrix ${CMAKE_SOURCE_DIR}/src/sparse_linear_algebra/Table.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -138,6 +154,8 @@ add_executable(testTradeGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -153,6 +171,8 @@ add_executable(testSetGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -172,6 +192,8 @@ add_executable(testLaph4 ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -189,6 +211,8 @@ add_executable(testBatchLaph4 ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -204,6 +228,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -225,6 +251,8 @@ add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -244,6 +272,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc From e37983b9a11a5ffca035254f0348fcbc246a5a75 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 09:46:33 -0400 Subject: [PATCH 34/53] Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private --- src/Ion.cc | 2 + src/Ion.h | 2 + src/Ions.cc | 109 +++++++++++++++++----------------------------- src/Ions.h | 31 +++++++------ tests/testIons.cc | 4 ++ 5 files changed, 66 insertions(+), 82 deletions(-) diff --git a/src/Ion.cc b/src/Ion.cc index 90350477..896a1961 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -24,6 +24,8 @@ unsigned short isqrt(unsigned value) static unsigned int _nlproj_gid = 0; static unsigned int _index = 0; +void Ion::resetIndexCount() { _index = 0; } + Ion::Ion(const Species& species, const std::string& name, const double crds[3], const double velocity[3], const bool lock) : name_(name), species_(species), index_(_index), nlproj_gid_(_nlproj_gid) diff --git a/src/Ion.h b/src/Ion.h index 871205a3..cf52db97 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -314,6 +314,8 @@ class Ion void getKBcoeffs(std::vector& coeffs); double energyDiff( Ion& ion, const double lattice[3], const short bc[3]) const; + + static void resetIndexCount(); }; #endif diff --git a/src/Ions.cc b/src/Ions.cc index 10baad10..f10fb834 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -9,7 +9,6 @@ #include "Ions.h" #include "Control.h" -#include "HDFrestart.h" #include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "MPIdata.h" @@ -20,13 +19,10 @@ #include "mgmol_mpi_tools.h" #include "tools.h" -#include - #include #include #include -#include -#include +#include Timer ions_setupInteractingIons_tm("ions_setupInteractingIons"); Timer ions_setup_tm("ions::setup"); @@ -187,6 +183,8 @@ void Ions::setup() ions_setup_tm.start(); + Ion::resetIndexCount(); + updateListIons(); //#ifndef NDEBUG @@ -394,7 +392,7 @@ void Ions::iiforce(const short bc[3]) std::vector::const_iterator ion1 = local_ions_.begin(); int ion1_index = 0; - ; + while (ion1 != local_ions_.end()) { const double z1 = (*ion1)->getZion(); @@ -1800,7 +1798,7 @@ int Ions::readAtomsFromXYZ( } int Ions::setAtoms( - const std::vector& crds, const std::vector& spec) + const std::vector& crds, const std::vector& anum) { const int natoms = crds.size() / 3; @@ -1813,7 +1811,7 @@ int Ions::setAtoms( while (it != species_.end()) { ++isp; - if (it->getAtomicNumber() == spec[ia]) + if (it->getAtomicNumber() == anum[ia]) { break; } @@ -1823,7 +1821,7 @@ int Ions::setAtoms( for (std::map::iterator itr = map_species_.begin(); itr != map_species_.end(); ++itr) { - if (itr->second == spec[ia]) + if (itr->second == anum[ia]) { spname = itr->first; break; @@ -1833,7 +1831,7 @@ int Ions::setAtoms( { (*MPIdata::serr) << "Ions::setAtoms() --- ERROR: unknown " "species for atomic number " - << spec[ia] << std::endl; + << anum[ia] << std::endl; return -1; } @@ -2212,6 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { + names.clear(); for (auto& ion : local_ions_) { names.push_back(ion->name()); @@ -2220,6 +2219,7 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { + names.clear(); for (auto& ion : list_ions_) { names.push_back(ion->name()); @@ -2513,7 +2513,6 @@ void Ions::gatherLockedNames( std::vector& names, const int root, const MPI_Comm comm) const { std::vector local_names; - for (auto& ion : local_ions_) { if (ion->locked()) local_names.push_back(ion->name()); @@ -2533,7 +2532,6 @@ void Ions::gatherIndexes( std::vector& indexes, const int root, const MPI_Comm comm) const { std::vector local_indexes; - for (auto& ion : local_ions_) { local_indexes.push_back(ion->index()); @@ -2553,7 +2551,6 @@ void Ions::gatherNLprojIds( std::vector& nlprojids, const int root, const MPI_Comm comm) const { std::vector local_nlprojids; - for (auto& ion : local_ions_) { local_nlprojids.push_back(ion->nlprojid()); @@ -2573,7 +2570,6 @@ void Ions::gatherAtomicNumbers( std::vector& atnumbers, const int root, const MPI_Comm comm) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2593,7 +2589,6 @@ void Ions::gatherRandStates(std::vector& rstates, const int root, const MPI_Comm comm) const { std::vector local_rstates; - for (auto& ion : local_ions_) { local_rstates.push_back(ion->randomState(0)); @@ -2615,10 +2610,8 @@ void Ions::gatherPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { - // get position of local ion double position[3]; ion->getPosition(&position[0]); local_positions.push_back(position[0]); @@ -2640,7 +2633,6 @@ void Ions::gatherPreviousPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { local_positions.push_back(ion->getPreviousPosition(0)); @@ -2662,17 +2654,14 @@ void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { std::vector local_forces; - for (auto& ion : local_ions_) { - // get position of local ion double force[3]; ion->getForce(&force[0]); local_forces.push_back(force[0]); local_forces.push_back(force[1]); local_forces.push_back(force[2]); } - // gather data to PE root std::vector data; mgmol_tools::gatherV(local_forces, data, root, comm); @@ -2687,7 +2676,6 @@ void Ions::gatherVelocities( std::vector& velocities, const int root, const MPI_Comm comm) const { std::vector local_velocities; - for (auto& ion : local_ions_) { local_velocities.push_back(ion->velocity(0)); @@ -2711,13 +2699,10 @@ void Ions::gatherPositions(std::vector& positions, const int root) const positions.resize(3 * num_ions_, 0.); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local positions - const int index = (*ion)->index(); - (*ion)->getPosition(&positions[3 * index]); - ++ion; + const int index = ion->index(); + ion->getPosition(&positions[3 * index]); } // gather data to PE root @@ -2726,17 +2711,17 @@ void Ions::gatherPositions(std::vector& positions, const int root) const void Ions::gatherForces(std::vector& forces, const int root) const { + assert(num_ions_ > 0); + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); forces.resize(3 * num_ions_, 0.); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local forces - const int index = (*ion)->index(); - (*ion)->getForce(&forces[3 * index]); - ion++; + const int index = ion->index(); + assert(forces.size() >= 3 * index); + assert(index < num_ions_); + ion->getForce(&forces[3 * index]); } // gather data to PE root @@ -2749,15 +2734,10 @@ void Ions::gatherLockedData(std::vector& locked_data, const int root) const MGmol_MPI& mmpi(*(MGmol_MPI::instance())); locked_data.resize(num_ions_, 0); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local ion index - const int index = (*ion)->index(); - if ((*ion)->locked()) locked_data[index] = 1; - - ++ion; + const int index = ion->index(); + if (ion->locked()) locked_data[index] = 1; } // gather data to PE root @@ -3214,11 +3194,9 @@ void Ions::updateTaupInteractingIons() void Ions::clearLists() { local_ions_.clear(); - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - delete *ion; - ion++; + delete ion; } list_ions_.clear(); } @@ -3335,24 +3313,21 @@ void Ions::initStepperData() { clearStepperData(); - std::vector::iterator lion = local_ions_.begin(); - while (lion != local_ions_.end()) + for (auto& lion : local_ions_) { - local_names_.push_back((*lion)->name()); - atmove_.push_back(!(*lion)->locked()); - pmass_.push_back((*lion)->getMass()); - gids_.push_back((*lion)->index()); + local_names_.push_back(lion->name()); + atmove_.push_back(!lion->locked()); + pmass_.push_back(lion->getMass()); + gids_.push_back(lion->index()); for (short i = 0; i < 3; i++) { - taum_.push_back((*lion)->getPreviousPosition(i)); - tau0_.push_back((*lion)->position(i)); - fion_.push_back((*lion)->force(i)); - velocity_.push_back((*lion)->velocity(i)); - rand_states_.push_back((*lion)->randomState(i)); + taum_.push_back(lion->getPreviousPosition(i)); + tau0_.push_back(lion->position(i)); + fion_.push_back(lion->force(i)); + velocity_.push_back(lion->velocity(i)); + rand_states_.push_back(lion->randomState(i)); } - - lion++; } // initialize taup to enable computing velocities int size_tau = (int)tau0_.size(); @@ -3399,11 +3374,9 @@ void Ions::updateIons() void Ions::shiftIons(const Vector3D& shift) { // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->shiftPositionXLBOMDTest(shift); - ion++; + ion->shiftPositionXLBOMDTest(shift); } // update various list of ions @@ -3421,12 +3394,10 @@ void Ions::rescaleVelocities(const double factor) std::cout << "Ions::rescaleVelocities() with factor " << factor << std::endl; } - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - (*ion)->rescaleVelocity(factor); - ion++; + for (auto& ion : local_ions_) + { + ion->rescaleVelocity(factor); } } diff --git a/src/Ions.h b/src/Ions.h index 33b8982b..3420d725 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -10,20 +10,20 @@ #ifndef MGMOL_IONS_H #define MGMOL_IONS_H -#include -#include -#include -#include -#include - #include "DistributedIonicData.h" +#include "HDFrestart.h" #include "Ion.h" -#include "hdf5.h" -class HDFrestart; +#include +#include +#include class Ions { +private: + /*! + * map species to atomic numbers + */ static std::map map_species_; static int num_ions_; @@ -37,7 +37,7 @@ class Ions std::vector list_ions_; - /* + /*! * ions located in local sub-domain */ std::vector local_ions_; @@ -61,6 +61,13 @@ class Ions bool has_locked_atoms_; + /*! + * Prevent usage of copy constructor by making it private and + * non-implemented + */ + Ions(const Ions&); + void operator=(const Ions&); + void readRestartVelocities(HDFrestart& h5_file); void readRestartRandomStates(HDFrestart& h5_file); void readRestartPositions(HDFrestart& h5_file); @@ -193,11 +200,9 @@ class Ions std::vector& getGids() { return gids_; } void resetForces() { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->resetForce(); - ion++; + ion->resetForce(); } } void resetPositionsToPrevious(); diff --git a/tests/testIons.cc b/tests/testIons.cc index e23e0e24..39c96f4a 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -166,8 +166,12 @@ int main(int argc, char** argv) const double fval = 1.12; for (auto& f : forces) f = fval; + + ions.getNames(names); ions.setLocalForces(forces, names); + ions.printForcesGlobal(std::cout); + int nlocal = ions.getNumLocIons(); std::vector lforces(3 * nlocal); ions.getLocalForces(lforces); From 89630b3fba8e8f0c998bc53433c8e4c904274b2e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 13:14:19 -0400 Subject: [PATCH 35/53] Fixes for build without HDF5P (#324) --- src/Ions.cc | 3 ++- tests/CMakeLists.txt | 18 ++++++++++-------- tests/MVP/mvp.cfg | 1 - 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index f10fb834..ed3d54e1 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -947,10 +947,11 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) num_ions_ = at_names.size(); #ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) +#endif { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } -#endif + if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index df774171..865ef354 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -547,14 +547,16 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -add_test(NAME testHDF5single - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz - ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(MGMOL_USE_HDF5P) + add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +endif() add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/mvp.cfg b/tests/MVP/mvp.cfg index bb3ab52a..868e5703 100644 --- a/tests/MVP/mvp.cfg +++ b/tests/MVP/mvp.cfg @@ -32,4 +32,3 @@ solver=MVP nb_inner_it=2 [Restart] output_level=2 -output_type=single_file From e80193b2095ded2daef4c821961653f3df5dc574 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 13:14:37 -0400 Subject: [PATCH 36/53] Fix testRhoVhRestart (#325) --- tests/RhoVhRestart/testRhoVhRestart.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc index 09da3320..ce40272d 100644 --- a/tests/RhoVhRestart/testRhoVhRestart.cc +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -46,8 +46,8 @@ int testRhoRestart(MGmolInterface* mgmol_) /* check if the recomputed density is the same */ for (int d = 0; d < (int)rho0.size(); d++) { - double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); - if (error > 1e-10) + double error = abs(rho0[d] - rho->rho_[0][d]); + if (error > 1e-10 * abs(rho0[d])) { printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, rho->rho_[0][d], d, rho0[d]); From bcad1127e11f4cedec5a21f774b661589b4eb8b1 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 14:21:30 -0400 Subject: [PATCH 37/53] Fix a few more issues with class Ions (#328) --- src/Ions.cc | 29 +++++++++++------------------ src/Ions.h | 6 +++--- tests/testIons.cc | 31 +++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index ed3d54e1..3b128b0e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -109,14 +109,12 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { - std::vector::const_iterator ion = ions.list_ions_.begin(); - while (ion != ions.list_ions_.end()) + for (const auto& ion : ions.list_ions_) { - Ion* newion = new Ion(**ion); + Ion* newion = new Ion(*ion); newion->shiftPosition(shift); newion->setup(); list_ions_.push_back(newion); - ion++; } for (short i = 0; i < 3; ++i) lattice_[i] = ions.lattice_[i]; @@ -2212,7 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { names.clear(); - for (auto& ion : local_ions_) + for (const auto& ion : local_ions_) { names.push_back(ion->name()); } @@ -2220,14 +2218,14 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { - names.clear(); - for (auto& ion : list_ions_) - { - names.push_back(ion->name()); - } + std::vector local_names; + getLocalNames(local_names); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(local_names, names); } -void Ions::getPositions(std::vector& tau) +void Ions::getPositions(std::vector& tau) const { std::vector tau_local(3 * local_ions_.size()); @@ -2237,10 +2235,9 @@ void Ions::getPositions(std::vector& tau) mmpi.allGatherV(tau_local, tau); } -void Ions::getAtomicNumbers(std::vector& atnumbers) +void Ions::getAtomicNumbers(std::vector& atnumbers) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2250,15 +2247,11 @@ void Ions::getAtomicNumbers(std::vector& atnumbers) mmpi.allGatherV(local_atnumbers, atnumbers); } -void Ions::getForces(std::vector& forces) +void Ions::getForces(std::vector& forces) const { std::vector forces_local(3 * local_ions_.size()); - getLocalForces(forces_local); - int n = getNumIons(); - forces.resize(3 * n); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allGatherV(forces_local, forces); } diff --git a/src/Ions.h b/src/Ions.h index 3420d725..3d382eb7 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -292,10 +292,10 @@ class Ions void getLocalPositions(std::vector& tau) const; void getLocalNames(std::vector& names) const; void getNames(std::vector& names) const; - void getPositions(std::vector& tau); - void getAtomicNumbers(std::vector& atnumbers); + void getPositions(std::vector& tau) const; + void getAtomicNumbers(std::vector& atnumbers) const; - void getForces(std::vector& forces); + void getForces(std::vector& forces) const; void getLocalForces(std::vector& tau) const; /*! diff --git a/tests/testIons.cc b/tests/testIons.cc index 39c96f4a..6bea5d64 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -107,6 +107,7 @@ int main(int argc, char** argv) ions.getAtomicNumbers(anumbers); if (myrank == 0) { + std::cout << "Positions:" << std::endl; int i = 0; for (auto& position : positions) { @@ -121,7 +122,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); // swap x and z - for (size_t i = 0; i < positions.size() - 2; i++) + for (size_t i = 0; i < positions.size() - 2; i += 3) { double x = positions[i]; double z = positions[i + 2]; @@ -162,11 +163,13 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); std::vector forces(3 * na); - // arbitrary value - const double fval = 1.12; + // set forces to a different arbitrary value for each component + int i = 0; for (auto& f : forces) - f = fval; - + { + f = (double)i; + i++; + } ions.getNames(names); ions.setLocalForces(forces, names); @@ -177,13 +180,29 @@ int main(int argc, char** argv) ions.getLocalForces(lforces); for (auto& f : lforces) { - if (std::abs(f - fval) > 1.e-14) + if (std::fmod(f, 1.) > 1.e-14) { std::cerr << "f = " << f << std::endl; MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + ions.getForces(forces); + if (myrank == 0) + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < 1.e-14) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 + << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + } mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From a78e260d4e19416217cd68a1e72d0f4a422ec5a6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 27 Mar 2025 14:46:11 -0400 Subject: [PATCH 38/53] Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ --- src/ABPG.cc | 6 +- src/DFTsolver.cc | 4 +- src/DavidsonSolver.cc | 8 +- src/Energy.cc | 29 ++- src/Energy.h | 12 +- src/GrassmanLineMinimization.cc | 4 +- src/HamiltonianMVPSolver.cc | 14 +- src/Ions.cc | 193 ++++++++---------- src/Ions.h | 19 +- src/MGmol.cc | 55 ++--- src/MGmol.h | 8 +- src/MGmolInterface.h | 3 - src/MVPSolver.cc | 4 +- src/PolakRibiereSolver.cc | 8 +- src/lbfgsrlx.cc | 1 - src/quench.cc | 4 +- .../testRestartEnergyAndForces.cc | 4 - 17 files changed, 183 insertions(+), 193 deletions(-) diff --git a/src/ABPG.cc b/src/ABPG.cc index 62cb6ba1..107edc63 100644 --- a/src/ABPG.cc +++ b/src/ABPG.cc @@ -38,7 +38,7 @@ void ABPG::setup(T& orbitals) // // orthof=true: wants orthonormalized updated wave functions template -int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, +int ABPG::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool /*orthof*/, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -51,8 +51,8 @@ int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, T res("Residual", orbitals, false); const bool check_res = (atol > 0.); - double normRes = mgmol_strategy_->computeResidual( - orbitals, work_orbitals, res, (print_res || check_res), check_res); + double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, + ions, res, (print_res || check_res), check_res); if (normRes < atol && check_res) { abpg_nl_update_tm_.stop(); diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index 91c4a0f6..206fba91 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -196,8 +196,8 @@ double DFTsolver::evaluateEnergy( // Get the new total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - eks_history_[0] - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + eks_history_[0] = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); sum_eig_[1] = sum_eig_[0]; sum_eig_[0] = 2. * proj_matrices_->getEigSum(); // 2.*sum in [Ry] diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index 0dc88ef4..a049fde7 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -500,7 +500,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(projmatrices, (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, projmatrices, orbitals, printE, os_); + ts0, projmatrices, ions_, orbitals, printE, os_); retval = checkConvergence(e0, outer_it, ct.conv_tol); if (retval == 0 || (outer_it == ct.max_electronic_steps)) @@ -549,7 +549,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, proj_mat2N_.get(), orbitals, printE, os_); + ts0, proj_mat2N_.get(), ions_, orbitals, printE, os_); } // 2N x 2N target... @@ -621,8 +621,8 @@ int DavidsonSolver::solve( const double ts1 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat2N_.get(), orbitals, ct.verbose - 1, os_); + const double e1 = energy_->evaluateTotal(ts1, proj_mat2N_.get(), + ions_, orbitals, ct.verbose - 1, os_); // line minimization beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); diff --git a/src/Energy.cc b/src/Energy.cc index 1cee6d67..f53dbc93 100644 --- a/src/Energy.cc +++ b/src/Energy.cc @@ -25,11 +25,10 @@ #define RY2HA 0.5 template -Energy::Energy(const pb::Grid& mygrid, const Ions& ions, - const Potentials& pot, const Electrostatic& es, const Rho& rho, - const XConGrid& xc, SpreadPenaltyInterface* spread_penalty) +Energy::Energy(const pb::Grid& mygrid, const Potentials& pot, + const Electrostatic& es, const Rho& rho, const XConGrid& xc, + SpreadPenaltyInterface* spread_penalty) : mygrid_(mygrid), - ions_(ions), pot_(pot), es_(es), rho_(rho), @@ -59,7 +58,7 @@ double Energy::getEVrhoRho() const } template -double Energy::evaluateEnergyIonsInVext() +double Energy::evaluateEnergyIonsInVext(Ions& ions) { double energy = 0.; @@ -69,12 +68,12 @@ double Energy::evaluateEnergyIonsInVext() //(*MPIdata::sout)<<"Energy::evaluateEnergyIonsInVext()"< positions; - positions.reserve(3 * ions_.local_ions().size()); + positions.reserve(3 * ions.local_ions().size()); // loop over ions int nions = 0; - std::vector::const_iterator ion = ions_.local_ions().begin(); - while (ion != ions_.local_ions().end()) + std::vector::const_iterator ion = ions.local_ions().begin(); + while (ion != ions.local_ions().end()) { (*ion)->getPosition(position); positions.push_back(position[0]); @@ -88,9 +87,9 @@ double Energy::evaluateEnergyIonsInVext() pot_.getValVext(positions, val); // loop over ions again - ion = ions_.local_ions().begin(); + ion = ions.local_ions().begin(); int ion_index = 0; - while (ion != ions_.local_ions().end()) + while (ion != ions.local_ions().end()) { const double z = (*ion)->getZion(); // int ion_index=(*ion)->index(); @@ -112,16 +111,16 @@ double Energy::evaluateEnergyIonsInVext() template double Energy::evaluateTotal(const double ts, // in [Ha] - ProjectedMatricesInterface* projmatrices, const T& phi, const int verbosity, - std::ostream& os) + ProjectedMatricesInterface* projmatrices, Ions& ions, const T& phi, + const int verbosity, std::ostream& os) { eval_te_tm_.start(); Control& ct = *(Control::instance()); - const double eself = ions_.energySelf(); - const double ediff = ions_.energyDiff(ct.bcPoisson); - const double eipot = evaluateEnergyIonsInVext(); + const double eself = ions.energySelf(); + const double ediff = ions.energyDiff(ct.bcPoisson); + const double eipot = evaluateEnergyIonsInVext(ions); const double eigsum = 0.5 * projmatrices->getExpectationH(); diff --git a/src/Energy.h b/src/Energy.h index e6cafd98..d55d4040 100644 --- a/src/Energy.h +++ b/src/Energy.h @@ -11,6 +11,7 @@ #define MGMOL_ENERGY_H #include "Grid.h" +#include "Ions.h" #include "Rho.h" #include "SpreadPenaltyInterface.h" #include "Timer.h" @@ -20,7 +21,6 @@ #include class Potentials; -class Ions; class Electrostatic; class ProjectedMatricesInterface; class XConGrid; @@ -29,7 +29,6 @@ template class Energy { const pb::Grid& mygrid_; - const Ions& ions_; const Potentials& pot_; const Electrostatic& es_; const Rho& rho_; @@ -45,16 +44,15 @@ class Energy double getEVrhoRho() const; public: - Energy(const pb::Grid&, const Ions&, const Potentials&, - const Electrostatic&, const Rho&, const XConGrid&, - SpreadPenaltyInterface*); + Energy(const pb::Grid&, const Potentials&, const Electrostatic&, + const Rho&, const XConGrid&, SpreadPenaltyInterface*); static Timer eval_te_tm() { return eval_te_tm_; } - double evaluateTotal(const double ts, ProjectedMatricesInterface*, + double evaluateTotal(const double ts, ProjectedMatricesInterface*, Ions&, const T& phi, const int, std::ostream&); - double evaluateEnergyIonsInVext(); + double evaluateEnergyIonsInVext(Ions&); void saveVofRho(); }; diff --git a/src/GrassmanLineMinimization.cc b/src/GrassmanLineMinimization.cc index a83d6b08..f642e0f4 100644 --- a/src/GrassmanLineMinimization.cc +++ b/src/GrassmanLineMinimization.cc @@ -29,7 +29,7 @@ Timer GrassmanLineMinimization::update_states_tm_("Grassman_update_states"); // // orthof=true: wants orthonormalized updated wave functions template -int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, +int GrassmanLineMinimization::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool orthof, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -61,7 +61,7 @@ int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, // Update wavefunctions const bool check_res = (atol > 0.); double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, - *new_grad_, (print_res || check_res), check_res); + ions, *new_grad_, (print_res || check_res), check_res); if (normRes < atol && check_res) { nl_update_tm_.stop(); diff --git a/src/HamiltonianMVPSolver.cc b/src/HamiltonianMVPSolver.cc index 4bc017dc..f90dd228 100644 --- a/src/HamiltonianMVPSolver.cc +++ b/src/HamiltonianMVPSolver.cc @@ -160,8 +160,8 @@ int HamiltonianMVPSolver::solve( // compute energy at origin const int printE = (ct.verbose > 1) ? 1 : 0; - double e0 - = energy_->evaluateTotal(ts0, projmatrices, orbitals, printE, os_); + double e0 = energy_->evaluateTotal( + ts0, projmatrices, ions_, orbitals, printE, os_); // // compute energy at end for new H @@ -189,8 +189,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at end (beta=1.) - double e1 - = energy_->evaluateTotal(ts1, projmatrices, orbitals, printE, os_); + double e1 = energy_->evaluateTotal( + ts1, projmatrices, ions_, orbitals, printE, os_); // // evaluate energy at mid-point @@ -226,8 +226,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at midpoint - double ei - = energy_->evaluateTotal(tsi, projmatrices, orbitals, printE, os_); + double ei = energy_->evaluateTotal( + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization double beta @@ -285,7 +285,7 @@ int HamiltonianMVPSolver::solve( // compute energy at end (beta=1.) ei = energy_->evaluateTotal( - tsi, projmatrices, orbitals, printE, os_); + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization beta = minQuadPolynomialFrom3values( diff --git a/src/Ions.cc b/src/Ions.cc index 3b128b0e..ef9d687c 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -70,8 +70,18 @@ void writeData2d(HDFrestart& h5f_file, std::string datasetname, } } -Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) +Ions::Ions(const double lat[3], const std::vector& sp) + : species_(sp), setup_(false), has_locked_atoms_(false) { + setupSubdomains(lat); +} + +void Ions::setupSubdomains(const double lat[3]) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + for (short i = 0; i < 3; i++) { assert(lat[i] > 0.); @@ -79,15 +89,10 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) { (*MPIdata::serr) << "Ions constructor: lattice[" << i << "]=" << lat[i] << "!!!" << std::endl; - exit(2); + mmpi.abort(); } lattice_[i] = lat[i]; } - setup_ = false; - has_locked_atoms_ = false; - - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); for (short i = 0; i < 3; i++) div_lattice_[i] = lattice_[i] / (double)(myPEenv.n_mpi_task(i)); @@ -107,6 +112,20 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) MPI_Cart_shift(cart_comm_, dir, disp, &source_[dir], &dest_[dir]); } +Ions::Ions(const std::vector& p, const std::vector& anum, + const double lat[3], const std::vector& sp) + : species_(sp) +{ + setupSubdomains(lat); + + double rmax = getMaxListRadius(); + setupListIonsBoundaries(rmax); + + num_ions_ = setAtoms(p, anum); + + setup(); +} + Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { for (const auto& ion : ions.list_ions_) @@ -181,8 +200,6 @@ void Ions::setup() ions_setup_tm.start(); - Ion::resetIndexCount(); - updateListIons(); //#ifndef NDEBUG @@ -197,7 +214,7 @@ void Ions::setup() has_locked_atoms_ = hasLockedAtoms(); // initialize data for constraints - setupContraintsData(interacting_ions_); + setupContraintsData(); computeMaxNumProjs(); @@ -297,13 +314,13 @@ void Ions::setupInteractingIons() // setup arrays to be used in constraints enforcement // using references to local_ions and extra "dummy" data -void Ions::setupContraintsData(std::vector& ions_for_constraints) +void Ions::setupContraintsData() { Control& ct = *(Control::instance()); if (ct.verbose > 0) printWithTimeStamp("Ions::setupContraintsData()...", std::cout); - const int nnloc = ions_for_constraints.size() - local_ions_.size(); + const int nnloc = interacting_ions_.size() - local_ions_.size(); // std::cout<<"interacting_ions_.size()="<::finalEnergy() // Get the total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] total_energy_ = energy_->evaluateTotal( - ts, proj_matrices_.get(), *current_orbitals_, 2, os_); + ts, proj_matrices_.get(), *ions_, *current_orbitals_, 2, os_); } template @@ -828,7 +827,7 @@ void MGmol::setupPotentials(Ions& ions) pot.initialize(ions); if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); + g_kbpsi_->setup(ions); electrostat_->setupRhoc(pot.rho_comp()); @@ -1170,8 +1169,8 @@ void MGmol::precond_mg(OrbitalsType& phi) template double MGmol::computeResidual(OrbitalsType& orbitals, - OrbitalsType& work_orbitals, OrbitalsType& res, const bool print_residual, - const bool norm_res) + OrbitalsType& work_orbitals, Ions& ions, OrbitalsType& res, + const bool print_residual, const bool norm_res) { assert(orbitals.getIterativeIndex() >= 0); @@ -1190,7 +1189,7 @@ double MGmol::computeResidual(OrbitalsType& orbitals, // get H*psi stored in work_orbitals.psi // and psi^T H psi in Hij - getHpsiAndTheta(*ions_, orbitals, work_orbitals); + getHpsiAndTheta(ions, orbitals, work_orbitals); double norm2Res = computeConstraintResidual( orbitals, work_orbitals, res, print_residual, norm_res); @@ -1428,17 +1427,22 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, Control& ct = *(Control::instance()); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); - quench(*dorbitals, *ions_, ct.max_electronic_steps, 20, eks); + quench(*dorbitals, ions, ct.max_electronic_steps, 20, eks); - force(*dorbitals, *ions_); + force(*dorbitals, ions); - ions_->getForces(forces); + ions.getForces(forces); return eks; } @@ -1450,37 +1454,42 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, { OrbitalsType* dorbitals = dynamic_cast(orbitals); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); // initialize electronic density rho_->update(*dorbitals); // initialize potential - update_pot(*ions_); + update_pot(ions); // initialize projected matrices - updateHmatrix(*dorbitals, *ions_); + updateHmatrix(*dorbitals, ions); proj_matrices_->updateThetaAndHB(); // compute DM std::shared_ptr> dm_strategy( DMStrategyFactory>::create(comm_, os_, *ions_, + dist_matrix::DistMatrix>::create(comm_, os_, ions, rho_.get(), energy_.get(), electrostat_.get(), this, proj_matrices_.get(), dorbitals)); dm_strategy->update(*dorbitals); // evaluate energy and forces - double ts = 0.; - double eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), *dorbitals, 2, os_); + double ts = 0.; + double eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, *dorbitals, 2, os_); - force(*dorbitals, *ions_); + force(*dorbitals, ions); - ions_->getForces(forces); + ions.getForces(forces); return eks; } diff --git a/src/MGmol.h b/src/MGmol.h index 12197dfd..c85e7b59 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -335,17 +335,13 @@ class MGmol : public MGmolInterface void precond_mg(OrbitalsType& orbitals); void setGamma(const pb::Lap& lapOper, const Potentials& pot); double computeResidual(OrbitalsType& orbitals, OrbitalsType& work_orbitals, - OrbitalsType& res, const bool print_residual, const bool norm_res); + Ions& ions, OrbitalsType& res, const bool print_residual, + const bool norm_res); void applyAOMMprojection(OrbitalsType&); void force(OrbitalsType& orbitals, Ions& ions) { forces_->force(orbitals, ions); } - void setPositions(const std::vector& positions, - const std::vector& atnumbers) - { - ions_->setPositions(positions, atnumbers); - } /* * simply dump current state diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index dc54a8e7..9a9bf8a6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -40,9 +40,6 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; - virtual void setPositions(const std::vector& positions, - const std::vector& atnumbers) - = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 7ab9f665..6990e29d 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -256,7 +256,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts0 = evalEntropyMVP(current_proj_mat, (ct.verbose > 1), os_); const double e0 = energy_->evaluateTotal( - ts0, current_proj_mat, orbitals, printE, os_); + ts0, current_proj_mat, ions_, orbitals, printE, os_); MatrixType target("target", numst_, numst_); @@ -309,7 +309,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts1 = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, orbitals, ct.verbose - 1, os_); + ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); // line minimization const double beta diff --git a/src/PolakRibiereSolver.cc b/src/PolakRibiereSolver.cc index 6b5f88de..3f46a00e 100644 --- a/src/PolakRibiereSolver.cc +++ b/src/PolakRibiereSolver.cc @@ -243,9 +243,9 @@ double PolakRibiereSolver::evaluateEnergy( const OrbitalsType& orbitals, const bool print_flag) { // Get the new total energy - const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - const double energy - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] + const double energy = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); // Control& ct(*(Control::instance())); // if( ct.verbose>2 && onpe0 )os_<<"energy="<::solve(OrbitalsType& orbitals, // evaluate residuals, preconditioned residuals for current orbitals double normRes = mgmol_strategy_->computeResidual(orbitals, - work_orbitals, *r_k_, (print_res || ct.checkResidual()), + work_orbitals, ions_, *r_k_, (print_res || ct.checkResidual()), ct.checkResidual()); if (normRes < ct.conv_tol && ct.checkResidual()) { diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index 6698eef0..ce05c1a3 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -14,7 +14,6 @@ #include "Energy.h" #include "Ions.h" #include "LBFGS.h" -#include "LBFGS_IonicStepper.h" #include "LocalizationRegions.h" #include "MGmol.h" #include "MGmol_blas1.h" diff --git a/src/quench.cc b/src/quench.cc index 073798f6..6eed74bb 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -600,8 +600,8 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, << " TS [Ha] = " << ts << std::endl; } } - last_eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), orbitals, 2, os_); + last_eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, orbitals, 2, os_); if (ct.computeCondGramMD()) { diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index 3027ab55..b1fb0094 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,8 +124,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -195,8 +193,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - // // evaluate energy and forces with wavefunctions just read // From c4817b8a1028923d71458a69a95ee05cd9486826 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 27 Mar 2025 16:44:32 -0700 Subject: [PATCH 39/53] Add cleanup and MGmolInterface (#331) --- src/MGmolInterface.h | 1 + tests/WFEnergyAndForces/testWFEnergyAndForces.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9a9bf8a6..ec913944 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -25,6 +25,7 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual void cleanup() = 0; virtual double evaluateEnergyAndForces(const std::vector& tau, const std::vector& atnumbers, std::vector& forces) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index c9e28d48..64765c7a 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,7 @@ int main(int argc, char** argv) } } + mgmol->cleanup(); delete mgmol; } // close main scope From a5f6f3a9906c8e4cee1bfe7a7ce2197531ee8f83 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 28 Mar 2025 10:58:24 -0400 Subject: [PATCH 40/53] Add function to set local forces (#333) * based on matching coordinates to local ions --- src/Ions.cc | 33 +++++++++++++++++++++++++++ src/Ions.h | 6 +++++ tests/testIons.cc | 58 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index ef9d687c..462373a9 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1322,6 +1322,39 @@ void Ions::setLocalForces( } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& coords) +{ + assert(forces.size() == coords.size()); + + // tolerance can be pretty loose, as long as it does not allow mix up + // with coordinates of other atoms + const double tol = 1.e-2; + + // loop over global list of forces and coordinates + std::vector::const_iterator cit = coords.begin(); + for (auto fit = forces.begin(); fit != forces.end(); fit += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + double p[3]; + ion->getPosition(&p[0]); + double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) + + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + double d = std::sqrt(d2); + if (d < tol) + { + ion->set_force(0, *fit); + ion->set_force(1, *(fit + 1)); + ion->set_force(2, *(fit + 2)); + } + } + cit += 3; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { diff --git a/src/Ions.h b/src/Ions.h index f39e1fed..aaf48e79 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -308,6 +308,12 @@ class Ions void setLocalForces(const std::vector& forces, const std::vector& names); + /*! + * set forces for ions in local_ions_ based on coordinates matching + */ + void setLocalForces( + const std::vector& forces, const std::vector& coords); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/tests/testIons.cc b/tests/testIons.cc index 6bea5d64..33c3cd24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -6,6 +6,29 @@ #include +// check that all forces components have integer values larger than 0 +// and differ from each other +int checkForces(std::vector& forces) +{ + const double tol = 1.e-14; + + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < tol || *f1 < tol || *f0 < tol) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 << std::endl; + return 1; + } + } + } + + return 0; +} + int main(int argc, char** argv) { int status = 0; @@ -164,7 +187,7 @@ int main(int argc, char** argv) std::vector forces(3 * na); // set forces to a different arbitrary value for each component - int i = 0; + int i = 1; for (auto& f : forces) { f = (double)i; @@ -189,20 +212,29 @@ int main(int argc, char** argv) ions.getForces(forces); if (myrank == 0) - for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + + // test Ions::setLocalForces based on coordinates matching + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + + ions.setLocalForces(forces, positions); + + ions.printForcesGlobal(std::cout); + + ions.getForces(forces); + if (myrank == 0) { - std::cout << "f0 = " << *f0 << std::endl; - for (auto f1 = f0 + 1; f1 != forces.end(); f1++) - { - // make sure each force component is different - if (std::abs(*f0 - *f1) < 1.e-14) - { - std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 - << std::endl; - MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); - } - } + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } + } + mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From 4e82239a124bea0128562e627b15a82d07ec2bcb Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Sun, 30 Mar 2025 10:48:27 -0700 Subject: [PATCH 41/53] Fix setLocalForces (#335) --- src/Ions.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index 462373a9..70ca5c36 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1341,8 +1341,8 @@ void Ions::setLocalForces( double p[3]; ion->getPosition(&p[0]); double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) - + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) - + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + + (p[1] - (*(cit + 1))) * (p[1] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[2] - (*(cit + 2))); double d = std::sqrt(d2); if (d < tol) { From f715dd9961afa5d2589be19e9423a4885a4f8ac9 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Apr 2025 15:36:16 -0400 Subject: [PATCH 42/53] Fix test WFEnergyAndForces (#336) * writing restart twice in same directory was failing --- tests/WFEnergyAndForces/testWFEnergyAndForces.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index 64765c7a..f9039abe 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,9 @@ int main(int argc, char** argv) } } + // rename output restart file + ct.out_restart_file = ct.out_restart_file + "1"; + mgmol->cleanup(); delete mgmol; From 89d502e776bd8150c8e127fdb9980741e73ab713 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Apr 2025 16:20:50 -0400 Subject: [PATCH 43/53] Add mixing option in MVP (#337) --- src/Control.cc | 4 +- src/MVPSolver.cc | 127 +++++++++++++++++++++++------------------ src/MVPSolver.h | 3 +- src/MVP_DMStrategy.cc | 2 +- tests/CMakeLists.txt | 7 +++ tests/MVP/test.py | 2 + tests/MVPmix/h2o.xyz | 6 ++ tests/MVPmix/mgmol.cfg | 36 ++++++++++++ tests/MVPmix/test.py | 103 +++++++++++++++++++++++++++++++++ 9 files changed, 230 insertions(+), 60 deletions(-) create mode 100644 tests/MVPmix/h2o.xyz create mode 100644 tests/MVPmix/mgmol.cfg create mode 100644 tests/MVPmix/test.py diff --git a/src/Control.cc b/src/Control.cc index 33fd93c4..22de9b27 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -699,8 +699,8 @@ void Control::setDefaultValues() void Control::adjust() { - // change dm_mix default to 1. if not using Davidson - if (it_algo_type_ != 2 && dm_mix < 0.) dm_mix = 1.; + // change dm_mix default to 1. if not using Davidson or MVP + if ((it_algo_type_ != 2 && DM_solver_ != 1) && dm_mix < 0.) dm_mix = 1.; if (nel_ - 2 * numst == 0) { diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6990e29d..770acdd0 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -47,13 +47,14 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm) + const short n_inner_steps, const double mixing, const bool use_old_dm) : comm_(comm), os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), ions_(ions), - numst_(numst) + numst_(numst), + mixing_(mixing) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -267,65 +268,79 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; - - double de0 = evaluateDerivative(dmInit, delta_dm, ts0); - - if (std::abs(de0) < tol_de0 && inner_it > 0) + double beta = 0.; + if (mixing_ > 0.) { - if (onpe0 && ct.verbose > 0) - std::cout << "MVP: de0 = " << de0 - << ", convergence achieved" << std::endl; - break; + beta = mixing_; + if (onpe0 && ct.verbose > 1) + { + os_ << "MVP with beta = " << beta << std::endl; + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << std::endl; + } } - - // - // evaluate free energy at beta=1 - // - if (onpe0 && ct.verbose > 2) - std::cout << "MVP --- Target energy..." << std::endl; - proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); - proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); - const double nel = proj_mat_work_->getNel(); - if (onpe0 && ct.verbose > 1) - os_ << "MVP --- Number of electrons at beta=1 : " << nel - << std::endl; - - rho_->computeRho(orbitals, target); - - mgmol_strategy_->update_pot(vh_init, ions_); - - energy_->saveVofRho(); - - // update h11 + else { - h11 = h11_nl; - mgmol_strategy_->addHlocal2matrix(orbitals, orbitals, h11); - } - - proj_mat_work_->assignH(h11); - proj_mat_work_->setHB2H(); - - const double ts1 - = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); + double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + + if (std::abs(de0) < tol_de0 && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + + // + // evaluate free energy at beta=1 + // + if (onpe0 && ct.verbose > 2) + std::cout << "MVP --- Target energy..." << std::endl; + proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); + proj_mat_work_->computeOccupationsFromDM(); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); + const double nel = proj_mat_work_->getNel(); + if (onpe0 && ct.verbose > 1) + os_ << "MVP --- Number of electrons at beta=1 : " << nel + << std::endl; + + rho_->computeRho(orbitals, target); + + mgmol_strategy_->update_pot(vh_init, ions_); + + energy_->saveVofRho(); + + // update h11 + { + h11 = h11_nl; + mgmol_strategy_->addHlocal2matrix( + orbitals, orbitals, h11); + } + + proj_mat_work_->assignH(h11); + proj_mat_work_->setHB2H(); + + const double ts1 + = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); + const double e1 = energy_->evaluateTotal(ts1, + proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); + + // line minimization + beta + = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); + assert(!std::isnan(beta)); - // line minimization - const double beta - = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); - assert(!std::isnan(beta)); - - if (onpe0 && ct.verbose > 0) - { - os_ << std::setprecision(12); - os_ << std::fixed << "MVP inner iteration " << inner_it - << ", E0=" << e0 << ", E1=" << e1; - os_ << std::scientific << ", E0'=" << de0 - << " -> beta=" << beta; - os_ << std::endl; + if (onpe0 && ct.verbose > 0) + { + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << ", E1=" << e1; + os_ << std::scientific << ", E0'=" << de0 + << " -> beta=" << beta; + os_ << std::endl; + } } - // update DM *work_ = dmInit; work_->axpy(beta, delta_dm); diff --git a/src/MVPSolver.h b/src/MVPSolver.h index 8e7a7c70..5558bcdd 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -33,6 +33,7 @@ class MVPSolver Ions& ions_; int numst_; + double mixing_; Rho* rho_; Energy* energy_; @@ -56,7 +57,7 @@ class MVPSolver Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm); + const short n_inner_steps, const double mixing, const bool use_old_dm); ~MVPSolver(); int solve(OrbitalsType& orbitals); diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 8712ae92..899bca46 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -54,7 +54,7 @@ int MVP_DMStrategy::update(OrbitalsType& orbitals) MVPSolver solver(comm_, os_, ions_, rho_, energy_, electrostat_, mgmol_strategy_, ct.numst, ct.occ_width, global_indexes_, - ct.dm_inner_steps, use_old_dm_); + ct.dm_inner_steps, ct.dm_mix, use_old_dm_); return solver.solve(orbitals); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 865ef354..d90b9ac0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -500,6 +500,13 @@ add_test(NAME testMVP ${CMAKE_CURRENT_SOURCE_DIR}/MVP/mvp.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MVP/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMVPmix + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME BandGapN2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/BandGapN2/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/test.py b/tests/MVP/test.py index bd5d708e..57d9522e 100644 --- a/tests/MVP/test.py +++ b/tests/MVP/test.py @@ -51,6 +51,8 @@ energies=[] print("Check forces are smaller than tol = {}".format(tol)) for line in lines: + if line.count(b'MVP'): + print(line) if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/MVPmix/h2o.xyz b/tests/MVPmix/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/MVPmix/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/MVPmix/mgmol.cfg b/tests/MVPmix/mgmol.cfg new file mode 100644 index 00000000..10edc464 --- /dev/null +++ b/tests/MVPmix/mgmol.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-8 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=random +initial_width=1.5 +temperature=10. +nempty=5 +[Restart] +output_level=0 +[DensityMatrix] +solver=MVP +nb_inner_it=2 +mixing=0.5 diff --git a/tests/MVPmix/test.py b/tests/MVPmix/test.py new file mode 100644 index 00000000..8ffafdbb --- /dev/null +++ b/tests/MVPmix/test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test MVP solver with mixing coefficient...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-4): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-4] +inp = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +cwd = os.getcwd() + +dst = 'pseudo.O_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +dst = 'pseudo.H_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True) + +#analyse mgmol standard output +#make sure force is below tolerance +lines=output.split(b'\n') + +convergence=0 +for line in lines: + if line.count(b'DFTsolver:') and line.count(b'convergence'): + convergence=1 + break + +if convergence==0: + print("MVP Solver did not converge") + sys.exit(1) + +flag = 0 +eigenvalues=[] +energies=[] +ecount=0 +for line in lines: + if line.count(b'FERMI'): + flag = 0 + if flag==1: + words=line.split() + for w in words: + eigenvalues.append(eval(w)) + if line.count(b'Eigenvalues'): + flag = 1 + eigenvalues=[] + if line.count(b'%%'): + words=line.split() + e=words[5][0:-1] + print(e) + ecount=ecount+1 + energies.append(eval(e)) +print(energies) + +print(eigenvalues) +tol = 1.e-4 +eigenvalue0 = -0.916 +if abs(eigenvalues[0]-eigenvalue0)>tol: + print("Expected eigenvalue 0 to be {}".format(eigenvalue0)) + sys.exit(1) +eigenvalue8 = 0.219 +if abs(eigenvalues[8]-eigenvalue8)>tol: + print("Expected eigenvalue 8 to be {}".format(eigenvalue8)) + sys.exit(1) + +niterations = ecount +print("MVP solver ran for {} iterations".format(niterations)) +if niterations>180: + print("MVP test FAILED for taking too many iterations") + sys.exit(1) + +print("Check energy...") +last_energy = energies[-1] +print("Energy = {}".format(last_energy)) +if last_energy>-17.16269: + print("Last energy = {}".format(last_energy)) + sys.exit(1) + +print("Test PASSED") +sys.exit(0) From 71ab082ec267e1e485eedf870f603e9d675887bb Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 8 Apr 2025 20:47:14 -0700 Subject: [PATCH 44/53] Timer --- examples/PinnedH2O/job.basis_1_50 | 2 +- examples/PinnedH2O/job.offline | 2 +- examples/PinnedH2O/job.ref | 2 +- examples/PinnedH2O/job.rom | 2 +- examples/PinnedH2O/job.rom_3DOF | 2 +- src/MGmol.cc | 2 ++ src/md.cc | 3 +++ tests/PinnedH2O_3DOF/job.basis | 2 +- tests/PinnedH2O_3DOF/job.offline | 2 +- tests/PinnedH2O_3DOF/job.online | 2 +- tests/PinnedH2O_3DOF/job.online_rotate | 2 +- 11 files changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/PinnedH2O/job.basis_1_50 b/examples/PinnedH2O/job.basis_1_50 index bf0a4c49..5291a7b7 100644 --- a/examples/PinnedH2O/job.basis_1_50 +++ b/examples/PinnedH2O/job.basis_1_50 @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.offline b/examples/PinnedH2O/job.offline index 64997e09..9d7150f8 100644 --- a/examples/PinnedH2O/job.offline +++ b/examples/PinnedH2O/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref index 2c4660cf..32024e6f 100644 --- a/examples/PinnedH2O/job.ref +++ b/examples/PinnedH2O/job.ref @@ -11,7 +11,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 set case = 2 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom index fc4ae1b0..45285136 100644 --- a/examples/PinnedH2O/job.rom +++ b/examples/PinnedH2O/job.rom @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF index 8b328199..0e38688b 100644 --- a/examples/PinnedH2O/job.rom_3DOF +++ b/examples/PinnedH2O/job.rom_3DOF @@ -11,7 +11,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 set case = 2 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/src/MGmol.cc b/src/MGmol.cc index facd362e..ec780d0e 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -102,6 +102,7 @@ extern Timer md_moveVnuc_tm; extern Timer md_updateMasks_tm; extern Timer md_extrapolateOrbitals_tm; extern Timer md_updateRhoAndPot_tm; +extern Timer md_updateDMandEnergy_tm; extern Timer quench_tm; extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; @@ -928,6 +929,7 @@ void MGmol::printTimers() init_nuc_tm_.print(os_); md_updateMasks_tm.print(os_); md_extrapolateOrbitals_tm.print(os_); + md_updateDMandEnergy_tm.print(os_); quench_tm.print(os_); evnl_tm_.print(os_); ions_setupInteractingIons_tm.print(os_); diff --git a/src/md.cc b/src/md.cc index bc986955..e8d06133 100644 --- a/src/md.cc +++ b/src/md.cc @@ -50,6 +50,7 @@ Timer md_tau_tm("md_tau"); Timer md_moveVnuc_tm("md_moveVnuc"); Timer md_updateMasks_tm("md_updateMasks"); Timer md_extrapolateOrbitals_tm("md_extrapolateOrbitals"); +Timer md_updateDMandEnergy_tm("md_updateDMandEnergy"); #define DUMP_MAX_NUM_TRY 5 @@ -485,7 +486,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ROM_MVP) { + md_updateDMandEnergy_tm.start(); updateDMandEnergy(**orbitals, *ROM_ions, eks); + md_updateDMandEnergy_tm.stop(); } else { diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis index cab929d1..35f564a3 100644 --- a/tests/PinnedH2O_3DOF/job.basis +++ b/tests/PinnedH2O_3DOF/job.basis @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline index 243f563a..5b4224f0 100644 --- a/tests/PinnedH2O_3DOF/job.offline +++ b/tests/PinnedH2O_3DOF/job.offline @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online index 1373e011..ea75e59f 100644 --- a/tests/PinnedH2O_3DOF/job.online +++ b/tests/PinnedH2O_3DOF/job.online @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH diff --git a/tests/PinnedH2O_3DOF/job.online_rotate b/tests/PinnedH2O_3DOF/job.online_rotate index 3f8a65ad..a2065b72 100644 --- a/tests/PinnedH2O_3DOF/job.online_rotate +++ b/tests/PinnedH2O_3DOF/job.online_rotate @@ -10,7 +10,7 @@ setenv OMP_NUM_THREADS 1 set ncpus = 8 -set maindir = /p/lustre2/cheung26/mgmol-20250219 +set maindir = /p/lustre2/cheung26/mgmol setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH From f198397436eb7b4b3bd7e6a72d0fd5f47348623c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 9 Apr 2025 10:48:07 -0400 Subject: [PATCH 45/53] Update MVP with mixing (#339) * use tolerance to terminate iterations * tune verbosity --- src/Control.cc | 6 +++++- src/MVPSolver.cc | 34 +++++++++++++++++++--------------- src/MVPSolver.h | 8 +++++++- src/MVP_DMStrategy.cc | 2 +- src/read_config.cc | 4 +++- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 22de9b27..57e12336 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,6 +224,7 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; + os << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " @@ -439,7 +440,7 @@ void Control::sync(void) memset(&int_buffer[0], 0, size_int_buffer * sizeof(int)); } - const short size_float_buffer = 43; + const short size_float_buffer = 44; float* float_buffer = new float[size_float_buffer]; if (mype_ == 0) { @@ -485,6 +486,7 @@ void Control::sync(void) float_buffer[40] = threshold_eigenvalue_gram_quench_; float_buffer[41] = pair_mlwf_distance_threshold_; float_buffer[42] = e0_; + float_buffer[43] = dm_tol; } else { @@ -680,6 +682,7 @@ void Control::sync(void) threshold_eigenvalue_gram_quench_ = float_buffer[40]; pair_mlwf_distance_threshold_ = float_buffer[41]; e0_ = float_buffer[42]; + dm_tol = float_buffer[43]; max_electronic_steps_loose_ = max_electronic_steps; delete[] short_buffer; @@ -1720,6 +1723,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) lrs_extrapolation = 10; dm_mix = vm["DensityMatrix.mixing"].as(); + dm_tol = vm["DensityMatrix.tol"].as(); dm_inner_steps = vm["DensityMatrix.nb_inner_it"].as(); dm_use_old_ = vm["DensityMatrix.use_old"].as() ? 1 : 0; str = vm["DensityMatrix.algo"].as(); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 770acdd0..1c093e53 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -47,14 +47,16 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const double mixing, const bool use_old_dm) + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm) : comm_(comm), os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), ions_(ions), numst_(numst), - mixing_(mixing) + mixing_(mixing), + tol_de0_(tol_de0) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -208,7 +210,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) kbpsi.computeHvnlMatrix(&kbpsi, ions_, h11_nl); - const double tol_de0 = 1.e-12; for (int inner_it = 0; inner_it < n_inner_steps_; inner_it++) { if (onpe0 && ct.verbose > 1) @@ -268,13 +269,26 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; + + double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + + // check for convergence + if (std::abs(de0) < tol_de0_ && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + double beta = 0.; if (mixing_ > 0.) { beta = mixing_; - if (onpe0 && ct.verbose > 1) + if (onpe0 && ct.verbose > 0) { - os_ << "MVP with beta = " << beta << std::endl; + if (ct.verbose > 1) + os_ << "MVP with beta = " << beta << std::endl; os_ << std::setprecision(12); os_ << std::fixed << "MVP inner iteration " << inner_it << ", E0=" << e0 << std::endl; @@ -282,16 +296,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) } else { - double de0 = evaluateDerivative(dmInit, delta_dm, ts0); - - if (std::abs(de0) < tol_de0 && inner_it > 0) - { - if (onpe0 && ct.verbose > 0) - std::cout << "MVP: de0 = " << de0 - << ", convergence achieved" << std::endl; - break; - } - // // evaluate free energy at beta=1 // diff --git a/src/MVPSolver.h b/src/MVPSolver.h index 5558bcdd..77a6a345 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -35,6 +35,11 @@ class MVPSolver int numst_; double mixing_; + /*! + * tolerance on energy slope in inner iterations + */ + double tol_de0_; + Rho* rho_; Energy* energy_; Electrostatic* electrostat_; @@ -57,7 +62,8 @@ class MVPSolver Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const double mixing, const bool use_old_dm); + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm); ~MVPSolver(); int solve(OrbitalsType& orbitals); diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 899bca46..3fb53dee 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -54,7 +54,7 @@ int MVP_DMStrategy::update(OrbitalsType& orbitals) MVPSolver solver(comm_, os_, ions_, rho_, energy_, electrostat_, mgmol_strategy_, ct.numst, ct.occ_width, global_indexes_, - ct.dm_inner_steps, ct.dm_mix, use_old_dm_); + ct.dm_inner_steps, ct.dm_mix, ct.dm_tol, use_old_dm_); return solver.solve(orbitals); } diff --git a/src/read_config.cc b/src/read_config.cc index 642b1b99..ed0e66e1 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -297,7 +297,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(0), "Flag for computing new centers from extrapolated orbitals.")( "DensityMatrix.mixing", po::value()->default_value(-1.), - "Mixing coefficient for Density Matrix")("DensityMatrix.solver", + "Mixing coefficient for Density Matrix")("DensityMatrix.tol", + po::value()->default_value(1.e-12), + "Tolerance for Density Matrix convergence")("DensityMatrix.solver", po::value()->default_value("Mixing"), "Algorithm for updating Density Matrix: Mixing, MVP, HMVP")( "DensityMatrix.nb_inner_it", po::value()->default_value(3), From cb3b8773a48648c4df7665ea3d270fab4c1a23eb Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Wed, 9 Apr 2025 20:17:45 -0700 Subject: [PATCH 46/53] Updates --- examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg | 6 ++++-- examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 6 ++++-- src/Control.cc | 4 +--- src/read_config.cc | 5 +---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg index adfdfc43..b7a78a0d 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg @@ -1,4 +1,4 @@ -verbosity=1 +verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] @@ -34,7 +34,9 @@ initial_width=2. nempty=30 [DensityMatrix] solver=MVP -nb_inner_it=100 +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 [Restart] output_level=4 [ROM] diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg index f0368790..24273c40 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -1,4 +1,4 @@ -verbosity=1 +verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] @@ -30,7 +30,9 @@ initial_width=2. nempty=30 [DensityMatrix] solver=MVP -nb_inner_it=100 +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 [Restart] output_level=4 [ROM] diff --git a/src/Control.cc b/src/Control.cc index bf2b44b3..cbf2eadd 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,7 +224,7 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; - os << " Density matrix tol = " << dm_tol << std::endl; + os << std::setprecision(4) << std::scientific << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " @@ -1743,8 +1743,6 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else dm_algo_ = 2; - dm_tol = vm["DensityMatrix.tol"].as(); - str = vm["DensityMatrix.solver"].as(); if (str.compare("Mixing") == 0) DM_solver_ = 0; if (str.compare("MVP") == 0) DM_solver_ = 1; diff --git a/src/read_config.cc b/src/read_config.cc index 4be0d39b..9dbd56d3 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -326,10 +326,7 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(100), "Maximum number of iterations for power method " "to compute interval for Chebyshev " - "approximation of density matrix. ")("DensityMatrix.tol", - po::value()->default_value(1.e-7), - "tolerance, used in iterative DM computation convergence " - "criteria"); + "approximation of density matrix. "); po::options_description cmdline_options; cmdline_options.add(generic); From 8f031013ab2ba6a15d1301ebfa4d932822d4eb7d Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 22 Apr 2025 07:43:59 -0700 Subject: [PATCH 47/53] use setupPotentials --- src/rom_workflows.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 8fbd4cb3..13c9ac92 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -861,7 +861,7 @@ void testROMIonDensity(MGmolInterface *mgmol_) /* compute resulting ion density */ /* NOTE: we exclude rescaling for the sake of verification */ - pot.initialize(*ions); + mgmol->setupPotentials(*ions); mgmol->electrostat_->setupRhoc(pot.rho_comp()); fom_rhoc[idx].resize(dim); From a8ec88bbd66c51a8f4bf69ba28846ab4688b7e9b Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 8 May 2025 09:38:51 -0700 Subject: [PATCH 48/53] Setup potentials again --- src/rom_workflows.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 13c9ac92..7ddd4c4d 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -919,6 +919,9 @@ void testROMIonDensity(MGmolInterface *mgmol_) for (int d = 0; d < 3; d++) CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + /* set up potentials */ + mgmol->setupPotentials(*ions); + /* eval ion density on sample grid points */ std::vector sampled_rhoc(sampled_row.size()); pot.evalIonDensityOnSamplePts(*ions, sampled_row, sampled_rhoc); From 7dae033d28b139c3f7132953cc7607b3d33fc245 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 8 May 2025 10:27:17 -0700 Subject: [PATCH 49/53] setupRhoc is done within setupPotentials --- src/rom_workflows.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 7ddd4c4d..db046067 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -863,7 +863,7 @@ void testROMIonDensity(MGmolInterface *mgmol_) /* NOTE: we exclude rescaling for the sake of verification */ mgmol->setupPotentials(*ions); - mgmol->electrostat_->setupRhoc(pot.rho_comp()); + //mgmol->electrostat_->setupRhoc(pot.rho_comp()); fom_rhoc[idx].resize(dim); mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); } From a2421cd77881373488bddc3ab34cd4b44a1f82de Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 8 May 2025 11:18:09 -0700 Subject: [PATCH 50/53] Avoid setPositions --- src/rom_workflows.cc | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index db046067..99aff8a5 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -841,6 +841,7 @@ void testROMIonDensity(MGmolInterface *mgmol_) mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); } + Ions* new_ions; /* Collect fictitious ion density based on each configuration */ std::vector> fom_rhoc(num_snap); /* Sanity check for overlappingVL_ions */ @@ -848,24 +849,27 @@ void testROMIonDensity(MGmolInterface *mgmol_) for (int idx = 0; idx < num_snap; idx++) { /* set ion positions */ - ions->setPositions(cfgs[idx], atnumbers); + //ions->setPositions(cfgs[idx], atnumbers); + new_ions = new Ions(cfgs[idx], atnumbers, lattice, ions->getSpecies()); /* save overlapping ions for sanity check */ - fom_overlap_ions[idx].resize(ions->overlappingVL_ions().size()); - for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + fom_overlap_ions[idx].resize(new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) { fom_overlap_ions[idx][k].resize(3); for (int d = 0; d < 3; d++) - fom_overlap_ions[idx][k][d] = ions->overlappingVL_ions()[k]->position(d); + fom_overlap_ions[idx][k][d] = new_ions->overlappingVL_ions()[k]->position(d); } /* compute resulting ion density */ /* NOTE: we exclude rescaling for the sake of verification */ - mgmol->setupPotentials(*ions); + mgmol->setupPotentials(*new_ions); //mgmol->electrostat_->setupRhoc(pot.rho_comp()); fom_rhoc[idx].resize(dim); mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); + + delete new_ions; } /* Initialize libROM classes */ @@ -911,26 +915,29 @@ void testROMIonDensity(MGmolInterface *mgmol_) if (rank == 0) printf("test index: %d\n", test_idx); /* set ion positions */ - ions->setPositions(cfgs[test_idx], atnumbers); + //ions->setPositions(cfgs[test_idx], atnumbers); + new_ions = new Ions(cfgs[test_idx], atnumbers, lattice, ions->getSpecies()); /* Sanity check for overlapping ions */ - CAROM_VERIFY(fom_overlap_ions[test_idx].size() == ions->overlappingVL_ions().size()); - for (int k = 0; k < ions->overlappingVL_ions().size(); k++) + CAROM_VERIFY(fom_overlap_ions[test_idx].size() == new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) for (int d = 0; d < 3; d++) - CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); + CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] - new_ions->overlappingVL_ions()[k]->position(d)) < 1.0e-12); /* set up potentials */ - mgmol->setupPotentials(*ions); + mgmol->setupPotentials(*new_ions); /* eval ion density on sample grid points */ std::vector sampled_rhoc(sampled_row.size()); - pot.evalIonDensityOnSamplePts(*ions, sampled_row, sampled_rhoc); + pot.evalIonDensityOnSamplePts(*new_ions, sampled_row, sampled_rhoc); for (int d = 0; d < sampled_row.size(); d++) { printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-12); } + + delete new_ions; } template void readRestartFiles(MGmolInterface *mgmol_); From 81c98df52bfd11ebe5f189952eea8e1c7a870762 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Thu, 8 May 2025 12:32:14 -0700 Subject: [PATCH 51/53] Rescaling ROM ion density --- src/Potentials.cc | 7 +++++++ src/rom_workflows.cc | 1 + 2 files changed, 8 insertions(+) diff --git a/src/Potentials.cc b/src/Potentials.cc index a620cdeb..11d28a28 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -927,6 +927,13 @@ void Potentials::evalIonDensityOnSamplePts( initializeRadialDataOnSampledPts(position, sp, local_idx, sampled_rhoc); } + // rescale rho_comp_ due to finite mesh effects + rescaleRhoComp(); + + initBackground(); + + addBackgroundToRhoComp(); + return; } diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 99aff8a5..e1aff907 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -841,6 +841,7 @@ void testROMIonDensity(MGmolInterface *mgmol_) mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); } + /* Artificial ions object to avoid repeated setPositions */ Ions* new_ions; /* Collect fictitious ion density based on each configuration */ std::vector> fom_rhoc(num_snap); From 82576384beaa62b70f7280a81663d9131d0f4a1a Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Wed, 14 May 2025 09:31:46 -0700 Subject: [PATCH 52/53] Relax threshold --- src/Potentials.cc | 7 ------- src/rom_workflows.cc | 9 +++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/Potentials.cc b/src/Potentials.cc index 11d28a28..a620cdeb 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -927,13 +927,6 @@ void Potentials::evalIonDensityOnSamplePts( initializeRadialDataOnSampledPts(position, sp, local_idx, sampled_rhoc); } - // rescale rho_comp_ due to finite mesh effects - rescaleRhoComp(); - - initBackground(); - - addBackgroundToRhoComp(); - return; } diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index e1aff907..2b054687 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -850,7 +850,6 @@ void testROMIonDensity(MGmolInterface *mgmol_) for (int idx = 0; idx < num_snap; idx++) { /* set ion positions */ - //ions->setPositions(cfgs[idx], atnumbers); new_ions = new Ions(cfgs[idx], atnumbers, lattice, ions->getSpecies()); /* save overlapping ions for sanity check */ @@ -866,7 +865,6 @@ void testROMIonDensity(MGmolInterface *mgmol_) /* NOTE: we exclude rescaling for the sake of verification */ mgmol->setupPotentials(*new_ions); - //mgmol->electrostat_->setupRhoc(pot.rho_comp()); fom_rhoc[idx].resize(dim); mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); @@ -916,7 +914,6 @@ void testROMIonDensity(MGmolInterface *mgmol_) if (rank == 0) printf("test index: %d\n", test_idx); /* set ion positions */ - //ions->setPositions(cfgs[test_idx], atnumbers); new_ions = new Ions(cfgs[test_idx], atnumbers, lattice, ions->getSpecies()); /* Sanity check for overlapping ions */ @@ -932,10 +929,14 @@ void testROMIonDensity(MGmolInterface *mgmol_) std::vector sampled_rhoc(sampled_row.size()); pot.evalIonDensityOnSamplePts(*new_ions, sampled_row, sampled_rhoc); + // For now, we relax the threshold to allow the slight difference of the values at the sampled indices. + // This is because the rescaling procedure after getting the radial data on mesh requires global information, + // thus cannot be done in the ROM level only with sampled values. + // After we have the DEIM reconstruction, we will do rescaling and revisit this. for (int d = 0; d < sampled_row.size(); d++) { printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); - CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-12); + CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-2 * fom_rhoc[test_idx][sampled_row[d]]); } delete new_ions; From 77d02df2cf982e7f899438a8cf70166cc0b29dfa Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Wed, 14 May 2025 09:50:01 -0700 Subject: [PATCH 53/53] Increase tolerance for testROMIonDensity --- src/rom_workflows.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc index 2b054687..f8d70eec 100644 --- a/src/rom_workflows.cc +++ b/src/rom_workflows.cc @@ -936,7 +936,7 @@ void testROMIonDensity(MGmolInterface *mgmol_) for (int d = 0; d < sampled_row.size(); d++) { printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], sampled_rhoc[d]); - CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-2 * fom_rhoc[test_idx][sampled_row[d]]); + CAROM_VERIFY(abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-2); } delete new_ions;