From 1ad1292720f89c93e4d8a4046d68028fdee449fe Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 26 Sep 2024 08:27:05 -0400 Subject: [PATCH 01/58] Fix EnergyAndForces tests (#277) * have them work in debug mode too --- src/DensityMatrix.cc | 7 ++++++- src/DensityMatrix.h | 13 +++++++++++-- src/ProjectedMatrices.cc | 1 + tests/WFEnergyAndForces/mgmol.cfg | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 6e6ea652..e2c7b473 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -28,7 +28,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) { - assert(ndim >= 0); + assert(ndim > 0); dim_ = ndim; @@ -45,6 +45,7 @@ DensityMatrix::DensityMatrix(const int ndim) kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); occupation_.resize(dim_); + setDummyOcc(); } template @@ -109,6 +110,7 @@ void DensityMatrix::build( const std::vector& occ, const int new_orbitals_index) { assert(dm_ != nullptr); + assert(!occ.empty()); setOccupations(occ); @@ -149,6 +151,8 @@ template void DensityMatrix::setUniform( const double nel, const int new_orbitals_index) { + assert(!occupation_.empty()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); const double occ = (double)((double)nel / (double)dim_); if (mmpi.instancePE0()) @@ -314,6 +318,7 @@ void DensityMatrix::computeOccupations(const MatrixType& ls) template void DensityMatrix::setOccupations(const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (mmpi.instancePE0()) diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 960ad18e..84804a86 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -84,17 +84,24 @@ class DensityMatrix *dm_ = mat; orbitals_index_ = orbitals_index; - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; stripped_ = false; } + // set occupations to meaningless values to catch uninitialized use + void setDummyOcc() + { + for (auto& occ : occupation_) + occ = -1.; + } + void initMatrix(const double* const val) { dm_->init(val, dim_); - occupation_.clear(); + setDummyOcc(); occ_uptodate_ = false; uniform_occ_ = false; @@ -105,8 +112,10 @@ class DensityMatrix void getOccupations(std::vector& occ) const { + assert(!occupation_.empty()); assert(occ_uptodate_); assert((int)occ.size() == dim_); + memcpy(&occ[0], &occupation_[0], dim_ * sizeof(double)); } diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e0e679e9..3606aa40 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -509,6 +509,7 @@ template void ProjectedMatrices::setOccupations( const std::vector& occ) { + assert(!occ.empty()); #ifdef PRINT_OPERATIONS if (mmpi.instancePE0()) (*MPIdata::sout) << "ProjectedMatrices::setOccupations()" diff --git a/tests/WFEnergyAndForces/mgmol.cfg b/tests/WFEnergyAndForces/mgmol.cfg index d543b626..12002703 100644 --- a/tests/WFEnergyAndForces/mgmol.cfg +++ b/tests/WFEnergyAndForces/mgmol.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H type=QUENCH [Quench] max_steps=50 -atol=1.e-9 +atol=1.e-8 num_lin_iterations=2 [Orbitals] initial_type=Gaussian From 8ac6cd60cd2c06f87c8be6a5806478e316f33c53 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 30 Sep 2024 11:53:23 -0400 Subject: [PATCH 02/58] Move factor 4pi out og linear solvers (#278) --- src/Hartree.cc | 9 ++--- src/Hartree.h | 2 -- src/Hartree_CG.cc | 1 + src/PCGSolver.cc | 59 ++++++++++++++++----------------- src/PCGSolver.h | 52 ++++++++++++++--------------- src/PCGSolver_Diel.cc | 68 +++++++++++++++++++------------------- src/PCGSolver_Diel.h | 30 +++++++++-------- src/pb/Mgm.h | 3 -- src/pb/Solver.h | 7 ++-- src/pb/SolverLap.cc | 77 ++----------------------------------------- src/pb/SolverLap.h | 2 +- src/pb/SolverPB.cc | 5 ++- src/pb/SolverPB.h | 4 +-- 13 files changed, 120 insertions(+), 199 deletions(-) diff --git a/src/Hartree.cc b/src/Hartree.cc index b3984b87..7d0ba4d8 100644 --- a/src/Hartree.cc +++ b/src/Hartree.cc @@ -7,10 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #include #include -using namespace std; #include "Control.h" #include "Hartree.h" @@ -82,6 +80,8 @@ void Hartree::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + // Hartree units + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else @@ -92,10 +92,11 @@ void Hartree::solve( double final_residual = poisson_solver_->getFinalResidual(); if (onpe0) - (*MPIdata::sout) << setprecision(2) << scientific + (*MPIdata::sout) << std::setprecision(2) << std::scientific << "Hartree: residual reduction = " << residual_reduction - << ", final residual = " << final_residual << endl; + << ", final residual = " << final_residual + << std::endl; Poisson::Int_vhrho_ = vel * Poisson::vh_->gdot(rho); Poisson::Int_vhrhoc_ = vel * Poisson::vh_->gdot(rhoc); diff --git a/src/Hartree.h b/src/Hartree.h index a7050e24..b87ff01e 100644 --- a/src/Hartree.h +++ b/src/Hartree.h @@ -7,14 +7,12 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id$ #ifndef included_Hartree #define included_Hartree #include "Poisson.h" #include "PoissonInterface.h" -// pb #include "SolverLap.h" template diff --git a/src/Hartree_CG.cc b/src/Hartree_CG.cc index f9bf3860..375237f8 100644 --- a/src/Hartree_CG.cc +++ b/src/Hartree_CG.cc @@ -81,6 +81,7 @@ void Hartree_CG::solve( // { /* solve with POTDTYPE precision */ pb::GridFunc rhs(work_rho); + rhs *= (4. * M_PI); poisson_solver_->solve(*Poisson::vh_, rhs); // } // else diff --git a/src/PCGSolver.cc b/src/PCGSolver.cc index 12103aaa..b3bd7ec3 100644 --- a/src/PCGSolver.cc +++ b/src/PCGSolver.cc @@ -11,14 +11,13 @@ #include #include -using namespace std; -template -void PCGSolver::clear() +template +void PCGSolver::clear() { - for (short i = 0; i < (short)pc_oper_.size(); i++) + for (short i = 0; i < (short)precond_oper_.size(); i++) { - delete pc_oper_[i]; + delete precond_oper_[i]; } for (short i = 0; i < (short)gf_work_.size(); i++) { @@ -35,26 +34,26 @@ void PCGSolver::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { delete grid_[i]; } - pc_oper_.clear(); + precond_oper_.clear(); grid_.clear(); gf_work_.clear(); gf_rcoarse_.clear(); gf_newv_.clear(); } -template -void PCGSolver::setupPrecon() +template +void PCGSolver::setupPrecon() { // check if precon is already setup // Assumes operator does not change, hence // a single setup is sufficient - if (is_pc_setup_) return; + if (is_precond_setup_) return; // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -63,7 +62,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*grid_[0], lap_type_); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); pb::GridFunc* gf_work = new pb::GridFunc( @@ -92,7 +91,7 @@ void PCGSolver::setupPrecon() pb::Lap* myoper = LapFactory::createLap(*coarse_grid, 1); - pc_oper_.push_back(myoper); + precond_oper_.push_back(myoper); gf_work = new pb::GridFunc( *coarse_grid, bc_[0], bc_[1], bc_[2]); @@ -109,12 +108,13 @@ void PCGSolver::setupPrecon() mygrid = coarse_grid; } - is_pc_setup_ = true; + is_precond_setup_ = true; } // MG V-cycle with no mask -template -void PCGSolver::preconSolve(pb::GridFunc& gf_v, +template +void PCGSolver::preconSolve( + pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve(pb::GridFunc& gf_v, ncycl = 4 > (nu1_ + nu2_) ? 4 : (nu1_ + nu2_); } - pb::Lap* myoper = pc_oper_[level]; + pb::Lap* myoper = precond_oper_[level]; // SMOOTHING for (short it = 0; it < ncycl; it++) @@ -161,25 +161,23 @@ void PCGSolver::preconSolve(pb::GridFunc& gf_v, } // Left Preconditioned CG -template -bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +template +bool PCGSolver::solve( + pb::GridFunc& gf_phi, const pb::GridFunc& gf_rhs) { bool converged = false; const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual: r := b - Ax /* compute Ax */ oper_.apply(gf_phi, lhs); /* set r = b */ - pb::GridFunc res(gf_rhs); + pb::GridFunc res(gf_rhs); oper_.transform(res); - // Hartree units - const double hu = 4. * M_PI; - res *= hu; /* compute r = r - Ax */ res -= lhs; @@ -199,11 +197,11 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) /* preconditioning step */ prec_z.setValues(0.); preconSolve(prec_z, prec_res, 0); - pb::GridFunc z(prec_z); + pb::GridFunc z(prec_z); // conjugate vectors - pb::GridFunc p(prec_z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(prec_z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -251,13 +249,14 @@ bool PCGSolver::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) } // Left Preconditioned CG -template -bool PCGSolver::solve(T2* phi, T2* rhs, const char dis) +template +bool PCGSolver::solve( + ScalarType* phi, ScalarType* rhs, const char dis) { - pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_phi(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_phi.assign(phi, dis); - pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc gf_work(oper_.grid(), bc_[0], bc_[1], bc_[2]); gf_work.assign(rhs, dis); bool converged = solve(gf_phi, gf_work); diff --git a/src/PCGSolver.h b/src/PCGSolver.h index 8ba76158..04e1028f 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_H_ -#define _PCG_SOLVER_H_ +#ifndef MGMOL_PCG_SOLVER_H +#define MGMOL_PCG_SOLVER_H #include "Control.h" #include "Lap.h" @@ -17,32 +17,41 @@ #include -template +template class PCGSolver { - private: std::vector grid_; short lap_type_; short bc_[3]; bool fully_periodic_; - // operators + + // operator to solve for T oper_; - std::vector*> pc_oper_; + + // preconditioner operator for each MG level + std::vector*> precond_oper_; std::vector*> gf_work_; std::vector*> gf_rcoarse_; std::vector*> gf_newv_; - // solver params + + // solver parameters int maxiters_; double tol_; double final_residual_; double residual_reduction_; - // precon params + + // preconditioner parameters short nu1_; short nu2_; short max_nlevels_; short nlevels_; - bool is_pc_setup_; + bool is_precond_setup_; + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + void setupPrecon(); + void clear(); public: PCGSolver(T& oper, const short px, const short py, const short pz) @@ -61,15 +70,10 @@ class PCGSolver bc_[1] = py; bc_[2] = pz; fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); - // fine grid info - // pb::Grid* mygrid=new pb::Grid(oper.grid()); - // grid_.push_back(mygrid); - // fine grid operator - Control& ct = *(Control::instance()); - lap_type_ = ct.lap_type; - is_pc_setup_ = false; - // pb::Lap* myoper = LapFactory::createLap(*grid_[0],lap_type_); - // pc_oper_.push_back(myoper); + + Control& ct = *(Control::instance()); + lap_type_ = ct.lap_type; + is_precond_setup_ = false; }; void setup(const short nu1, const short nu2, const short max_sweeps, @@ -83,16 +87,10 @@ class PCGSolver setupPrecon(); } - void clear(); - - void setupPrecon(); - - void preconSolve(pb::GridFunc& gf_v, - const pb::GridFunc& gf_f, const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve(pb::GridFunc& gf_phi, + const pb::GridFunc& gf_rhs); - bool solve(T2* phi, T2* rhs, const char dis); + bool solve(ScalarType* phi, ScalarType* rhs, const char dis); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/PCGSolver_Diel.cc b/src/PCGSolver_Diel.cc index 0cdc5aed..4e7c88a9 100644 --- a/src/PCGSolver_Diel.cc +++ b/src/PCGSolver_Diel.cc @@ -9,10 +9,8 @@ #include "PCGSolver_Diel.h" -using namespace std; - -template -void PCGSolver_Diel::clear() +template +void PCGSolver_Diel::clear() { for (short i = 0; i < (short)pc_oper_.size(); i++) { @@ -33,7 +31,7 @@ void PCGSolver_Diel::clear() assert(gf_newv_[i] != nullptr); delete gf_newv_[i]; } - // delete grids after pb::GridFunc objects since those + // delete grids after pb::GridFunc objects since those // have data members references to grids for (short i = 0; i < (short)grid_.size(); i++) { @@ -46,8 +44,8 @@ void PCGSolver_Diel::clear() gf_newv_.clear(); } -template -void PCGSolver_Diel::setupPrecon() +template +void PCGSolver_Diel::setupPrecon() { // fine level pb::Grid* mygrid = new pb::Grid(oper_.grid()); @@ -57,8 +55,8 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(oper_); pc_oper_.push_back(myoper); - pb::GridFunc* gf_work - = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_work + = new pb::GridFunc(*grid_[0], bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); // coarse levels @@ -86,24 +84,25 @@ void PCGSolver_Diel::setupPrecon() T* myoper = new T(pc_oper_[ln - 1]->coarseOp(*mygrid)); pc_oper_.push_back(myoper); - gf_work = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + gf_work = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_work_.push_back(gf_work); - pb::GridFunc* gf_rcoarse - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_rcoarse = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_rcoarse_.push_back(gf_rcoarse); - pb::GridFunc* gf_newv - = new pb::GridFunc(*coarse_grid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc* gf_newv = new pb::GridFunc( + *coarse_grid, bc_[0], bc_[1], bc_[2]); gf_newv_.push_back(gf_newv); mygrid = coarse_grid; } } -template +template // MG V-cycle with no mask -void PCGSolver_Diel::preconSolve( - pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) +void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level) { //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::preconSolve( // COARSE GRID CORRECTION // restrictions - pb::GridFunc* rcoarse = gf_rcoarse_[level]; + pb::GridFunc* rcoarse = gf_rcoarse_[level]; gf_work_[level]->restrict3D(*rcoarse); // storage functions for coarse grid - pb::GridFunc* newv = gf_newv_[level]; + pb::GridFunc* newv = gf_newv_[level]; // call mgrid solver on a coarser level newv->resetData(); @@ -149,15 +148,16 @@ void PCGSolver_Diel::preconSolve( if (bc_[0] != 1 || bc_[2] != 1 || bc_[2] != 1) gf_v.trade_boundaries(); } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve( - pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) +bool PCGSolver_Diel::solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) { if (!oper_.initialized()) { - cout << "Error in PCGSolver_Diel::solve: operator not initialized" - << endl; + std::cout + << "Error in PCGSolver_Diel::solve: operator not initialized" + << std::endl; return 0.; } @@ -165,13 +165,13 @@ bool PCGSolver_Diel::solve( const pb::Grid& finegrid = gf_phi.grid(); // initial data and residual - We assume a nonzero initial guess - pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); - pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc lhs(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); // compute initial residual oper_.apply(gf_phi, lhs); - pb::GridFunc rhs(gf_rhs); + pb::GridFunc rhs(gf_rhs); oper_.transform(rhs); // Hartree units rhs *= (4. * M_PI); @@ -180,13 +180,13 @@ bool PCGSolver_Diel::solve( double rnorm = init_rnorm; // preconditioned residual - pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); + pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); // preconditioning step z = 0.; preconSolve(z, res, 0); // conjugate vectors - pb::GridFunc p(z); - pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); + pb::GridFunc p(z); + pb::GridFunc ap(p.grid(), bc_[0], bc_[1], bc_[2]); double rtz = res.gdot(z); @@ -225,11 +225,11 @@ bool PCGSolver_Diel::solve( return converged; } -template +template // Left Preconditioned CG -bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, - pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, - pb::GridFunc& gf_vks) +bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks) { // initialize the linear system operator and the preconditioner oper_.init(gf_rhod); diff --git a/src/PCGSolver_Diel.h b/src/PCGSolver_Diel.h index 9fd97914..f2001a4c 100644 --- a/src/PCGSolver_Diel.h +++ b/src/PCGSolver_Diel.h @@ -7,8 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef _PCG_SOLVER_DIEL_H_ -#define _PCG_SOLVER_DIEL_H_ +#ifndef MGMOL_PCG_SOLVER_DIEL_H_ +#define MGMOL_PCG_SOLVER_DIEL_H_ #include "Control.h" #include "PB.h" @@ -21,7 +21,7 @@ #include -template +template class PCGSolver_Diel { @@ -32,9 +32,9 @@ class PCGSolver_Diel // operators T oper_; std::vector pc_oper_; - std::vector*> gf_work_; - std::vector*> gf_rcoarse_; - std::vector*> gf_newv_; + std::vector*> gf_work_; + std::vector*> gf_rcoarse_; + std::vector*> gf_newv_; // solver params int maxiters_; double tol_; @@ -47,6 +47,11 @@ class PCGSolver_Diel short nlevels_; void setupPrecon(); + void clear(); + + void preconSolve(pb::GridFunc& gf_v, + const pb::GridFunc& gf_f, const short level = 0); + public: PCGSolver_Diel(T& oper, const short px, const short py, const short pz) : oper_(oper) @@ -78,15 +83,12 @@ class PCGSolver_Diel max_nlevels_ = max_nlevels; } - void clear(); - - void preconSolve(pb::GridFunc& gf_v, const pb::GridFunc& gf_f, - const short level = 0); - - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); + bool solve( + pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs); - bool solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs, - pb::GridFunc& gf_rhod, pb::GridFunc& gf_vks); + bool solve(pb::GridFunc& gf_phi, + pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, + pb::GridFunc& gf_vks); double getFinalResidual() const { return final_residual_; } double getResidualReduction() const { return residual_reduction_; } diff --git a/src/pb/Mgm.h b/src/pb/Mgm.h index d16f5610..4b5ad68d 100644 --- a/src/pb/Mgm.h +++ b/src/pb/Mgm.h @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Mgm.h,v 1.13 2010/01/28 22:56:47 jeanluc Exp $ #ifndef PB_MGM_H #define PB_MGM_H @@ -41,8 +40,6 @@ bool Mgm(T1& A, T2& vh, const GridFunc& rho, const short cogr, A.rhs(res, rhs); // Hartree units - rhs *= (4. * M_PI); - // work GridFunc GridFunc lhs(finegrid, bcx, bcy, bcz); diff --git a/src/pb/Solver.h b/src/pb/Solver.h index 062fbaa2..f82b3b6a 100644 --- a/src/pb/Solver.h +++ b/src/pb/Solver.h @@ -7,9 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: Solver.h,v 1.14 2010/01/28 22:56:47 jeanluc Exp $ -#ifndef SOLVE_H -#define SOLVE_H +#ifndef PB_SOLVER_H +#define PB_SOLVER_H #include "GridFunc.h" @@ -33,7 +32,7 @@ class Solver fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); } - virtual bool solve(GridFunc&, GridFunc&) = 0; + virtual bool solve(GridFunc&, const GridFunc&) = 0; virtual void setup(const short nu1, const short nu2, const short max_sweeps, const double tol, const short max_nlevels, diff --git a/src/pb/SolverLap.cc b/src/pb/SolverLap.cc index 5a85fefe..062755a8 100644 --- a/src/pb/SolverLap.cc +++ b/src/pb/SolverLap.cc @@ -7,8 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverLap.cc,v 1.15 2010/01/28 22:56:31 jeanluc Exp $ - #include "SolverLap.h" #include "Laph2.h" #include "Laph4.h" @@ -26,9 +24,8 @@ Timer vcycle_repvcycle_tm("vcycle_repvcycle"); namespace pb { - // explicit instantiation declaration -#if 1 + // double template class SolverLap, double>; template class SolverLap, double>; @@ -45,77 +42,7 @@ template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; template class SolverLap, float>; -#else -// double -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); -template bool SolverLap, double>::solve( - double*, double*, const char, const int, const int); - -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, double>::solve( - GridFunc&, GridFunc&, const int); -// float -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - float*, float*, const char, const int, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -template bool SolverLap, float>::solve( - GridFunc&, GridFunc&, const int); -#endif -/* -template bool Mgm(Laph4MP&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4M&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph2&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph4&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(Laph6&, GridFunc&, const GridFunc&, const -short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -template bool Mgm(ShiftedLaph4M&, GridFunc&, const GridFunc&, -const short, const short, const double, const short, const short, const bool, - double&,double&,double&,short&); -*/ -/* -template int Vcycle(ShiftedLaph4M&, GridFunc&, const GridFunc -&, const short, const short, const short, const bool); template int -Vcycle(Laph4MP&, GridFunc&, const GridFunc &, const short, -const short, const short, const bool); template int Vcycle(Laph4M&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); template int Vcycle(Laph2&, GridFunc&, const -GridFunc &, const short, const short, const short, const bool); -template int Vcycle(Laph4&, GridFunc&, const GridFunc &, const -short, const short, const short, const bool); template int Vcycle(Laph6&, -GridFunc&, const GridFunc &, const short, const short, const -short, const bool); -*/ template bool SolverLap::solve(T2* phi, T2* rhs, const char dis) { @@ -139,7 +66,7 @@ bool SolverLap::solve(T2* phi, T2* rhs, const char dis) } template -bool SolverLap::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverLap::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { bool conv = Mgm(oper_, gf_phi, gf_rhs, max_nlevels_, max_sweeps_, tol_, nu1_, nu2_, gather_coarse_level_, final_residual_, diff --git a/src/pb/SolverLap.h b/src/pb/SolverLap.h index 939e15bc..567bed05 100644 --- a/src/pb/SolverLap.h +++ b/src/pb/SolverLap.h @@ -64,7 +64,7 @@ class SolverLap : public Solver bool solve(T2* phi, T2* rhs, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; short getNbSweeps() const override { return nb_sweeps_; } double getFinalResidual() const override { return final_residual_; } diff --git a/src/pb/SolverPB.cc b/src/pb/SolverPB.cc index 53252c26..756b49d8 100644 --- a/src/pb/SolverPB.cc +++ b/src/pb/SolverPB.cc @@ -7,7 +7,6 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// $Id: SolverPB.cc,v 1.13 2010/01/28 22:56:32 jeanluc Exp $ #include "SolverPB.h" #include "Mgm.h" #include "PBh2.h" @@ -85,7 +84,7 @@ bool SolverPB::solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis) } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks) { oper_.init(gf_rhod); @@ -101,7 +100,7 @@ bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs, } template -bool SolverPB::solve(GridFunc& gf_phi, GridFunc& gf_rhs) +bool SolverPB::solve(GridFunc& gf_phi, const GridFunc& gf_rhs) { if (!oper_.initialized()) { diff --git a/src/pb/SolverPB.h b/src/pb/SolverPB.h index a05e1d49..99eae0e3 100644 --- a/src/pb/SolverPB.h +++ b/src/pb/SolverPB.h @@ -64,9 +64,9 @@ class SolverPB : public Solver } bool solve(T2* phi, T2* rhs, T2* rhod, T2* vks, const char dis); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs, + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs, GridFunc& gf_rhod, GridFunc& gf_vks); - bool solve(GridFunc& gf_phi, GridFunc& gf_rhs) override; + bool solve(GridFunc& gf_phi, const GridFunc& gf_rhs) override; ~SolverPB() override{}; From a2ece1adc7c16334d03d785c63018259cd80208b Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 30 Sep 2024 14:08:59 -0400 Subject: [PATCH 03/58] Move some code into PoissonSolverFactory (#279) --- src/Electrostatic.cc | 200 ++------------------------------ src/Electrostatic.h | 1 + src/PoissonSolverFactory.h | 228 +++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 192 deletions(-) create mode 100644 src/PoissonSolverFactory.h diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 80cb754b..6ac4fef3 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -11,6 +11,7 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" #include "GridFactory.h" +#include "GridFunc.h" #include "Hartree.h" #include "Hartree_CG.h" #include "Ions.h" @@ -23,15 +24,6 @@ #include "ShiftedHartree.h" #include "mputils.h" -#include "GridFunc.h" -#include "Laph2.h" -#include "Laph4.h" -#include "Laph4M.h" -#include "Laph4MP.h" -#include "Laph6.h" -#include "Laph8.h" -#include "ShiftedLaph4M.h" - Timer Electrostatic::solve_tm_("Electrostatic::solve"); Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], @@ -49,109 +41,9 @@ Electrostatic::Electrostatic(PoissonFDtype lap_type, const short bcPoisson[3], Mesh* mymesh = Mesh::instance(); const pb::Grid& myGrid = mymesh->grid(); - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } - else // use PCG for Poisson Solver - { - if (screening_const > 0.) - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new ShiftedHartree>( - myGrid, bc_, screening_const); - break; - default: - (*MPIdata::sout) - << "PCG Electrostatic, shifted, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - else - { - switch (lap_type) - { - case PoissonFDtype::h4M: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h2: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h6: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h8: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - case PoissonFDtype::h4MP: - poisson_solver_ - = new Hartree_CG>(myGrid, bc_); - break; - default: - (*MPIdata::sout) << "PCG Electrostatic, Undefined option: " - << static_cast(lap_type) << std::endl; - } - } - } + // create Poisson solver + poisson_solver_ = PoissonSolverFactory::create( + myGrid, lap_type, bcPoisson, screening_const); grhoc_ = nullptr; diel_flag_ = false; @@ -244,73 +136,8 @@ void Electrostatic::setupPB( ngpts, origin, cell, static_cast(laptype_), true, myPEenv); if (poisson_solver_ != nullptr) delete poisson_solver_; - Control& ct = *(Control::instance()); - if (ct.MGPoissonSolver()) // use MG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } - else // use PCG for Poisson Solver - { - switch (laptype_) - { - case PoissonFDtype::h4M: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h2: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h6: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h8: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - case PoissonFDtype::h4MP: - poisson_solver_ = new PBdiel_CG>( - *pbGrid_, bc_, e0, rho0, drho0); - break; - default: - (*MPIdata::sout) - << "Electrostatic, Undefined option" << std::endl; - } - } + poisson_solver_ = PoissonSolverFactory::createDiel( + *pbGrid_, laptype_, bc_, e0, rho0, drho0); if (grhoc_ != nullptr) { @@ -330,6 +157,7 @@ void Electrostatic::setupPB( poisson_solver_->set_vh(gf_vh); } +// This function is only useful for Hartree problem with dielectric continuum void Electrostatic::fillFuncAroundIons(const Ions& ions) { assert(grhod_ != nullptr); @@ -352,7 +180,6 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) std::vector::const_iterator ion = rc_ions.begin(); while (ion != rc_ions.end()) { - double rc = (*ion)->getRC(); // Special case: silicon if ((*ion)->isMass28()) rc = 2.0; @@ -373,43 +200,32 @@ void Electrostatic::fillFuncAroundIons(const Ions& ions) #endif for (unsigned int ix = 0; ix < pbGrid_->dim(0); ix++) { - xc[1] = pbGrid_->start(1); const int ix1 = (ix + shift) * incx; for (unsigned int iy = 0; iy < pbGrid_->dim(1); iy++) { - xc[2] = pbGrid_->start(2); const int iy1 = ix1 + (iy + shift) * incy; for (unsigned int iz = 0; iz < pbGrid_->dim(2); iz++) { - const double r = (*ion)->minimage(xc, lattice, bc_); if (r < rc) { const double alpha = 0.2 * (1. + cos(r * pi_rc)); - - const int iz1 = iy1 + iz + shift; + const int iz1 = iy1 + iz + shift; vv[iz1] += alpha; } - xc[2] += pbGrid_->hgrid(2); } - xc[1] += pbGrid_->hgrid(1); - } // end for iy - xc[0] += pbGrid_->hgrid(0); - } // end for ix } - ion++; - } // end loop on list of ions return; diff --git a/src/Electrostatic.h b/src/Electrostatic.h index a3b86879..072ce0cc 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -13,6 +13,7 @@ #include "Control.h" #include "GridFunc.h" #include "Poisson.h" +#include "PoissonSolverFactory.h" #include "Rho.h" #include "Timer.h" diff --git a/src/PoissonSolverFactory.h b/src/PoissonSolverFactory.h new file mode 100644 index 00000000..91b99bf2 --- /dev/null +++ b/src/PoissonSolverFactory.h @@ -0,0 +1,228 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE +#ifndef MGMOL_PoissonSolverFactory +#define MGMOL_PoissonSolverFactory + +#include "Control.h" +#include "Hartree.h" +#include "Hartree_CG.h" +#include "Mesh.h" +#include "PBdiel.h" +#include "PBdiel_CG.h" +#include "ShiftedHartree.h" +#include "mputils.h" + +#include "GridFunc.h" +#include "Laph2.h" +#include "Laph4.h" +#include "Laph4M.h" +#include "Laph4MP.h" +#include "Laph6.h" +#include "Laph8.h" +#include "ShiftedLaph4M.h" + +class PoissonSolverFactory +{ + +public: + /*! + * return specific Poisson solver needed to solve Hartree problem + */ + static Poisson* create(const pb::Grid& myGrid, PoissonFDtype lap_type, + const short bc[3], const double screening_const) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + else // use PCG for Poisson Solver + { + if (screening_const > 0.) + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new ShiftedHartree>( + myGrid, bc, screening_const); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, shifted, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + else + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h2: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h6: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h8: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + case PoissonFDtype::h4MP: + poisson_solver + = new Hartree_CG>(myGrid, bc); + break; + default: + (*MPIdata::sout) + << "PCG Electrostatic, Undefined option: " + << static_cast(lap_type) << std::endl; + } + } + } + + return poisson_solver; + } + + static Poisson* createDiel(pb::Grid& pbGrid, PoissonFDtype lap_type, + const short bc[3], const double e0, const double rho0, + const double drho0) + { + Poisson* poisson_solver = nullptr; + + Control& ct = *(Control::instance()); + if (ct.MGPoissonSolver()) // use MG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + else // use PCG for Poisson Solver + { + switch (lap_type) + { + case PoissonFDtype::h4M: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h2: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h6: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h8: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + case PoissonFDtype::h4MP: + poisson_solver = new PBdiel_CG>( + pbGrid, bc, e0, rho0, drho0); + break; + default: + (*MPIdata::sout) + << "Electrostatic, Undefined option" << std::endl; + } + } + return poisson_solver; + } +}; + +#endif From 8b577ea75f805f6288e0c9e91fba75b960a59a39 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 3 Oct 2024 10:03:02 -0400 Subject: [PATCH 04/58] Clean up class Potentials (#280) --- src/MGmol.cc | 2 +- src/Potentials.cc | 24 +++----------------- src/Potentials.h | 56 ++++++++++++++++++++++++++++++++--------------- src/tools.cc | 17 +++++++++++++- src/tools.h | 1 + 5 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index be4f28aa..4aa1fa26 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -837,7 +837,7 @@ void MGmol::initNuc(Ions& ions) pot.initialize(ions); // Check compensating charges - double comp_rho = pot.getCharge(pot.rho_comp()); + double comp_rho = getCharge(pot.rho_comp()); if (onpe0 && ct.verbose > 1) { diff --git a/src/Potentials.cc b/src/Potentials.cc index ee874f10..380f05a0 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -27,6 +27,7 @@ #include using namespace std; +// unit conversion factor Ha -> Ry const double ha2ry = 2.; Potentials::~Potentials() @@ -57,13 +58,9 @@ Potentials::Potentials(const bool vh_frozen) size_ = dim_[0] * dim_[1] * dim_[2]; - mix_ = 1.; - scf_dvrho_ = 1000.; scf_dv_ = 1000.; - vh_frozen_ = vh_frozen; - vtot_.resize(size_); vtot_old_.resize(size_); @@ -101,7 +98,7 @@ void Potentials::initWithVnuc() double one = 1.; LinearAlgebraUtils::MPaxpy( size_, one, &v_ext_[0], &vtot_[0]); - // factor 2 to get total potential in [Ry] for calculations + // factor ha2ry to get total potential in [Ry] for calculations LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); } @@ -568,9 +565,8 @@ template void Potentials::setVxc(const T* const vxc, const int iterativeIndex) { assert(iterativeIndex >= 0); - // int ione=1; + itindex_vxc_ = iterativeIndex; - // Tcopy(&size_, vxc, &ione, &vxc_rho_[0], &ione); MPcpy(&vxc_rho_[0], vxc, size_); } void Potentials::setVh(const POTDTYPE* const vh, const int iterativeIndex) @@ -849,20 +845,6 @@ void Potentials::rescaleRhoComp() if (comp_rho < 0.) mmpi.abort(); } -double Potentials::getCharge(RHODTYPE* rho) -{ - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - double charge = mygrid.integralOverMesh(rho); - - if (onpe0 && ct.verbose > 0) - cout << setprecision(8) << fixed << "Charge: " << charge << endl; - - return charge; -} - void Potentials::addBackgroundToRhoComp() { if (fabs(background_charge_) > 0.) diff --git a/src/Potentials.h b/src/Potentials.h index 0cd7a8a6..205fb538 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -30,7 +30,6 @@ class Potentials int gdim_[3]; int dim_[3]; bool diel_; - double mix_; double scf_dvrho_; double scf_dv_; @@ -39,8 +38,9 @@ class Potentials double charge_in_cell_; double ionic_charge_; - bool vh_frozen_; - + /*! + * Total KS potential seen by electrons + */ std::vector vtot_; std::vector vtot_old_; @@ -48,16 +48,30 @@ class Potentials std::vector vh_rho_; std::vector vxc_rho_; - // nuclei local potential + /* + * Potential contribution from atomic cores (local pseudopotential) + */ std::vector v_nuc_; - // external potential (read from input) + /*! + * Optional external potential (read from input) + * Used only in special cases. + */ std::vector v_ext_; #ifdef HAVE_TRICUBIC pb::TriCubic* vext_tricubic_; #endif + /*! + * Potential associated with the sum of Gaussian charge distributions + * compensating the Coulomb potential of each atom + */ std::vector v_comp_; + + /*! + * Sum of Gaussian charge distributions compensating the Coulomb potential + * of each atom + */ std::vector rho_comp_; std::vector dv_; @@ -121,14 +135,11 @@ class Potentials void turnOnDiel() { diel_ = true; } int size() const { return size_; } - bool vh_frozen() const { return vh_frozen_; } - void freeze_vh() { vh_frozen_ = true; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return &vtot_[0]; } POTDTYPE* vh_rho() { return &vh_rho_[0]; } - POTDTYPE* vxc_rho() { return &vxc_rho_[0]; } RHODTYPE* rho_comp() { return &rho_comp_[0]; } const std::vector& vnuc() const { return v_nuc_; } @@ -136,12 +147,6 @@ class Potentials POTDTYPE* vext() { return &v_ext_[0]; } POTDTYPE* vepsilon() { return &vepsilon_[0]; } - void set_vcomp(const POTDTYPE val) - { - const int n = (int)v_comp_.size(); - for (int i = 0; i < n; i++) - v_comp_[i] = val; - } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -154,13 +159,29 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + /*! + * initialize total potential as local pseudopotential + */ void initWithVnuc(); void getVofRho(std::vector& vrho) const; - double delta_v(const std::vector>&); - double update(const std::vector>&); - void update(const double); + /*! + * evaluate potential correction associated with a new rho + */ + double delta_v(const std::vector>& rho); + + /*! + * update potentials based on argument rho + */ + double update(const std::vector>& rho); + + /*! + * update potentials based on potential correction delta v and mixing + * parameter + */ + void update(const double mix); + double max() const; double min() const; void readAll(std::vector& sp); @@ -172,7 +193,6 @@ class Potentials void initialize(Ions& ions); void rescaleRhoComp(); - double getCharge(RHODTYPE* rho); void initBackground(Ions& ions); void addBackgroundToRhoComp(); diff --git a/src/tools.cc b/src/tools.cc index 4e1bd314..95790c59 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -228,7 +228,7 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) int mpierr=MPI_Reduce(&s, &r, 1, MPI_INT, MPI_SUM, 0, mmpi.commGlobal()); if( mpierr!=MPI_SUCCESS ) { - cerr << " Error in MPI!!!" << endl; + cerr << " Error in MPI!!!" << std::endl; MPI_Abort(mmpi.commGlobal(),1); } if( r!=mmpi.size()*s && onpe0 ) @@ -388,3 +388,18 @@ void getkvector(const int index, const int kmax, int kvector[3]) // std::cout << " k=(" << kvector[0] << "," << kvector[1] << "," // << kvector[2] << ")" << std::endl; } + +double getCharge(double* rho) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double charge = mygrid.integralOverMesh(rho); + + if (onpe0 && ct.verbose > 0) + std::cout << std::setprecision(8) << std::fixed << "Charge: " << charge + << std::endl; + + return charge; +} diff --git a/src/tools.h b/src/tools.h index 4e8374da..be3d13e0 100644 --- a/src/tools.h +++ b/src/tools.h @@ -37,5 +37,6 @@ double minQuadPolynomial(const double e0, const double e1, const double de0, double minQuadPolynomialFrom3values(const double e0, const double e1, const double e12, const bool print_flag, std::ostream& os); void getkvector(const int index, const int kmax, int kvector[3]); +double getCharge(double* rho); #endif From 7ac691c7126e45223c5ea33d043f256de1a52519 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 11 Oct 2024 21:44:42 -0400 Subject: [PATCH 05/58] Clean up class Ions, add test for it (#281) --- src/Control.cc | 1 + src/Ions.cc | 324 ++++++++++++++++----------------------- src/Ions.h | 23 ++- src/KBprojectorSparse.cc | 4 +- tests/CMakeLists.txt | 8 + tests/testIons.cc | 104 +++++++++++++ 6 files changed, 265 insertions(+), 199 deletions(-) create mode 100644 tests/testIons.cc diff --git a/src/Control.cc b/src/Control.cc index af3d8ed9..dd786678 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -70,6 +70,7 @@ Control::Control() dm_approx_ndigits = 1; dm_approx_power_maxits = 100; wf_extrapolation_ = 1; + verbose = 0; // undefined values dm_algo_ = -1; diff --git a/src/Ions.cc b/src/Ions.cc index 940e12e8..852151c7 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -584,11 +584,9 @@ void Ions::printPositionsLocal(std::ostream& os, const int root) const os.setf(std::ios::right, std::ios::adjustfield); os.setf(std::ios::fixed, std::ios::floatfield); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->printPosition(os); - ion++; + ion->printPosition(os); } os << std::endl; @@ -627,14 +625,12 @@ void Ions::writeAtomicNumbers(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - assert((*ion)->atomic_number() > 0); - assert((*ion)->atomic_number() < 200); + assert(ion->atomic_number() > 0); + assert(ion->atomic_number() < 200); - data.push_back((*ion)->atomic_number()); - ion++; + data.push_back(ion->atomic_number()); } } @@ -685,17 +681,15 @@ void Ions::writeAtomNames(HDFrestart& h5f_file) void Ions::lockAtom(const std::string& name) { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - std::string name_ion((*ion)->name()); + std::string name_ion(ion->name()); if (name.compare(name_ion) == 0) { - (*ion)->lock(); + ion->lock(); if (onpe0) (*MPIdata::sout) << "Lock atom " << name << std::endl; break; } - ion++; } } @@ -975,12 +969,10 @@ void Ions::readRestartPositions(HDFrestart& h5_file) std::vector data; h5_file.readAtomicPositions(data); - int i = 0; - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1109,14 +1101,12 @@ void Ions::removeMassCenterMotion() #endif } - ion = local_ions_.begin(); - i = 0; - while (ion != local_ions_.end()) + i = 0; + for (auto& ion : local_ions_) { const int threei = 3 * i; - (*ion)->setVelocity(velocities[threei] - tmp[0], + ion->setVelocity(velocities[threei] - tmp[0], velocities[threei + 1] - tmp[1], velocities[threei + 2] - tmp[2]); - ion++; i++; } @@ -1124,16 +1114,15 @@ void Ions::removeMassCenterMotion() mv[0] = 0.; mv[1] = 0.; mv[2] = 0.; - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - velocities[3 * i] = (*ion)->velocity(0); - velocities[3 * i + 1] = (*ion)->velocity(1); - velocities[3 * i + 2] = (*ion)->velocity(2); + velocities[3 * i] = ion->velocity(0); + velocities[3 * i + 1] = ion->velocity(1); + velocities[3 * i + 2] = ion->velocity(2); mv[0] += mass[i] * velocities[3 * i]; mv[1] += mass[i] * velocities[3 * i + 1]; mv[2] += mass[i] * velocities[3 * i + 2]; - ion++; i++; } @@ -1155,12 +1144,10 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) std::vector data; h5_file.readAtomicVelocities(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setVelocity(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1174,12 +1161,10 @@ void Ions::readRestartRandomStates(HDFrestart& h5f_file) std::vector data; h5f_file.readRestartRandomStates(data); - std::vector::iterator ion = local_ions_.begin(); - int i = 0; - while (ion != local_ions_.end()) + int i = 0; + for (auto& ion : local_ions_) { - (*ion)->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); - ion++; + ion->setRandomState(data[3 * i], data[3 * i + 1], data[3 * i + 2]); i++; } } @@ -1202,17 +1187,14 @@ void Ions::writeForces(HDFrestart& h5f_file) } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get position of local ion double force[3]; - (*ion)->getForce(&force[0]); + ion->getForce(&force[0]); data.push_back(force[0]); data.push_back(force[1]); data.push_back(force[2]); - - ++ion; } } @@ -1364,64 +1346,54 @@ void Ions::printForcesLocal(std::ostream& os, const int root) const << "FX" << std::setw(10) << "FY" << std::setw(10) << "FZ" << std::endl; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { + ion->printPositionAndForce(os); - (*ion)->printPositionAndForce(os); - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } else { - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - - if (!(*ion)->locked()) + if (!ion->locked()) { + avg_forces[0] += fabs(ion->force(0)); + avg_forces[1] += fabs(ion->force(1)); + avg_forces[2] += fabs(ion->force(2)); - avg_forces[0] += fabs((*ion)->force(0)); - avg_forces[1] += fabs((*ion)->force(1)); - avg_forces[2] += fabs((*ion)->force(2)); - - double ff = (*ion)->norm2F(); + double ff = ion->norm2F(); maxf[0] = std::max(maxf[0], ff); - max_forces[0] = std::max(max_forces[0], fabs((*ion)->force(0))); - max_forces[1] = std::max(max_forces[1], fabs((*ion)->force(1))); - max_forces[2] = std::max(max_forces[2], fabs((*ion)->force(2))); + max_forces[0] = std::max(max_forces[0], fabs(ion->force(0))); + max_forces[1] = std::max(max_forces[1], fabs(ion->force(1))); + max_forces[2] = std::max(max_forces[2], fabs(ion->force(2))); num_movable++; } for (short ii = 0; ii < 3; ii++) - sum_forces[ii] += (*ion)->force(ii); + sum_forces[ii] += ion->force(ii); num_atoms++; - - ion++; } } // global statistics @@ -1479,12 +1451,10 @@ int Ions::countIonsHere() const { return (int)local_ions_.size(); } int Ions::countProjectorsHere() const { - int count = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int count = 0; + for (auto& ion : local_ions_) { - count += (*ion)->nProjectors(); - ion++; + count += ion->nProjectors(); } return count; } @@ -1497,12 +1467,10 @@ int Ions::countProjectors() const Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - int nproj = 0; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + int nproj = 0; + for (auto& ion : local_ions_) { - nproj += (*ion)->nProjectors(); - ion++; + nproj += ion->nProjectors(); } int tmp = nproj; MPI_Allreduce(&tmp, &nproj, 1, MPI_INT, MPI_SUM, myPEenv.comm()); @@ -1569,13 +1537,10 @@ void Ions::setLocalPositions(const std::vector& tau) { assert(tau.size() == 3 * local_ions_.size()); - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); - - ion++; + ion->setPosition(tau[3 * ia + 0], tau[3 * ia + 1], tau[3 * ia + 2]); ia++; } @@ -1780,45 +1745,8 @@ int Ions::setAtoms( if (ia < 1000) aname.append("0"); if (ia < 10000) aname.append("0"); aname.append(ss.str()); - Ion* new_ion - = new Ion(species_[isp], aname, &crds[3 * ia], velocity, locked); - new_ion->bcast(mmpi.commGlobal()); - - // Populate list_ions_ list - // std::cout<<"crds: "< 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " << crds[3 * ia + 0] - << "," << crds[3 * ia + 1] << "," << crds[3 * ia + 2] - << " added to the list... on PE" << mmpi.mypeGlobal() - << std::endl; - // populate local_ions_ list - if (inLocalIons( - crds[3 * ia + 0], crds[3 * ia + 1], crds[3 * ia + 2])) - { - (new_ion)->set_here(true); - local_ions_.push_back(new_ion); - if (onpe0 && ct.verbose > 2) - (*MPIdata::sout) - << "Ion " << aname << " at position " - << crds[3 * ia + 0] << "," << crds[3 * ia + 1] << "," - << crds[3 * ia + 2] - << " added to the list of local ions... on PE" - << mmpi.mypeGlobal() << std::endl; - } - else - (new_ion)->set_here(false); - } - else - { - //(*MPIdata::sout)<<"Ion "<set_here(true); + local_ions_.push_back(new_ion); + if (onpe0 && ct.verbose > 2) + (*MPIdata::sout) << "Ion " << name << " at position " << crds[0] + << "," << crds[1] << "," << crds[2] + << " added to the list of local ions... on PE" + << mmpi.mypeGlobal() << std::endl; + } + else + (new_ion)->set_here(false); + } + else + { + // delete Ion if not put in list + delete new_ion; + } +} + int Ions::readNatoms(const std::string& filename, const bool cell_relative) { Control& ct(*(Control::instance())); @@ -2117,9 +2085,8 @@ void Ions::setVelocities(const std::vector& tau0, assert(tau0.size() == 3 * local_ions_.size()); assert(taup.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { double v[3]; for (short i = 0; i < 3; i++) @@ -2128,8 +2095,7 @@ void Ions::setVelocities(const std::vector& tau0, v[i] -= tau0[3 * ia + i]; v[i] /= dt; } - (*iion)->setVelocity(v[0], v[1], v[2]); - iion++; + ion->setVelocity(v[0], v[1], v[2]); ia++; } } @@ -2138,12 +2104,10 @@ void Ions::getLocalPositions(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau[3 * ia]); - iion++; + ion->getPosition(&tau[3 * ia]); ia++; } } @@ -2188,12 +2152,10 @@ void Ions::setTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->getPosition(&tau0_[3 * ia]); - iion++; + ion->getPosition(&tau0_[3 * ia]); ia++; } } @@ -2202,13 +2164,11 @@ void Ions::setPositionsToTau0() { assert(tau0_.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*iion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - iion++; ia++; } } @@ -2244,13 +2204,11 @@ void Ions::getLocalForces(std::vector& tau) const { assert(tau.size() == 3 * local_ions_.size()); - int ia = 0; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { assert(3 * ia + 2 < (int)tau.size()); - (*iion)->getForce(&tau[3 * ia]); - iion++; + ion->getForce(&tau[3 * ia]); ia++; } } @@ -2351,12 +2309,10 @@ double Ions::computeMaxVlRadius() const { double radius = 0.; - std::vector::const_iterator iion = local_ions_.begin(); - while (iion != local_ions_.end()) + for (auto& ion : local_ions_) { - double r = (*iion)->computeRadiusVl(); + double r = ion->computeRadiusVl(); radius = r > radius ? r : radius; - iion++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -2393,13 +2349,11 @@ void Ions::gatherNames(std::map& names, const int root, std::vector data; std::vector local_names; - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { // get local name and index - local_names.push_back((*ion)->name()); - local_indexes.push_back((*ion)->index()); - ++ion; + local_names.push_back(ion->name()); + local_indexes.push_back(ion->index()); } // gather data to PE root @@ -2695,15 +2649,13 @@ bool Ions::hasLockedAtoms() const return (flag == 1); } -double Ions::getMaxNLradius() const +double Ions::getSpeciesMaxNLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double nlradius = spi->nlradius(); + const double nlradius = spi.nlradius(); radius = radius > nlradius ? radius : nlradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2711,15 +2663,13 @@ double Ions::getMaxNLradius() const return radius; } -double Ions::getMaxLradius() const +double Ions::getSpeciesMaxLradius() const { - double radius = 0; - std::vector::const_iterator spi = species_.begin(); - while (spi != species_.end()) + double radius = 0.; + for (const auto& spi : species_) { - const double lradius = spi->lradius(); + const double lradius = spi.lradius(); radius = radius > lradius ? radius : lradius; - spi++; } MGmol_MPI& mmpi(*(MGmol_MPI::instance())); mmpi.allreduce(&radius, 1, MPI_MAX); @@ -2918,8 +2868,8 @@ void Ions::computeNumIons(void) double Ions::getMaxListRadius() const { // get radius of projectors - const double nlradius = getMaxNLradius(); - const double lradius = getMaxLradius(); + const double nlradius = getSpeciesMaxNLradius(); + const double lradius = getSpeciesMaxLradius(); double rmax = nlradius > lradius ? nlradius : lradius; @@ -3289,21 +3239,15 @@ void Ions::updateIons() Control& ct(*(Control::instance())); // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - int ia = 0; - while (ion != local_ions_.end()) + int ia = 0; + for (auto& ion : local_ions_) { - (*ion)->setPosition( + ion->setPosition( tau0_[3 * ia + 0], tau0_[3 * ia + 1], tau0_[3 * ia + 2]); - // (*ion)->setForce(fion_[3*ia+0], - // fion_[3*ia+1], - // fion_[3*ia+2]); - - (*ion)->setRandomState(rand_states_[3 * ia + 0], - rand_states_[3 * ia + 1], rand_states_[3 * ia + 2]); + ion->setRandomState(rand_states_[3 * ia + 0], rand_states_[3 * ia + 1], + rand_states_[3 * ia + 2]); - ion++; ia++; } diff --git a/src/Ions.h b/src/Ions.h index 772c51d1..ae4f6adb 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -71,13 +71,14 @@ class Ions void setMapVL(); double computeMaxVlRadius() const; double computeMaxNLprojRadius() const; - double getMaxNLradius() const; - double getMaxLradius() const; - void updateListIons(); - // compute boundary for box containing all atoms to be known on local - // processor that is values for list_boundary_left_, list_boundary_right_ - void setupListIonsBoundaries(const double rmax); + /* + * Evaluate maximum pseudopotential radius among all species in class + */ + double getSpeciesMaxNLradius() const; + double getSpeciesMaxLradius() const; + + void updateListIons(); void augmentIonsData(const int nsteps, const int dir, const int disp, const int locSize, const int maxLocSize, std::vector& data, @@ -163,6 +164,8 @@ class Ions bool hasLockedAtoms() const; void clearLists(); + void rescaleVelocities(const double factor); + public: Ions(const double lat[3], const std::vector& sp); @@ -172,6 +175,10 @@ class Ions void setup(); + // compute boundary for box containing all atoms to be known on local + // processor that is values for list_boundary_left_, list_boundary_right_ + void setupListIonsBoundaries(const double rmax); + std::vector& getLocalNames() { return local_names_; } std::vector& getTau0() { return tau0_; } std::vector& getTaup() { return taup_; } @@ -334,13 +341,15 @@ class Ions void updateForcesInteractingIons(); void updateTaupInteractingIons(); - void rescaleVelocities(const double factor); /*! * Calculate minimum distance between local pairs */ double computeMinLocalSpacing() const; + void addIonToList(const Species& sp, const std::string& name, + const double crds[3], const double velocity[3], const bool lock); + // void checkUnicityLocalIons(); }; diff --git a/src/KBprojectorSparse.cc b/src/KBprojectorSparse.cc index afd8871c..937b95d2 100644 --- a/src/KBprojectorSparse.cc +++ b/src/KBprojectorSparse.cc @@ -103,7 +103,7 @@ void KBprojectorSparse::setNLindex( for (int i = 0; i < size_nl; i++) { - assert(i < lnumpt); + // assert(i < lnumpt); if ((pvec[i] < iloc * lnumpt) || (pvec[i] >= (iloc + 1) * lnumpt)) { (*MPIdata::sout) << " iloc=" << iloc << ", i=" << i @@ -960,7 +960,7 @@ bool KBprojectorSparse::setIndexesAndProjectors() // get "pvec" and "is_in_domain" int icount = get_index_array(pvec, iloc, index_low, index_high); assert(icount <= nl3); - assert(icount <= mymesh->npointsPatch()); + // assert(icount <= mymesh->npointsPatch()); assert(icount > 0); setNLindex(iloc, icount, pvec); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc205167..6de34971 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -209,6 +209,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) +add_executable(testIons + ${CMAKE_SOURCE_DIR}/tests/testIons.cc) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -334,6 +336,10 @@ add_test(NAME testBatchLaph4 add_test(NAME testtMGkernels COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMGkernels) +add_test(NAME testIons + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testIons + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) @@ -541,6 +547,7 @@ target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -550,6 +557,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testIons PRIVATE mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/testIons.cc b/tests/testIons.cc new file mode 100644 index 00000000..30b2ad0c --- /dev/null +++ b/tests/testIons.cc @@ -0,0 +1,104 @@ +#include "Control.h" +#include "Ions.h" +#include "MGmol_MPI.h" +#include "Mesh.h" +#include "Species.h" + +#include + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + + MPI_Comm comm = MPI_COMM_WORLD; + + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + MGmol_MPI::setup(comm, std::cout); + Control::setup(comm, false, 0.); + + // create a domain [0.10.]^3 + const double origin[3] = { 0., 0., 0. }; + const double ll = 10.; + const double lattice[3] = { ll, ll, ll }; + const unsigned ngpts[3] = { 32, 24, 20 }; + short lap_type = 0; + + Mesh::setup(comm, ngpts, origin, lattice, lap_type); + + const double h[3] = { ll / (double(ngpts[0])), ll / (double(ngpts[1])), + ll / (double(ngpts[2])) }; + + // random number generator + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_real_distribution<> dis(0.0, 1.0); + + // create one species + Species sp(MPI_COMM_WORLD); + + // read species info from pseudopotential file + std::string file_path = argv[1]; + std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); + std::cout << "Potential = " << filename << std::endl; + + sp.read_1species(filename); + sp.set_dim_nl(h[0]); + sp.set_dim_l(h[0]); + sp.initPotentials('f', h[0], true); + + // put species into a vector + std::vector vsp; + vsp.push_back(sp); + + Ions ions(lattice, vsp); + ions.setupListIonsBoundaries(10000.); + + double velocity[3] = { 0., 0., 0. }; + + // set "na" atoms coordinates and add them to "ions" + const int na = 10; + for (int i = 0; i < na; i++) + { + double x[3] = { origin[0] + lattice[0] * dis(gen), + origin[1] + lattice[1] * dis(gen), + origin[2] + lattice[2] * dis(gen) }; + if (myrank == 0) + std::cout << "x,y,z = " << x[0] << ", " << x[1] << ", " << x[2] + << std::endl; + + // set all x to the values of PE0 + MPI_Bcast(&x[0], 3, MPI_DOUBLE, 0, comm); + + // make a name for atom based on species and order of reading in + std::string stri = std::to_string(i); + std::string aname("C" + stri); + + ions.addIonToList(sp, aname, &x[0], velocity, false); + } + + ions.setup(); + + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + return 1; + } + + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + + return 0; +} From c05ba7f4cccb1077faa226e9bb79fb7294c93130 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 25 Nov 2024 13:35:01 -0500 Subject: [PATCH 06/58] Add test MD_MVP (#290) --- tests/CMakeLists.txt | 8 ++++ tests/MD_MVP/li2.xyz | 4 ++ tests/MD_MVP/md.cfg | 39 ++++++++++++++++++ tests/MD_MVP/quench.cfg | 34 ++++++++++++++++ tests/MD_MVP/test.py | 89 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+) create mode 100644 tests/MD_MVP/li2.xyz create mode 100644 tests/MD_MVP/md.cfg create mode 100644 tests/MD_MVP/quench.cfg create mode 100755 tests/MD_MVP/test.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6de34971..72378704 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -491,6 +491,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMD_MVP + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/quench.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/li2.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testLBFGS COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/LBFGS/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/MD_MVP/li2.xyz b/tests/MD_MVP/li2.xyz new file mode 100644 index 00000000..fce3cd72 --- /dev/null +++ b/tests/MD_MVP/li2.xyz @@ -0,0 +1,4 @@ +2 + +Li 1.33 0.0 0.0 +Li -1.33 0.0 0.0 diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg new file mode 100644 index 00000000..83d5aa4c --- /dev/null +++ b/tests/MD_MVP/md.cfg @@ -0,0 +1,39 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=MD +[MD] +num_steps=5 +dt=15. +[Quench] +solver=PSD +max_steps=30 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +input_filename=wave.out +input_level=3 +output_level=3 diff --git a/tests/MD_MVP/quench.cfg b/tests/MD_MVP/quench.cfg new file mode 100644 index 00000000..135dbde7 --- /dev/null +++ b/tests/MD_MVP/quench.cfg @@ -0,0 +1,34 @@ +verbosity=0 +xcFunctional=LDA +FDtype=4th +[Mesh] +nx=32 +ny=32 +nz=32 +[Domain] +ox=-5. +oy=-5. +oz=-5. +lx=10. +ly=10. +lz=10. +[Potentials] +pseudopotential=pseudo.Li_ONCVPSP_LDA +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +nempty=1 +initial_type=random +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 +mixing=1. +[Restart] +output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py new file mode 100755 index 00000000..489c9c8c --- /dev/null +++ b/tests/MD_MVP/test.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string +import shutil + +print("Test MD with MVP solver...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-5): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-5] +inp1 = sys.argv[nargs-4] +inp2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst = 'pseudo.Li_ONCVPSP_LDA' +src = sys.argv[-1] + '/' + dst + +if not os.path.exists(dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run quench +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp1,coords) +print("Run command: {}".format(command)) +output1 = subprocess.check_output(command,shell=True) +lines=output1.split(b'\n') + +#analyse output of quench +for line in lines: + num_matches = line.count(b'%%') + if num_matches: + print(line) + +#run MD +for i in range(2): + command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + print(command) + restart_file = subprocess.check_output(command,shell=True) + restart_file=str(restart_file[:-1],'utf-8') + print(restart_file) + + os.rename(restart_file, 'wave.out') + + #run MGmol + command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) + output2 = subprocess.check_output(command,shell=True) + + #remove used restart files + shutil.rmtree('wave.out') + + #analyse mgmol standard output + lines=output2.split(b'\n') + + print("Check energy conservation...") + tol = 1.e-4 + energy = 0. + count = 0 + for line in lines: + if line.count(b'%%'): + print(line) + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +#remove last restart files +command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +restart_file = subprocess.check_output(command,shell=True) +restart_file=str(restart_file[:-1],'utf-8') +shutil.rmtree(restart_file) + +sys.exit(0) From bb8051d2cce7299c4baf61f863a51488476aea1a Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 2 Dec 2024 08:17:13 -0500 Subject: [PATCH 07/58] Clean up code related to DM restart data (#292) --- src/DensityMatrix.cc | 58 +++++- src/DensityMatrix.h | 10 +- src/ExtendedGridOrbitals.cc | 28 +-- src/ExtendedGridOrbitals.h | 3 +- src/HDFrestart.cc | 8 +- src/HDFrestart.h | 32 ++-- src/LBFGS_IonicStepper.cc | 13 +- src/LocGridOrbitals.cc | 27 +-- src/LocGridOrbitals.h | 3 +- src/MGmol.h | 2 +- src/ProjectedMatrices.cc | 106 +---------- src/ProjectedMatrices.h | 11 +- src/ProjectedMatricesInterface.h | 10 +- src/hdf_tools.cc | 292 ++++++++++++++++++++----------- src/hdf_tools.h | 6 +- src/md.cc | 31 ++-- src/restart.cc | 7 +- tests/CMakeLists.txt | 5 +- 18 files changed, 338 insertions(+), 314 deletions(-) diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index e2c7b473..78d7008e 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -12,6 +12,8 @@ #include "DistMatrix.h" #include "MGmol_MPI.h" #include "ReplicatedMatrix.h" +#include "ReplicatedWorkSpace.h" +#include "hdf_tools.h" #include #include @@ -22,25 +24,28 @@ const double factor_kernel4dot = 10.; #define PROCRUSTES 0 +#define MGMOL_DENSITYMATRIX_FAIL(X) \ + { \ + std::cerr << "DensityMatrix failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } + // occupations in [0,1] // DM eigenvalues in [0,orbital_occupation] template DensityMatrix::DensityMatrix(const int ndim) + : dim_(ndim), + orbitals_index_(-1), + occ_uptodate_(false), + uniform_occ_(false), + stripped_(false) { assert(ndim > 0); - dim_ = ndim; - - occ_uptodate_ = false; - stripped_ = false; - uniform_occ_ = false; - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; - orbitals_index_ = -1; - dm_ = new MatrixType("DM", ndim, ndim); kernel4dot_ = new MatrixType("K4dot", ndim, ndim); work_ = new MatrixType("work", ndim, ndim); @@ -438,6 +443,43 @@ void DensityMatrix::mix( orbitals_index_ = new_orbitals_index; } +template +int DensityMatrix::write(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + + wspace.initSquareMatrix(*dm_); + + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); +} + +template +int DensityMatrix::read(HDFrestart& h5f_file, std::string& name) +{ + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + int ierr = 0; + if (mmpi.instancePE0()) + { + hid_t file_id = h5f_file.file_id(); + ierr = mgmol_tools::read_matrix(file_id, name, work_matrix); + } + mmpi.bcast(&ierr, 1); + + if (ierr >= 0) wspace.mpiBcastSquareMatrix(); + if (ierr >= 0) initMatrix(work_matrix); + + return ierr; +} + template class DensityMatrix>; #ifdef HAVE_MAGMA template class DensityMatrix; diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 84804a86..1d09e79e 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -10,6 +10,7 @@ #ifndef MGMOL_DENSITYMATRIX_H #define MGMOL_DENSITYMATRIX_H +#include "HDFrestart.h" #include "MGmol_MPI.h" #include "global.h" @@ -23,7 +24,7 @@ template class DensityMatrix { - int dim_; + const int dim_; std::vector occupation_; MatrixType* dm_; @@ -35,6 +36,10 @@ class DensityMatrix bool occ_uptodate_; bool uniform_occ_; bool stripped_; + + /*! + * Max. occupation of an orbital: 1 with spin, 2 otherwise + */ double orbital_occupation_; DensityMatrix(); @@ -146,6 +151,9 @@ class DensityMatrix double getExpectation(const MatrixType& A); void mix( const double mix, const MatrixType& matA, const int new_orbitals_index); + + int write(HDFrestart& h5f_file, std::string& name); + int read(HDFrestart& h5f_file, std::string& name); }; #endif diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 12e5b04c..19c0d26b 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -584,7 +584,6 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -603,7 +602,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -618,28 +617,7 @@ int ExtendedGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int ExtendedGridOrbitals::write_hdf5( - HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int ExtendedGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int ExtendedGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { if (onpe0) (*MPIdata::sout) << "ExtendedGridOrbitals::write_func_hdf5()...\n"; @@ -829,7 +807,7 @@ int ExtendedGridOrbitals::read_func_hdf5( const std::string datasetname(getDatasetName(name, icolor)); // check if dataset exists... - int err_id = h5f_file.dset_exists(datasetname); + int err_id = h5f_file.checkDataExists(datasetname); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index fb7a4b42..c1a718d0 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -374,8 +374,7 @@ class ExtendedGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, ExtendedGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index f95292ac..08018d47 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -933,12 +933,12 @@ int HDFrestart::getLRCenters(std::multimap& centers, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) @@ -1052,12 +1052,12 @@ int HDFrestart::getLRs(std::shared_ptr lrs, std::string datasetname(getDatasetName(name, color)); - int err_id = dset_exists(datasetname); + int err_id = checkDataExistsLocal(datasetname); if (err_id == 0) { // dataset does not exists // try older version datasetname = getDatasetName_old(name, color); - err_id = dset_exists(datasetname); + err_id = checkDataExistsLocal(datasetname); } if (err_id == 0) { // dataset does not exists diff --git a/src/HDFrestart.h b/src/HDFrestart.h index fef634c2..7ffadff7 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -155,28 +155,30 @@ class HDFrestart return 0; } - hid_t dset_exists(const std::string& datasetname) const + hid_t checkDataExists(const std::string& datasetname) const { - if (active_) - { - return dset_exists(datasetname.c_str()); - } - return 0; + return checkDataExists(datasetname.c_str()); } - herr_t dset_exists(const char* const datasetname) const + herr_t checkDataExists(const char* const datasetname) const + { + herr_t err_id = checkDataExistsLocal(datasetname); + + short id = (short)err_id; + MPI_Bcast(&id, 1, MPI_SHORT, 0, comm_data_); + return (herr_t)id; + } + + hid_t checkDataExistsLocal(const std::string& datasetname) const + { + return checkDataExistsLocal(datasetname.c_str()); + } + + herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; if (active_) { err_id = H5LTfind_dataset(file_id_, datasetname); - - if (err_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::dset_exists() failed for dataset " - << datasetname << std::endl; - } } return err_id; } diff --git a/src/LBFGS_IonicStepper.cc b/src/LBFGS_IonicStepper.cc index caea5f84..93ce03f5 100644 --- a/src/LBFGS_IonicStepper.cc +++ b/src/LBFGS_IonicStepper.cc @@ -328,11 +328,11 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) short check_data = 0; std::vector attr_d(16); + // Open an existing dataset. + std::string datasetname("/LBFGS"); + int err_id = h5f_file.checkDataExists(datasetname); if (onpe0) { - // Open an existing dataset. - std::string datasetname("/LBFGS"); - int err_id = h5f_file.dset_exists(datasetname); if (err_id < 0) { // dataset does not exists (*MPIdata::sout) << "Warning: no dataset " << datasetname @@ -343,10 +343,9 @@ int LBFGS_IonicStepper::read_lbfgs(HDFrestart& h5f_file) dataset_id = H5Dopen2(file_id, "/LBFGS", H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " - "no restart info for LBFGS" - << std::endl; + (*MPIdata::sout) << "Warning: H5Dopen failed for /LBFGS-> " + "no restart info for LBFGS" + << std::endl; } else { diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index ae8a4d1d..4a1b9442 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -966,7 +966,6 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) Control& ct = *(Control::instance()); - hid_t file_id = h5f_file.file_id(); std::string name = "Function"; int ierr = read_func_hdf5(h5f_file, name); if (ierr < 0) @@ -984,7 +983,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) // Read DM if (!ct.fullyOccupied()) { - ierr = proj_matrices_->read_dm_hdf5(file_id); + ierr = proj_matrices_->readDM(h5f_file); if (ierr < 0) { (*MPIdata::serr) @@ -997,27 +996,7 @@ int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) return ierr; } -int LocGridOrbitals::write_hdf5(HDFrestart& h5f_file, const std::string& name) -{ - assert(proj_matrices_ != nullptr); - Control& ct = *(Control::instance()); - - if (!ct.fullyOccupied()) - { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - mmpi.barrier(); - - int ierr = proj_matrices_->writeDM_hdf5(h5f_file); - if (ierr < 0) return ierr; - } - - int ierr = write_func_hdf5(h5f_file, name); - - return ierr; -} - -int LocGridOrbitals::write_func_hdf5( - HDFrestart& h5f_file, const std::string& name) +int LocGridOrbitals::write(HDFrestart& h5f_file, const std::string& name) { Control& ct = *(Control::instance()); hid_t file_id = h5f_file.file_id(); @@ -1249,7 +1228,7 @@ int LocGridOrbitals::read_func_hdf5( const std::string key(itcenter->first); // checkif dataset exists... - int err_id = h5f_file.dset_exists(key); + int err_id = h5f_file.checkDataExistsLocal(key); if (h5f_file.gatherDataX()) mmpi.bcast(&err_id, 1); if (err_id == 0) break; // dataset does not exists diff --git a/src/LocGridOrbitals.h b/src/LocGridOrbitals.h index 5616c6b6..951e418f 100644 --- a/src/LocGridOrbitals.h +++ b/src/LocGridOrbitals.h @@ -410,8 +410,7 @@ class LocGridOrbitals : public Orbitals void multiplyByMatrix2states(const int st1, const int st2, const double* mat, LocGridOrbitals& product); - int write_hdf5(HDFrestart& h5f_file, const std::string& name = "Function"); - int write_func_hdf5(HDFrestart&, const std::string& name = "Function"); + int write(HDFrestart&, const std::string& name = "Function"); int read_hdf5(HDFrestart& h5f_file); int read_func_hdf5(HDFrestart&, const std::string& name = "Function"); diff --git a/src/MGmol.h b/src/MGmol.h index aee2b660..4d82dd64 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -130,7 +130,7 @@ class MGmol : public MGmolInterface KBPsiMatrixSparse* kbpsi, dist_matrix::DistMatrix& hij); void computeHnlPhiAndAdd2HPhi(Ions& ions, OrbitalsType& phi, OrbitalsType& hphi, const KBPsiMatrixSparse* const kbpsi); - int dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, + int dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count); diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 3606aa40..e50e98bc 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -26,6 +26,7 @@ #include "SparseDistMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include "fermi.h" +#include "hdf_tools.h" #include #include @@ -732,114 +733,23 @@ double ProjectedMatrices::checkCond( (*MPIdata::sout) << " CONDITION NUMBER OF THE OVERLAP MATRIX EXCEEDS TOL: " << rcond << "!!!" << std::endl; - Control& ct = *(Control::instance()); if (flag) mmpi.abort(); } return rcond; } -////// TEMPLATE THIS FOR FLOAT OPTION ?? template -int ProjectedMatrices::writeDM_hdf5(HDFrestart& h5f_file) +int ProjectedMatrices::writeDM(HDFrestart& h5f_file) { - hid_t file_id = h5f_file.file_id(); - - ReplicatedWorkSpace& wspace( - ReplicatedWorkSpace::instance()); - - wspace.initSquareMatrix(dm_->getMatrix()); - - if (file_id < 0) return 0; - - hsize_t dims[2] = { dim_, dim_ }; - - // filespace identifier - hid_t dataspace = H5Screate_simple(2, dims, nullptr); - - hid_t dset_id = H5Dcreate2(file_id, "/Density_Matrix", H5T_NATIVE_DOUBLE, - dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5: " - "H5Dcreate2 failed!!!" - << std::endl; - return -1; - } - - hid_t memspace = dataspace; - hid_t filespace = dataspace; - - DISTMATDTYPE* work_matrix = wspace.square_matrix(); - herr_t status = H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, - H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) << "Orbitals: H5Dwrite failed!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Dclose failed!!!" - << std::endl; - return -1; - } - status = H5Sclose(dataspace); - if (status < 0) - { - (*MPIdata::serr) << "ProjectedMatrices::write_dm_hdf5(), " - "H5Sclose failed!!!" - << std::endl; - return -1; - } - - return 0; + std::string name("/Density_Matrix"); + return dm_->write(h5f_file, name); } -////// TEMPLATE THIS FOR FLOAT OPTION ?? + template -int ProjectedMatrices::read_dm_hdf5(hid_t file_id) +int ProjectedMatrices::readDM(HDFrestart& h5f_file) { - ReplicatedWorkSpace& wspace( - ReplicatedWorkSpace::instance()); - DISTMATDTYPE* work_matrix = wspace.square_matrix(); - - int ierr = 0; - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.instancePE0()) - { - hid_t dset_id = H5Dopen2(file_id, "/Density_Matrix", H5P_DEFAULT); - if (dset_id < 0) - { - (*MPIdata::serr) - << "H5Dopen failed for /Density_Matrix!!!" << std::endl; - } - else - { - ierr = 1; - herr_t status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, work_matrix); - if (status < 0) - { - (*MPIdata::serr) - << "H5Dread failed for /Density_Matrix!!!" << std::endl; - return -1; - } - - status = H5Dclose(dset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - } - mmpi.bcast(&ierr, 1); - if (ierr >= 0) wspace.mpiBcastSquareMatrix(); - if (ierr >= 0) dm_->initMatrix(work_matrix); - - return ierr; + std::string name("/Density_Matrix"); + return dm_->read(h5f_file, name); } template diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 6725b827..9bc5e3c2 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -93,7 +93,14 @@ class ProjectedMatrices : public ProjectedMatricesInterface std::unique_ptr matHB_; std::unique_ptr matH_; + /*! + * Density Matrix + */ std::unique_ptr> dm_; + + /*! + * Gram matrix of orbitals overlaps + */ std::unique_ptr> gm_; // work matrix for tmp usage @@ -311,8 +318,8 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropy() override; double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; - int writeDM_hdf5(HDFrestart& h5f_file) override; - int read_dm_hdf5(hid_t file_id) override; + int writeDM(HDFrestart& h5f_file) override; + int readDM(HDFrestart& h5f_file) override; void printEigenvalues(std::ostream& os) const; void updateDM(const int iterative_index) override; void updateDMwithEigenstates(const int iterative_index); diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index 653b386d..e02e0b45 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -265,19 +265,19 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction exitWithErrorMessage("updateDMwithRelax"); } - virtual int read_dm_hdf5(hid_t file_id) + virtual int readDM(HDFrestart& h5f_file) { - (void)file_id; + (void)h5f_file; - exitWithErrorMessage("read_dm_hdf5"); + exitWithErrorMessage("readDM"); return 0; } - virtual int writeDM_hdf5(HDFrestart& h5f_file) + virtual int writeDM(HDFrestart& h5f_file) { (void)h5f_file; - exitWithErrorMessage("writeDM_hdf5"); + exitWithErrorMessage("writeDM"); return 0; } diff --git a/src/hdf_tools.cc b/src/hdf_tools.cc index 69a105dc..e56c0478 100644 --- a/src/hdf_tools.cc +++ b/src/hdf_tools.cc @@ -14,12 +14,16 @@ #include #include -using namespace std; +#define MGMOL_HDF5_FAIL(X) \ + { \ + std::cerr << "MGMOL_HDF5 failure:" << std::endl; \ + std::cerr << "Error Message: " << X << std::endl; \ + } namespace mgmol_tools { -void write1d( - hid_t file_id, const string& datasetname, vector& data, size_t length) +void write1d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t length) { assert(file_id >= 0); @@ -31,7 +35,7 @@ void write1d( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "write1d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write1d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -40,7 +44,7 @@ void write1d( dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write1d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write1d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -49,24 +53,24 @@ void write1d( dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write1d(), H5Dwrite failed!!!" << endl; + std::cerr << "write1d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write1d(), H5Dclose failed!!!" << endl; + std::cerr << "write1d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d( - hid_t file_id, const string& datasetname, vector& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -175,7 +179,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); @@ -184,28 +188,28 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; return; } } -void write2d(hid_t file_id, const string& datasetname, vector& data, - size_t* dims) +void write2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims) { assert(file_id >= 0); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); - // cout<<"Write "<& data, hid_t dataspace_id = H5Screate_simple(2, dimsm, nullptr); if (dataspace_id < 0) { - cerr << "write2d(), H5Screate_simple failed!!!" << endl; + std::cerr << "write2d(), H5Screate_simple failed!!!" << std::endl; return; } @@ -224,14 +228,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (dataset_id < 0) { - cerr << "write2d(), H5Dcreate2 failed!!!" << endl; + std::cerr << "write2d(), H5Dcreate2 failed!!!" << std::endl; return; } H5Sclose(dataspace_id); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -239,7 +244,7 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, tc.push_back(t); } - string attname("String_Length"); + std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; hid_t dataspaceA_id = H5Screate_simple(1, dimsA, nullptr); hid_t attribute_id = H5Acreate2(dataset_id, attname.c_str(), H5T_NATIVE_INT, @@ -248,15 +253,15 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, = H5Awrite(attribute_id, H5T_NATIVE_USHORT, &IonData_MaxStrLength); if (status < 0) { - cerr << "write2d(), Attribute: " << attname << " --- H5Awrite failed!!!" - << endl; + std::cerr << "write2d(), Attribute: " << attname + << " --- H5Awrite failed!!!" << std::endl; } status = H5Dwrite(dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &tc[0]); if (status < 0) { - cerr << "write2d(), H5Dwrite failed!!!" << endl; + std::cerr << "write2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -265,25 +270,25 @@ void write2d(hid_t file_id, const string& datasetname, vector& data, status = H5Sclose(dataspaceA_id); if (status < 0) { - cerr << "write2d(), H5Sclose failed!!!" << endl; + std::cerr << "write2d(), H5Sclose failed!!!" << std::endl; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "write2d(), H5Aclose failed!!!" << endl; + std::cerr << "write2d(), H5Aclose failed!!!" << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - cerr << "write2d(), H5Dclose failed!!!" << endl; + std::cerr << "write2d(), H5Dclose failed!!!" << std::endl; } } #ifdef MGMOL_USE_HDF5P -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -298,15 +303,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -317,8 +324,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -333,7 +340,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -345,7 +353,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_INT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -355,8 +363,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -371,15 +379,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -390,8 +400,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -406,7 +416,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -418,7 +429,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_USHORT, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -428,8 +439,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); @@ -444,15 +455,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -463,8 +476,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -479,7 +492,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -491,7 +505,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, &data[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -501,13 +515,13 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Sclose(memspace); } -void parallelWrite2d(hid_t file_id, const string& datasetname, - vector& data, size_t* dims, MPI_Comm comm) +void parallelWrite2d(hid_t file_id, const std::string& datasetname, + std::vector& data, size_t* dims, MPI_Comm comm) { assert(file_id >= 0); assert(!data.empty()); - // create type for strings of length IonData_MaxStrLength + // create type for std::strings of length IonData_MaxStrLength hid_t strtype = H5Tcopy(H5T_C_S1); H5Tset_size(strtype, IonData_MaxStrLength); @@ -521,15 +535,17 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, hid_t filespace = H5Screate_simple(2, dimsf, nullptr); if (filespace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for filespace!!!" + << std::endl; return; } hid_t memspace = H5Screate_simple(2, dimsm, nullptr); if (memspace < 0) { - cerr << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" - << endl; + std::cerr + << "parallelWrite2d(), H5Screate_simple failed for memspace!!!" + << std::endl; return; } @@ -540,8 +556,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5P_DEFAULT, plist_id, H5P_DEFAULT); if (dset_id < 0) { - cerr << "parallelWrite2d() for dataset " << datasetname - << ", H5Dcreate2() failed!!!" << endl; + std::cerr << "parallelWrite2d() for dataset " << datasetname + << ", H5Dcreate2() failed!!!" << std::endl; return; } H5Pclose(plist_id); @@ -556,7 +572,8 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, filespace, H5S_SELECT_SET, offset, stride, count, block); if (status < 0) { - cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Sselect_hyperslab() failed!!!" + << std::endl; return; } @@ -565,8 +582,9 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); // First copy the contents of the vector into a temporary container - vector tc; - for (vector::const_iterator i = data.begin(), end = data.end(); + std::vector tc; + for (std::vector::const_iterator i = data.begin(), + end = data.end(); i != end; ++i) { FixedLengthString t; @@ -576,7 +594,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { - cerr << "parallelWrite2d(), H5Dwrite failed!!!" << endl; + std::cerr << "parallelWrite2d(), H5Dwrite failed!!!" << std::endl; return; } @@ -589,7 +607,7 @@ void parallelWrite2d(hid_t file_id, const string& datasetname, #endif void addAttribute2Dataset( - hid_t dset_id, const char* attname, const vector& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -600,30 +618,30 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); - if (status < 0) cerr << "H5Sclose failed!!!" << endl; + if (status < 0) std::cerr << "H5Sclose failed!!!" << std::endl; //(*MPIdata::sout)<<"Write attribute "<& attr_data) + hid_t dset_id, const char* attname, const std::vector& attr_data) { assert(dset_id > -1); @@ -634,21 +652,21 @@ void addAttribute2Dataset( hid_t dataspace_id = H5Screate_simple(1, &dim, nullptr); if (dataspace_id < 0) { - cerr << "H5Screate failed!!!" << endl; + std::cerr << "H5Screate failed!!!" << std::endl; return; } hid_t attribute_id = H5Acreate2(dset_id, attname, H5T_NATIVE_INT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); if (attribute_id < 0) { - cerr << "H5Acreate failed!!!" << endl; + std::cerr << "H5Acreate failed!!!" << std::endl; return; } herr_t status = H5Sclose(dataspace_id); if (status < 0) { - cerr << "H5Sclose failed!!!" << endl; + std::cerr << "H5Sclose failed!!!" << std::endl; return; } @@ -656,14 +674,14 @@ void addAttribute2Dataset( status = H5Awrite(attribute_id, H5T_NATIVE_INT, &attr_data[0]); if (status < 0) { - cerr << "H5Awrite failed!!!" << endl; + std::cerr << "H5Awrite failed!!!" << std::endl; return; } status = H5Aclose(attribute_id); if (status < 0) { - cerr << "H5Aclose failed!!!" << endl; + std::cerr << "H5Aclose failed!!!" << std::endl; } } @@ -676,7 +694,7 @@ int whatisopen(hid_t fid) if (cnt <= 0) return cnt; - if (cnt > 1) cout << "HDF5 file: " << cnt << " object(s) open\n"; + if (cnt > 1) std::cout << "HDF5 file: " << cnt << " object(s) open\n"; // objs = malloc(cnt * sizeof(hid_t)); hid_t* objs = new hid_t[cnt]; @@ -699,4 +717,80 @@ int whatisopen(hid_t fid) return howmany; } + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim) +{ + if (file_id < 0) return 0; + + hsize_t dims[2] = { (hsize_t)dim, (hsize_t)dim }; + + // filespace identifier + hid_t dataspace = H5Screate_simple(2, dims, nullptr); + + hid_t dset_id = H5Dcreate2(file_id, name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dcreate2 failed!!!"); + return -1; + } + + hid_t memspace = dataspace; + hid_t filespace = dataspace; + + herr_t status = H5Dwrite( + dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dwrite failed!!!"); + return -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + return -1; + } + status = H5Sclose(dataspace); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Sclose failed!!!"); + return -1; + } + + return 0; +} + +int read_matrix(hid_t file_id, std::string& name, double* matrix) +{ + int ierr = 0; + hid_t dset_id = H5Dopen2(file_id, name.c_str(), H5P_DEFAULT); + if (dset_id < 0) + { + MGMOL_HDF5_FAIL("H5Dopen failed!!"); + ierr = -1; + } + else + { + herr_t status = H5Dread( + dset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dread failed!!"); + ierr = -1; + } + + status = H5Dclose(dset_id); + if (status < 0) + { + MGMOL_HDF5_FAIL("H5Dclose failed!!!"); + ierr = -1; + } + } + + return ierr; } + +} // namespace diff --git a/src/hdf_tools.h b/src/hdf_tools.h index e1a0ed1f..7a64bdce 100644 --- a/src/hdf_tools.h +++ b/src/hdf_tools.h @@ -6,8 +6,8 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE - #include "hdf5.h" + #include #include #include @@ -39,4 +39,8 @@ void addAttribute2Dataset( void addAttribute2Dataset( hid_t dset_id, const char* attname, const std::vector& attr_data); int whatisopen(hid_t fid); + +int write_matrix( + hid_t file_id, std::string& name, const double* matrix, const int dim); +int read_matrix(hid_t file_id, std::string& name, double* matrix); } diff --git a/src/md.cc b/src/md.cc index 7c6e2aa8..b773c27c 100644 --- a/src/md.cc +++ b/src/md.cc @@ -226,7 +226,7 @@ void checkMaxForces(const std::vector& fion, } template -int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, +int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, Rho& rho, const bool write_extrapolated_wf, const short count) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); @@ -245,9 +245,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); - OrbitalsType previous_orbitals("ForDumping", **orbitals, false); + OrbitalsType previous_orbitals("ForDumping", orbitals, false); if (!orbitals_extrapol_->getRestartData(previous_orbitals)) - previous_orbitals.assign(**orbitals); + previous_orbitals.assign(orbitals); + + // write all restart info in HDF5 file int ierr = write_hdf5(h5file, rho.rho_, ions, previous_orbitals, lrs_); mmpi.allreduce(&ierr, 1, MPI_MIN); @@ -259,12 +261,11 @@ int MGmol::dumpMDrestartFile(OrbitalsType** orbitals, Ions& ions, << std::endl; return ierr; } - // write_hdf5(h5file, rho.rho_, ions, *orbitals_minus1); - // stepper->write_hdf5(h5file); if (write_extrapolated_wf && ct.out_restart_info > 2) { - ierr = (*orbitals)->write_func_hdf5(h5file, "ExtrapolatedFunction"); + // write extra info needed for seamless MD restart + ierr = orbitals.write(h5file, "ExtrapolatedFunction"); mmpi.allreduce(&ierr, 1, MPI_MIN); if (ierr < 0) { @@ -354,16 +355,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ct.restart_info > 1) { - int flag_extrapolated_data = 0; - if (onpe0) - { + int flag_extrapolated_data + = h5f_file_->checkDataExists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0000"); - if (flag_extrapolated_data == 0) - flag_extrapolated_data - = h5f_file_->dset_exists("ExtrapolatedFunction0"); - } - MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + = h5f_file_->checkDataExists("ExtrapolatedFunction0"); if (ct.restart_info > 2) { @@ -377,7 +373,6 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) // need to reset a few things as we just read new orbitals (*orbitals)->computeGramAndInvS(); - dm_strategy_->update(*current_orbitals_); } DFTsolver::setItCountLarge(); @@ -615,7 +610,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) std::cout @@ -641,7 +636,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) { dump_tm_.start(); ierr = dumpMDrestartFile( - orbitals, ions, *rho_, extrapolated_flag, count); + **orbitals, ions, *rho_, extrapolated_flag, count); dump_tm_.stop(); if (onpe0 && ierr < 0 && count < (DUMP_MAX_NUM_TRY - 1)) diff --git a/src/restart.cc b/src/restart.cc index 944b1d8e..c12bb258 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -163,9 +163,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, // Write wavefunctions and old centers. if (ct.out_restart_info > 2) { - int ierr = orbitals.write_hdf5(h5f_file); + int ierr = orbitals.write(h5f_file); if (ierr < 0) return ierr; + if (!ct.fullyOccupied()) + { + int ierr = proj_matrices_->writeDM(h5f_file); + if (ierr < 0) return ierr; + } if (ct.isLocMode() && ct.WFExtrapolation() == WFExtrapolationType::Reversible) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 72378704..53793986 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -247,6 +247,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc + ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc + ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) @@ -552,7 +554,7 @@ target_include_directories(testConditionDistMatrix PRIVATE ${Boost_INCLUDE_DIRS} target_include_directories(testConditionDistMatrixPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) @@ -566,6 +568,7 @@ target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) +target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} From a227c04a43d32c932d9bb8e2b3a08695e9ab084f Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 2 Dec 2024 11:53:42 -0500 Subject: [PATCH 08/58] Write dm (#291) * Update use of DM in restart --- src/ProjectedMatrices.cc | 27 +++++++++++++++++++++++++++ src/ProjectedMatrices.h | 2 ++ src/ProjectedMatricesInterface.h | 16 ++++++++++++++++ src/md.cc | 18 ++++++++++++++++++ src/restart.cc | 2 +- 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index e50e98bc..74ff01ed 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -745,6 +745,23 @@ int ProjectedMatrices::writeDM(HDFrestart& h5f_file) return dm_->write(h5f_file, name); } +template +int ProjectedMatrices::writeSavedDM(HDFrestart& h5f_file) +{ + std::string name("/Density_Matrix_WF"); + + ReplicatedWorkSpace& wspace( + ReplicatedWorkSpace::instance()); + + const MatrixType* matrix = mat_X_old_.get(); + wspace.initSquareMatrix(*matrix); + + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + hid_t file_id = h5f_file.file_id(); + return mgmol_tools::write_matrix(file_id, name, work_matrix, dim_); +} + template int ProjectedMatrices::readDM(HDFrestart& h5f_file) { @@ -752,6 +769,16 @@ int ProjectedMatrices::readDM(HDFrestart& h5f_file) return dm_->read(h5f_file, name); } +template +int ProjectedMatrices::readWFDM(HDFrestart& h5f_file) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.barrier(); + if (mmpi.PE0()) std::cout << "ProjectedMatrices::readWFDM..." << std::endl; + std::string name("/Density_Matrix_WF"); + return dm_->read(h5f_file, name); +} + template void ProjectedMatrices::printEigenvalues(std::ostream& os) const { diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 9bc5e3c2..f2881763 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -319,7 +319,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; int writeDM(HDFrestart& h5f_file) override; + int writeSavedDM(HDFrestart& h5f_file); int readDM(HDFrestart& h5f_file) override; + int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; void updateDM(const int iterative_index) override; void updateDMwithEigenstates(const int iterative_index); diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index e02e0b45..c52595e1 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -273,6 +273,14 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } + virtual int readWFDM(HDFrestart& h5f_file) + { + (void)h5f_file; + + exitWithErrorMessage("readWFDM"); + + return 0; + } virtual int writeDM(HDFrestart& h5f_file) { (void)h5f_file; @@ -281,6 +289,14 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } + virtual int writeSavedDM(HDFrestart& h5f_file) + { + (void)h5f_file; + + exitWithErrorMessage("writeSavedDM"); + + return 0; + } virtual void updateDMwithChebApproximation(const int iterative_index) { (void)iterative_index; diff --git a/src/md.cc b/src/md.cc index b773c27c..b50618c8 100644 --- a/src/md.cc +++ b/src/md.cc @@ -275,6 +275,10 @@ int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, << std::endl; return ierr; } + + // write DM associated with non-extrapolated wavefunctions + // (last computed solution of KS equations) + proj_matrices_->writeSavedDM(h5file); } ierr = h5file.close(); @@ -579,6 +583,13 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) lrs_->clearOldCenters(); } + // save DM for possible restart write + // note: extrapolation is going to modify it! + if ((ct.out_restart_info > 2) + && (((md_iteration_ % ct.checkpoint) == 0) + || (mdstep == ct.num_MD_steps))) + proj_matrices_->saveDM(); + preWFextrapolation(); if (ct.dt > 0. @@ -656,6 +667,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) template void MGmol::loadRestartFile(const std::string filename) { + if (onpe0) std::cout << "loadRestartFile..." << std::endl; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); @@ -678,6 +690,12 @@ void MGmol::loadRestartFile(const std::string filename) global_exit(0); } + if (!ct.fullyOccupied()) + { + // overwrite DM with restart data in dataset Density_Matrix_WF + if (h5file.checkDataExists("Density_Matrix_WF")) + ierr = proj_matrices_->readWFDM(h5file); + } ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); diff --git a/src/restart.cc b/src/restart.cc index c12bb258..2fa76ecb 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -168,7 +168,7 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, if (!ct.fullyOccupied()) { - int ierr = proj_matrices_->writeDM(h5f_file); + ierr = proj_matrices_->writeDM(h5f_file); if (ierr < 0) return ierr; } if (ct.isLocMode() From b263532e4f585267d8eab837abb86076bb044195 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 23 Dec 2024 15:04:37 -0500 Subject: [PATCH 09/58] Remove unused function in Control (#294) --- src/Control.cc | 72 -------------------------------------------------- src/Control.h | 1 - 2 files changed, 73 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index dd786678..2272d3cb 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1119,78 +1119,6 @@ void Control::printThermostatInfo(std::ostream& os) const } } -int Control::readOccupations(std::ifstream* tfile) -{ - int count = 0; - float nel = 0.; - do - { - float t1 = 0.; - int nst = 0; - if (mype_ == 0) - { -#ifdef DEBUG - (*MPIdata::sout) << " Occupations of states..." << std::endl; -#endif - (*tfile) >> nst; - if (nst <= 0) - { - (*MPIdata::sout) - << "Control::readOccupations: numst=" << numst << std::endl; - (*MPIdata::sout) << "Control::readOccupations: nst=" << nst - << ", count=" << count << std::endl; - (*MPIdata::sout) << "Control::readOccupations: Bad repeat " - "count for state occupations" - << std::endl; - return -1; - } - if ((count + nst) > numst) - { - (*MPIdata::sout) << "Control::readOccupations: Occupations " - "specified for too many states" - << std::endl; - return -1; - } - - (*tfile) >> t1; - if (t1 < 0.) - { - (*MPIdata::sout) - << "Control::readOccupations: occupation=" << t1 - << std::endl; - (*MPIdata::sout) << "Control::readOccupations: occupation " - "should be a positive number" - << std::endl; - return -1; - } - finishRead(*tfile); - } - int mpirc = MPI_Bcast(&nst, 1, MPI_INT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation numbers failed!!!" << std::endl; - return -1; - } - mpirc = MPI_Bcast(&t1, 1, MPI_FLOAT, 0, comm_global_); - if (mpirc != MPI_SUCCESS) - { - (*MPIdata::sout) - << "MPI Bcast of occupation failed!!!" << std::endl; - return -1; - } - nel += nst * t1; - count += nst; - - } while (count < numst); - - nel_ = (int)nel; - - nempty_ = (2 * numst - (int)nel) / 2; - - return count; -} - void Control::setLocMode(const float radius, const float lx, const float ly, const float lz, const float mind_centers) { diff --git a/src/Control.h b/src/Control.h index 5134c718..2e781f79 100644 --- a/src/Control.h +++ b/src/Control.h @@ -294,7 +294,6 @@ class Control const float dtol, const short kim, const short itmax, const short lfil, const short maxfill, const short ilutype); void setSpreadRadius(); - int readOccupations(std::ifstream* tfile); bool checkTimeout() { return timeout_.check(); } bool occupationWidthIsZero() { return occ_width < 1.e-12; } From 3eb52d68f09e997f21b93de90ef7c5a1401e731c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 3 Jan 2025 09:07:27 -0500 Subject: [PATCH 10/58] Change symlink to restart in tests (#295) * enable tests when old link present --- tests/FIRE/test.py | 6 +++++- tests/LBFGS/test.py | 8 +++++++- tests/MD_D72/test.py | 6 +++++- tests/ShortSighted/test.py | 6 +++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/FIRE/test.py b/tests/FIRE/test.py index eaf380d9..6866e3ac 100755 --- a/tests/FIRE/test.py +++ b/tests/FIRE/test.py @@ -49,7 +49,11 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) print(command) diff --git a/tests/LBFGS/test.py b/tests/LBFGS/test.py index 2ff69c9a..324fd129 100755 --- a/tests/LBFGS/test.py +++ b/tests/LBFGS/test.py @@ -49,7 +49,12 @@ restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') + +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp2,coords,lrs) print(command) @@ -75,6 +80,7 @@ os.remove('wave.out') if force>tol: + print("Force larger than tol {}".format(tol)) sys.exit(1) sys.exit(0) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index a664b305..17b70de2 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 9b731f4c..27da7f7d 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -48,7 +48,11 @@ restart_file=str(restart_file[:-1],'utf-8') print(restart_file) -os.symlink(restart_file, 'wave.out') +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) From c1d46bc129b8b0ba532676892a08a93a7f51f5b2 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 7 Jan 2025 13:53:37 -0500 Subject: [PATCH 11/58] Extract number empty orbitals from restart file (#296) --- src/Control.h | 3 + src/HDFrestart.cc | 15 ++ src/HDFrestart.h | 3 + src/Hamiltonian.cc | 7 +- src/MVPSolver.cc | 4 +- src/setup.cc | 17 +- tests/CMakeLists.txt | 12 + tests/RestartEnergyAndForces/h2o.xyz | 6 + tests/RestartEnergyAndForces/mgmol.cfg | 33 +++ tests/RestartEnergyAndForces/restart.cfg | 32 +++ tests/RestartEnergyAndForces/test.py | 88 ++++++++ .../testRestartEnergyAndForces.cc | 206 ++++++++++++++++++ 12 files changed, 418 insertions(+), 8 deletions(-) create mode 100644 tests/RestartEnergyAndForces/h2o.xyz create mode 100644 tests/RestartEnergyAndForces/mgmol.cfg create mode 100644 tests/RestartEnergyAndForces/restart.cfg create mode 100755 tests/RestartEnergyAndForces/test.py create mode 100644 tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc diff --git a/src/Control.h b/src/Control.h index 2e781f79..b3401c1d 100644 --- a/src/Control.h +++ b/src/Control.h @@ -242,6 +242,7 @@ class Control const float total_spin, std::string run_directory = "."); void setDefaultValues(); + bool withSpin() { return with_spin_; } bool globalColoring() const { return (coloring_algo_ / 10 == 0); } @@ -263,6 +264,8 @@ class Control float getSpin() const { return total_spin_; } + void setNempty(const int nempty) { nempty_ = nempty; } + short getMGlevels() { return mg_levels_; } bool withPreconditioner() const { return (mg_levels_ >= 0); } diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 08018d47..ef658955 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -2725,6 +2725,21 @@ void HDFrestart::gatherDataXdir(std::vector& data) } } +int HDFrestart::countFunctionObjects(std::string& name) const +{ + int count = 0; + int found = 0; + do + { + std::string datasetname(getDatasetName(name, count)); + // check if dataset exists... + found = checkDataExists(datasetname); + if (found) count++; + } while (found); // dataset exists + + return count; +} + template int HDFrestart::read_1func_hdf5(float*, const std::string&); template int HDFrestart::read_1func_hdf5(double*, const std::string&); diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 7ffadff7..72c5c1f5 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -173,6 +173,7 @@ class HDFrestart return checkDataExistsLocal(datasetname.c_str()); } + // Returns zero (false), a positive (true) or a negative (failure) value. herr_t checkDataExistsLocal(const char* const datasetname) const { herr_t err_id = 0; @@ -282,6 +283,8 @@ class HDFrestart int getMDstepFromFile() const; int getFromFile(const std::string& attname) const; + int countFunctionObjects(std::string& name) const; + hid_t createPlist() { hid_t plist_id = H5P_DEFAULT; diff --git a/src/Hamiltonian.cc b/src/Hamiltonian.cc index ba0aa6c4..5a5d2a53 100644 --- a/src/Hamiltonian.cc +++ b/src/Hamiltonian.cc @@ -59,9 +59,9 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) { (*MPIdata::sout) << "Hamiltonian::applyLocal(), new_index =" - << new_index << endl; + << new_index << std::endl; (*MPIdata::sout) << "Hamiltonian::applyLocal(), itindex_ =" - << itindex_ << endl; + << itindex_ << std::endl; } #endif if (force || new_index != itindex_) @@ -76,7 +76,8 @@ const T& Hamiltonian::applyLocal(T& phi, const bool force) if (onpe0) (*MPIdata::sout) << "Hamiltonian::hlphi up to date, itindex_=" << itindex_ - << endl; + << ", Potential index=" << pot_->getIterativeIndex() + << std::endl; #endif } return *hlphi_; diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6115f933..7ab9f665 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -143,7 +143,6 @@ void MVPSolver::buildTarget_MVP( proj_mat_work_->setHB2H(); proj_mat_work_->updateDM(orbitals_index); - target = proj_mat_work_->dm(); if (ct.verbose > 2) @@ -286,7 +285,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) std::cout << "MVP --- Target energy..." << std::endl; proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) current_proj_mat->printOccupations(os_); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); const double nel = proj_mat_work_->getNel(); if (onpe0 && ct.verbose > 1) os_ << "MVP --- Number of electrons at beta=1 : " << nel @@ -346,7 +345,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) os_ << "Number of electrons for interpolated DM = " << pnel << std::endl; } - // if( onpe0 )os_<<"Rho..."<computeRho(orbitals, *work_); } diff --git a/src/setup.cc b/src/setup.cc index f022678c..9963bfc5 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -50,6 +50,7 @@ int MGmol::setupFromInput(const std::string filename) if (ct.isLocMode()) mymesh->subdivGridx(ct.getMGlevels()); const pb::PEenv& myPEenv = mymesh->peenv(); + if (ct.restart_info > 0) h5f_file_.reset( new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type)); @@ -58,8 +59,20 @@ int MGmol::setupFromInput(const std::string filename) if (status == -1) return -1; const short myspin = mmpi.myspin(); - const int nval = ions_->getNValenceElectrons(); - ct.setNumst(myspin, nval); + const int nel = ions_->getNValenceElectrons(); + // for the case of extended wavefunctions, we can determine the number + // of empty states from the number of wavefunctions in restart file + if (ct.restart_info > 2 && !ct.short_sighted) + { + std::string name = "Function"; + int count = h5f_file_->countFunctionObjects(name); + std::cout << "found " << count << " functions in restart file..." + << std::endl; + int nempty = ct.withSpin() ? count - nel : count - int(0.5 * nel); + ct.setNempty(nempty); + } + ct.setNumst(myspin, nel); + ct.setTolEnergy(); ct.setSpreadRadius(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53793986..b29c1aad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -256,6 +256,8 @@ add_executable(testWFEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/WFEnergyAndForces/testWFEnergyAndForces.cc) add_executable(testDMandEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) +add_executable(testRestartEnergyAndForces + ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -371,6 +373,15 @@ add_test(NAME testDMandEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/DMandEnergyAndForces/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRestartEnergyAndForces + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRestartEnergyAndForces + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -567,6 +578,7 @@ target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz new file mode 100644 index 00000000..cdc906f6 --- /dev/null +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -0,0 +1,6 @@ +3 +https://pubchem.ncbi.nlm.nih.gov/compound/Water +O 2.5369 -0.1550 0.0 +H 3.0739 0.1550 0.0 +H 2.0000 0.1550 0.0 + diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg new file mode 100644 index 00000000..e590f810 --- /dev/null +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -0,0 +1,33 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +nempty=2 +[Restart] +output_level=3 +output_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=1 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg new file mode 100644 index 00000000..99bc77d8 --- /dev/null +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -0,0 +1,32 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-3.4 +oy=-6.4 +oz=-6.4 +lx=12.8 +ly=12.8 +lz=12.8 +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +input_level=3 +input_filename=WF +[DensityMatrix] +solver=MVP +nb_inner_it=20 diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py new file mode 100755 index 00000000..349434e8 --- /dev/null +++ b/tests/RestartEnergyAndForces/test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test RestartEnergyAndForces...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-6] +test_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#analyse output +ref_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'achieved'): + ref_energy=energy + break + +#sys.exit(0) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +test_energy=1.e18 +for line in lines: + if line.count(b'%%'): + print(line) + words=line.split() + words=words[5].split(b',')[0] + energy = words.decode() + if line.count(b'Eks'): + print(line) + words=line.split() + print(words) + test_energy = words[2] + break + + +tol = 1.e-6 +diff=eval(test_energy)-eval(ref_energy) +print(diff) +if abs(diff)>tol: + print("Energies differ: {} vs {} !!!".format(ref_energy,test_energy)) + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc new file mode 100644 index 00000000..e323afbc --- /dev/null +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -0,0 +1,206 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "mgmol_run.h" + +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + HDFrestart h5file(ct.restart_file, myPEenv, ct.restart_file_type); + std::string name = "Function"; + int count = h5file.countFunctionObjects(name); + if (count != ct.numst) + { + std::cerr << "The number of functions in the restart file, " + << count << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, mymesh->subdivx(), + ct.numst, ct.bcWF, projmatrices.get(), nullptr, nullptr, nullptr, + nullptr); + + // read numst_ wavefunction + int nread = orbitals.read_func_hdf5(h5file, name); + if (nread != ct.numst) + { + std::cerr << "The number of functions read from the restart file, " + << nread << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + // set the iterative index to 1 to differentiate it from first instance + // in MGmol initial() function. This is not very clean and could be + // better designed, but works for now + orbitals.setIterativeIndex(1); + + // set initial DM with uniform occupations + projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->printDM(std::cout); + + // + // evaluate energy and forces with wavefunctions just read + // + std::vector forces; + double eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks : " << eks << std::endl; + std::cout << "Forces :" << std::endl; + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From eb539633b79a4db3386559408188469fb42163ca Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 16 Jan 2025 12:45:26 -0500 Subject: [PATCH 12/58] Clean up MD_IonicStepper restart data write (#297) --- src/Ion.h | 2 +- src/IonicStepper.cc | 83 ++++++++++++++------------- src/Ions.cc | 2 +- src/MD_IonicStepper.cc | 125 +++++------------------------------------ src/MD_IonicStepper.h | 13 +---- 5 files changed, 63 insertions(+), 162 deletions(-) diff --git a/src/Ion.h b/src/Ion.h index 336dee72..ae870820 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -176,7 +176,7 @@ class Ion } double position(const short i) const { return position_[i]; } - double old_position(const short i) const { return old_position_[i]; } + double getPreviousPosition(const short i) const { return old_position_[i]; } void setPosition(const double x, const double y, const double z) { old_position_[0] = position_[0]; diff --git a/src/IonicStepper.cc b/src/IonicStepper.cc index ca8d53cc..aa3a7f48 100644 --- a/src/IonicStepper.cc +++ b/src/IonicStepper.cc @@ -13,8 +13,6 @@ #include "MGmol_blas1.h" #include -using namespace std; - IonicStepper::IonicStepper(const double dt, const std::vector& atmove, std::vector& tau0, std::vector& taup) : atmove_(atmove), tau0_(tau0), taup_(taup) @@ -51,7 +49,8 @@ IonicStepper::IonicStepper(const double dt, const std::vector& atmove, } int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, - const vector& data, const string& name, const bool create) const + const std::vector& data, const std::string& name, + const bool create) const { hid_t file_id = h5f_file.file_id(); if (file_id < 0) return 0; @@ -62,7 +61,6 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (create) { - // Create the data space for new datasets hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; dataspace_id = H5Screate_simple(2, dims, nullptr); @@ -70,7 +68,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } @@ -81,7 +79,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dcreate2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -93,7 +91,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -103,7 +101,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeAtomicFields: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -111,14 +109,15 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeAtomicFields, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" << endl; + << "IonicStepper::writeAtomicFields, H5Dclose failed!!!" + << std::endl; return -1; } @@ -129,7 +128,7 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -137,7 +136,8 @@ int IonicStepper::writeAtomicFields(HDFrestart& h5f_file, return 0; } -int IonicStepper::writePositions(HDFrestart& h5f_file, const string& name) const +int IonicStepper::writePositions( + HDFrestart& h5f_file, const std::string& name) const { return writeAtomicFields(h5f_file, tau0_, name, false); } @@ -155,7 +155,8 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); if (dataspace_id < 0) { - (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" << endl; + (*MPIdata::serr) << "IonicStepper: H5Screate_simple failed!!!" + << std::endl; return -1; } @@ -170,7 +171,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { std::cerr << "IonicStepper::writeVelocities, H5Dopen2 " "failed for dataset " - << name << "!!!" << endl; + << name << "!!!" << std::endl; return -1; } } @@ -182,12 +183,12 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const { (*MPIdata::serr) << "IonicStepper:: H5Dcreate2 /Ionic_velocities failed!!!" - << endl; + << std::endl; return -1; } } - vector data(taup_); + std::vector data(taup_); double minus = -1.; int n = (int)tau0_.size(), ione = 1; DAXPY(&n, &minus, &tau0_[0], &ione, &data[0], &ione); @@ -202,20 +203,20 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const if (status < 0) { (*MPIdata::serr) << "IonicStepper::H5Dwrite velocities failed!!!" - << endl; + << std::endl; return -1; } else { if (onpe0) (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } H5Sclose(dataspace_id); @@ -224,7 +225,7 @@ int IonicStepper::writeVelocities(HDFrestart& h5f_file) const } int IonicStepper::readAtomicFields( - HDFrestart& h5f_file, vector& data, const string& name) + HDFrestart& h5f_file, std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -239,7 +240,7 @@ int IonicStepper::readAtomicFields( if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper, H5Dopen2 failed for " << name - << " !!!" << endl; + << " !!!" << std::endl; return -1; } @@ -249,14 +250,15 @@ int IonicStepper::readAtomicFields( H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper, H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper, H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } @@ -265,13 +267,14 @@ int IonicStepper::readAtomicFields( return 0; } -int IonicStepper::readPositions_hdf5(HDFrestart& h5f_file, const string& name) +int IonicStepper::readPositions_hdf5( + HDFrestart& h5f_file, const std::string& name) { return readAtomicFields(h5f_file, tau0_, name); } int IonicStepper::writeRandomStates(HDFrestart& h5f_file, - vector& data, const string& name) const + std::vector& data, const std::string& name) const { hid_t file_id = h5f_file.file_id(); bool create = false; @@ -282,7 +285,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) << "IonicStepper:: H5Dopen2 " << name - << " failed!!! Creating new data file " << endl; + << " failed!!! Creating new data file " << std::endl; // Create the data space for the dataset hsize_t dims[2] = { (hsize_t)data.size() / 3, 3 }; @@ -291,7 +294,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "Ions::writeRandomStates: H5Screate_simple failed!!!" - << endl; + << std::endl; return -1; } // Create the dataset @@ -300,7 +303,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (dataset_id < 0) { (*MPIdata::serr) - << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << endl; + << "Ions::writeRandomStates: H5Dcreate2 failed!!!" << std::endl; return -1; } create = true; @@ -312,7 +315,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (status < 0) { (*MPIdata::serr) << "IonicStepper::writeRandomStates: H5Dwrite " << name - << " failed!!!" << endl; + << " failed!!!" << std::endl; return -1; } else @@ -320,14 +323,15 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, if (onpe0) (*MPIdata::sout) << "IonicStepper::writeRandomStates, Data written into file " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; } status = H5Dclose(dataset_id); if (status < 0) { (*MPIdata::serr) - << "IonicStepper::writeRandomStates, H5Dclose failed!!!" << endl; + << "IonicStepper::writeRandomStates, H5Dclose failed!!!" + << std::endl; return -1; } @@ -338,7 +342,7 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, { (*MPIdata::serr) << "IonicStepper::writeAtomicFields, H5Sclose failed!!!" - << endl; + << std::endl; return -1; } } @@ -346,8 +350,8 @@ int IonicStepper::writeRandomStates(HDFrestart& h5f_file, return 0; } -int IonicStepper::readRandomStates( - HDFrestart& h5f_file, vector& data, const string& name) +int IonicStepper::readRandomStates(HDFrestart& h5f_file, + std::vector& data, const std::string& name) { hid_t file_id = h5f_file.file_id(); @@ -360,10 +364,10 @@ int IonicStepper::readRandomStates( if (onpe0) { (*MPIdata::sout) - << "H5Dopen failed for /Ionic_RandomStates" << endl; + << "H5Dopen failed for /Ionic_RandomStates" << std::endl; (*MPIdata::sout) << "Set random states to default computed in Ion.cc" - << endl; + << std::endl; } } else @@ -373,19 +377,20 @@ int IonicStepper::readRandomStates( if (onpe0) (*MPIdata::sout) << "Read Ionic random states from " - << h5f_file.filename() << endl; + << h5f_file.filename() << std::endl; herr_t status = H5Dread(dataset_id, H5T_NATIVE_USHORT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::serr) << "IonicStepper: H5Dread failed!!!" << endl; + (*MPIdata::serr) + << "IonicStepper: H5Dread failed!!!" << std::endl; return -1; } // close dataset status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; + (*MPIdata::serr) << "H5Dclose failed!!!" << std::endl; return -1; } } diff --git a/src/Ions.cc b/src/Ions.cc index 852151c7..8b956706 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -3209,7 +3209,7 @@ void Ions::initStepperData() for (short i = 0; i < 3; i++) { - taum_.push_back((*lion)->old_position(i)); + taum_.push_back((*lion)->getPreviousPosition(i)); tau0_.push_back((*lion)->position(i)); fion_.push_back((*lion)->force(i)); velocity_.push_back((*lion)->velocity(i)); diff --git a/src/MD_IonicStepper.cc b/src/MD_IonicStepper.cc index 7e797602..a1b4acb1 100644 --- a/src/MD_IonicStepper.cc +++ b/src/MD_IonicStepper.cc @@ -114,7 +114,8 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) (*MPIdata::sout) << "MD_IonicStepper::init() --- use positions " "from restart file with dt=" << dt_ << endl; - // taum_: velocities -> displacements = -dt*vel + // taum_ was initialized with velocities + // set taum_ to displacements = -dt*vel double alpha = -1. * dt_; DSCAL(&size_tau, &alpha, &taum_[0], &ione); @@ -123,117 +124,13 @@ int MD_IonicStepper::init(HDFrestart& /*h5f_file*/) taup_ = tau0_; DAXPY(&size_tau, &minus_one, &taum_[0], &ione, &taup_[0], &ione); - // taum_ -> previous positions: tau0_ - dt*vel + // Now set taum_ to previous positions: tau0_ - dt*vel DAXPY(&size_tau, &one, &tau0_[0], &ione, &taum_[0], &ione); } return 0; } -int MD_IonicStepper::writeForces(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)tau0_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_forces", H5T_NATIVE_DOUBLE, - dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_forces failed!!!" << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &fion_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite forces failed!!!" - << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) - << "Ionic forces written into " << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - -int MD_IonicStepper::writeTaum(HDFrestart& h5f_file) -{ - hid_t file_id = h5f_file.file_id(); - - // Create the data space for new datasets - hsize_t dims[2] = { (hsize_t)taum_.size() / 3, 3 }; - - hid_t dataspace_id = H5Screate_simple(2, dims, nullptr); - if (dataspace_id < 0) - { - (*MPIdata::serr) << "MD_IonicStepper: H5Screate_simple failed!!!" - << endl; - return -1; - } - - // Open dataset - hid_t dataset_id = H5Dcreate2(file_id, "/Ionic_velocities", - H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::serr) - << "MD_IonicStepper::H5Dcreate2 /Ionic_velocities failed!!!" - << endl; - return -1; - } - - // Write forces - herr_t status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &taum_[0]); - if (status < 0) - { - (*MPIdata::serr) << "MD_IonicStepper::H5Dwrite taum failed!!!" << endl; - return -1; - } - else - { - if (onpe0) - (*MPIdata::sout) << "Ionic velocities written into " - << h5f_file.filename() << endl; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::serr) << "H5Dclose failed!!!" << endl; - return -1; - } - H5Sclose(dataspace_id); - - return 0; -} - int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) { hid_t file_id = h5f_file.file_id(); @@ -251,11 +148,18 @@ int MD_IonicStepper::write_hdf5(HDFrestart& h5f_file) if (status < 0) return status; if (dt_ > 0.) + { writeVelocities(h5f_file); + std::string datasetname("/Ionic_previous_positions"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } else - writeTaum(h5f_file); - - writeForces(h5f_file); + { + std::string datasetname("/Ionic_velocities"); + writeAtomicFields(h5f_file, taum_, datasetname, true); + } + std::string datasetname("/Ionic_forces"); + writeAtomicFields(h5f_file, fion_, datasetname, true); } // @@ -469,7 +373,8 @@ void MD_IonicStepper::updateTau() tau0_ = taup_; if (dt_ > 0.) { - // update taup to be able to compute velocity... + // update taup_ to be able to compute velocity: + // taup_ <- tau0_-taum_ int size_tau = (int)tau0_.size(); int ione = 1; double alpha = 1.; diff --git a/src/MD_IonicStepper.h b/src/MD_IonicStepper.h index c62b57a5..24705a1a 100644 --- a/src/MD_IonicStepper.h +++ b/src/MD_IonicStepper.h @@ -7,15 +7,8 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -//////////////////////////////////////////////////////////////////////////////// -// -// MD_IonicStepper.h: -// -//////////////////////////////////////////////////////////////////////////////// -// $Id$ - -#ifndef MD_IONICSTEPPER_H -#define MD_IONICSTEPPER_H +#ifndef MGMOL_MD_IONICSTEPPER_H +#define MGMOL_MD_IONICSTEPPER_H #include "IonicStepper.h" #include @@ -72,8 +65,6 @@ class MD_IonicStepper : public IonicStepper void updateTau(); double etol(void) const override; int write_hdf5(HDFrestart&) override; - int writeForces(HDFrestart& h5f_file); - int writeTaum(HDFrestart& h5f_file); int init(HDFrestart&) override; void printVelocities(std::ostream& os) const; From 857673f8219c16724e39119685f5bfd9df86e9c4 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 16 Jan 2025 21:53:54 -0500 Subject: [PATCH 13/58] Clean up code based on compiler warnings (#299) --- scripts/build_ubuntu22_openmpi.sh | 2 +- src/Control.cc | 17 ----------------- src/DistributedIonicData.cc | 13 ++++++------- src/HDFrestart.cc | 2 +- src/HamiltonianMVP_DMStrategy.cc | 2 +- src/IonData.cc | 17 ++++++++--------- src/Ions.cc | 6 ------ src/MGmol.cc | 2 ++ src/MVP_DMStrategy.h | 2 +- src/Potentials.cc | 3 +-- src/Potentials.h | 2 +- src/hdf_tools.cc | 30 ++++++++++++++---------------- src/mgmol_run.cc | 1 - tests/testSetGhostValues.cc | 4 ---- 14 files changed, 36 insertions(+), 67 deletions(-) diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..474d61e5 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra -Wno-cast-function-type" \ .. # call make install diff --git a/src/Control.cc b/src/Control.cc index 2272d3cb..2c501aa9 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -31,23 +31,6 @@ float Control::total_spin_ = 0.; std::string Control::run_directory_("."); bool Control::with_spin_ = false; -static void finishRead(std::ifstream& tfile) -{ - // while( tfile.get()!='\n'); - // string str; - // getline(tfile,str); - char str[256]; - tfile.getline(str, 256); - - char cc = (char)tfile.peek(); - while (cc == ('#') || (cc == '\n') || cc == ' ') - { - while (tfile.get() != '\n') - ; - cc = (char)tfile.peek(); // look at next character - } -} - Control::Control() { assert(comm_global_ != MPI_COMM_NULL); diff --git a/src/DistributedIonicData.cc b/src/DistributedIonicData.cc index 6652c41a..05399cac 100644 --- a/src/DistributedIonicData.cc +++ b/src/DistributedIonicData.cc @@ -12,7 +12,6 @@ #include "tools.h" #include -using namespace std; DistributedIonicData::DistributedIonicData( const std::vector& local_names, @@ -31,16 +30,16 @@ DistributedIonicData::DistributedIonicData( int DistributedIonicData::pack(char* cbuff, double* dbuff) { - vector::iterator itf = data_.begin(); - double* dptr = dbuff; + std::vector::iterator itf = data_.begin(); + double* dptr = dbuff; int idx = 0; - for (vector::iterator it = ion_names_.begin(); + for (std::vector::iterator it = ion_names_.begin(); it != ion_names_.end(); ++it) { - string s(*it); + std::string s(*it); FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; @@ -58,7 +57,7 @@ void DistributedIonicData::unpack(char*& cptr, double*& dptr, const short ndata) for (short i = 0; i < ndata; i++) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_names_.push_back(name); cptr += IonData_MaxStrLength; diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index ef658955..ac856ff2 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -428,7 +428,7 @@ void HDFrestart::addReleaseNumber2File(const char* release) // if( onpe0 ) { HDF_FixedLengthString t; - strncpy(t.mystring, release, MyHDFStrLength); + strncpy(t.mystring, release, MyHDFStrLength - 1); herr_t status = H5Awrite(attribute_id, strtype, &t); if (status < 0) { diff --git a/src/HamiltonianMVP_DMStrategy.cc b/src/HamiltonianMVP_DMStrategy.cc index 2b5f8913..be62b894 100644 --- a/src/HamiltonianMVP_DMStrategy.cc +++ b/src/HamiltonianMVP_DMStrategy.cc @@ -57,7 +57,7 @@ HamiltonianMVP_DMStrategy void HamiltonianMVP_DMStrategy::initialize(OrbitalsType& orbitals) + OrbitalsType>::initialize(OrbitalsType&) { } diff --git a/src/IonData.cc b/src/IonData.cc index 8f62ddd8..89c1bd0c 100644 --- a/src/IonData.cc +++ b/src/IonData.cc @@ -10,12 +10,11 @@ #include "IonData.h" #include "tools.h" #include -using namespace std; void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) { // get name - string name(cptr, IonData_MaxStrLength); + std::string name(cptr, IonData_MaxStrLength); stripLeadingAndTrailingBlanks(name); ion_name = name; cptr += IonData_MaxStrLength; @@ -52,16 +51,16 @@ void IonData::unpack(char*& cptr, int*& iptr, double*& dptr) // pack Ions data for communication void IonData::packIonData( - char* cbuff, int* ibuff, double* dbuff, vector& data) + char* cbuff, int* ibuff, double* dbuff, std::vector& data) { // pack ion_names buffer int idx = 0; - for (vector::iterator idata = data.begin(); idata != data.end(); - ++idata) + for (std::vector::iterator idata = data.begin(); + idata != data.end(); ++idata) { - string s = (*idata).ion_name; + std::string s = (*idata).ion_name; FixedLengthString t; - strncpy(t.mystring, s.c_str(), IonData_MaxStrLength); + strncpy(t.mystring, s.c_str(), IonData_MaxStrLength - 1); memcpy(&cbuff[idx], t.mystring, IonData_MaxStrLength); idx += IonData_MaxStrLength; } @@ -69,8 +68,8 @@ void IonData::packIonData( // pack integer datatypes int* iptr = &ibuff[0]; // first pack local data size - *(iptr++) = data.size(); - vector::iterator idata = data.begin(); + *(iptr++) = data.size(); + std::vector::iterator idata = data.begin(); while (idata != data.end()) { // pack atomic_num diff --git a/src/Ions.cc b/src/Ions.cc index 8b956706..11c42f64 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1608,7 +1608,6 @@ int Ions::readAtomsFromXYZ( const std::string& filename, const bool cell_relative) { MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); // set up list boundaries // get radius of projectors @@ -1698,9 +1697,6 @@ int Ions::readAtomsFromXYZ( int Ions::setAtoms( const std::vector& crds, const std::vector& spec) { - MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); - const int natoms = crds.size() / 3; double velocity[3] = { 0., 0., 0. }; @@ -1748,8 +1744,6 @@ int Ions::setAtoms( addIonToList(species_[isp], aname, &crds[3 * ia], velocity, locked); } - // std::cout<& data, std::vector& tc) +{ + tc.clear(); + for (auto& d : data) + { + FixedLengthString t; + strncpy(t.mystring, d.c_str(), IonData_MaxStrLength - 1); + tc.push_back(t); + } +} + void write1d(hid_t file_id, const std::string& datasetname, std::vector& data, size_t length) { @@ -235,14 +247,7 @@ void write2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); std::string attname("String_Length"); hsize_t dimsA[1] = { 1 }; @@ -583,14 +588,7 @@ void parallelWrite2d(hid_t file_id, const std::string& datasetname, // First copy the contents of the vector into a temporary container std::vector tc; - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) - { - FixedLengthString t; - strncpy(t.mystring, i->c_str(), IonData_MaxStrLength); - tc.push_back(t); - } + string2fixedlength(data, tc); status = H5Dwrite(dset_id, strtype, memspace, filespace, plist_id, &tc[0]); if (status < 0) { diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index e746611c..a5de1ad5 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -91,7 +91,6 @@ int mgmol_check() Control& ct = *(Control::instance()); Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (myPEenv.color() > 0) { diff --git a/tests/testSetGhostValues.cc b/tests/testSetGhostValues.cc index f8520055..c3680087 100644 --- a/tests/testSetGhostValues.cc +++ b/tests/testSetGhostValues.cc @@ -18,8 +18,6 @@ TEST_CASE("Set ghost values", "[set ghosts") const double ll = 1.; const double lattice[3] = { ll, ll, ll }; - const int nfunc = 10; - MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -101,8 +99,6 @@ TEST_CASE("Set ghost values", "[set ghosts") gf.assign(inner_data.data(), 'd'); gf.set_updated_boundaries(false); - double norm_before = gf.norm2(); - // fill ghost values const bool direction[3] = { true, true, true }; gf.defaultTrade_boundaries(); From f294d3c262801ff173e198113533413b5fa0e225 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 23 Jan 2025 05:19:12 -0800 Subject: [PATCH 14/58] Add getForces for evaluateDMandEnergyAndForces (#300) From 98a79dee2df4c5d653e088fde18e4ce691262b2b Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 24 Jan 2025 12:52:46 -0500 Subject: [PATCH 15/58] Add check for compatibility MVP/Mehrstellen (#301) --- src/Control.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Control.cc b/src/Control.cc index 2c501aa9..d266aa2e 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1818,6 +1818,14 @@ int Control::checkOptions() return -1; } + if (DM_solver_ > 0 && lap_type == 0) + { + std::cerr << "DM_solver_ = " << DM_solver_ << std::endl; + std::cerr << "ERROR: Mehrstellen not compatible with MVP inner solvers!" + << std::endl; + return -1; + } + if (it_algo_type_ == 3 && lap_type == 0) { std::cerr From 8eda83e52d9132892337daf92383522bfee3726c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Feb 2025 11:14:17 -0500 Subject: [PATCH 16/58] Add support for Br atom (#302) --- potentials/pseudo.Br_ONCV_PBE_SG15 | 2529 ++++++++++++++++++++++++++++ src/Ions.cc | 14 +- 2 files changed, 2536 insertions(+), 7 deletions(-) create mode 100644 potentials/pseudo.Br_ONCV_PBE_SG15 diff --git a/potentials/pseudo.Br_ONCV_PBE_SG15 b/potentials/pseudo.Br_ONCV_PBE_SG15 new file mode 100644 index 00000000..c135617b --- /dev/null +++ b/potentials/pseudo.Br_ONCV_PBE_SG15 @@ -0,0 +1,2529 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# Br 35.00 6 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 6.00 +# 3 0 2.00 +# 3 1 6.00 +# 3 2 10.00 +# 4 0 2.00 +# 4 1 5.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 2 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 2.49793 -0.73993 5 8 5.10687 +# 1 2.49793 -0.28884 5 8 4.46675 +# 2 2.49793 0.09432 5 8 4.91674 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 2.24365 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.26029 +# 1 2 1.41324 +# 2 2 1.05278 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +Br_ONCV_PBE +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +35 +# Atomic mass +79.90399933 +# Number of valence electrons +7 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +4 +# l-value for state which is local, then type of potential format +3 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 -0.3626325499E+01 0.1750463805E+01 +1 2 0.4803824052E+02 0.7506916045E+00 +2 2 0.1514320363E+02 0.8114306630E-01 +# l= 0 +0.0 -3.0708154786 -2.4047930787 +0.01 -3.070772567 -2.4042133705 +0.02 -3.0706536405 -2.4024662354 +0.03 -3.0704488283 -2.3995597251 +0.04 -3.0701482591 -2.3955018914 +0.05 -3.0697381761 -2.3903039657 +0.06 -3.0692009986 -2.3839803178 +0.07 -3.0685154018 -2.376548404 +0.08 -3.0676564136 -2.3680287033 +0.09 -3.0665955275 -2.3584446428 +0.1 -3.0653008322 -2.3478225123 +0.11 -3.063737157 -2.3361913688 +0.12 -3.0618662322 -2.3235829307 +0.13 -3.0596468635 -2.3100314626 +0.14 -3.0570351213 -2.2955736506 +0.15 -3.053984542 -2.2802484691 +0.16 -3.0504463422 -2.2640970395 +0.17 -3.0463696444 -2.2471624809 +0.18 -3.0417017127 -2.2294897538 +0.19 -3.0363881987 -2.2111254979 +0.2 -3.030373396 -2.1921178621 +0.21 -3.0236005018 -2.1725163322 +0.22 -3.0160118859 -2.1523715511 +0.23 -3.0075493645 -2.1317351366 +0.24 -2.9981544792 -2.1106594957 +0.25 -2.987768779 -2.0891976363 +0.26 -2.976334104 -2.0674029769 +0.27 -2.9637928718 -2.0453291551 +0.28 -2.9500883622 -2.0230298363 +0.29 -2.9351650013 -2.0005585216 +0.3 -2.9189686442 -1.9779683581 +0.31 -2.9014468534 -1.9553119497 +0.32 -2.8825491704 -1.9326411716 +0.33 -2.8622273897 -1.910006986 +0.34 -2.840435816 -1.8874592637 +0.35 -2.8171315174 -1.8650466102 +0.36 -2.7922745751 -1.8428161943 +0.37 -2.7658283185 -1.820813584 +0.38 -2.7377595404 -1.7990825945 +0.39 -2.7080387236 -1.7776651304 +0.4 -2.6766402368 -1.7566010457 +0.41 -2.643542518 -1.7359280108 +0.42 -2.6087282508 -1.7156813851 +0.43 -2.5721845223 -1.6958940993 +0.44 -2.5339029669 -1.6765965483 +0.45 -2.4938798925 -1.6578164926 +0.46 -2.4521163913 -1.639578971 +0.47 -2.4086184283 -1.6219062255 +0.48 -2.3633969093 -1.6048176402 +0.49 -2.3164677785 -1.588329662 +0.5 -2.2678520179 -1.5724557793 +0.51 -2.2175756537 -1.5572064995 +0.52 -2.165669807 -1.5425892978 +0.53 -2.1121706776 -1.5286086079 +0.54 -2.0571194038 -1.5152658873 +0.55 -2.0005622031 -1.5025595131 +0.56 -1.9425500828 -1.4904849389 +0.57 -1.8831389418 -1.4790346138 +0.58 -1.8223892906 -1.4681981355 +0.59 -1.760366311 -1.4579621933 +0.6 -1.6971394766 -1.4483107855 +0.61 -1.6327826828 -1.439225116 +0.62 -1.5673737973 -1.4306838573 +0.63 -1.5009946281 -1.4226631485 +0.64 -1.4337307139 -1.415136705 +0.65 -1.365670995 -1.4080760077 +0.66 -1.2969076816 -1.4014503615 +0.67 -1.227536009 -1.3952270271 +0.68 -1.1576539096 -1.3893714093 +0.69 -1.087361756 -1.3838471989 +0.7 -1.0167621481 -1.3786164792 +0.71 -0.94595957813 -1.3736399185 +0.72 -0.87506012463 -1.3688769435 +0.73 -0.80417116303 -1.3642858995 +0.74 -0.7334010578 -1.3598242236 +0.75 -0.66285885089 -1.355448623 +0.76 -0.59265395182 -1.351115238 +0.77 -0.52289580245 -1.3467798445 +0.78 -0.45369356504 -1.3423980267 +0.79 -0.38515580233 -1.337925358 +0.8 -0.31739015855 -1.3333175826 +0.81 -0.25050304259 -1.3285307948 +0.82 -0.18459931451 -1.3235216181 +0.83 -0.11978197659 -1.3182473801 +0.84 -0.056151869857 -1.3126662853 +0.85 0.0061926226578 -1.3067375838 +0.86 0.067155864931 -1.3004217357 +0.87 0.12664524827 -1.2936805697 +0.88 0.18457146187 -1.2864774371 +0.89 0.2408487527 -1.2787773586 +0.9 0.29539517376 -1.2705471649 +0.91 0.34813281968 -1.2617556292 +0.92 0.39898804895 -1.2523735929 +0.93 0.44789169169 -1.2423740812 +0.94 0.49477924217 -1.2317324118 +0.95 0.53959097092 -1.2204262598 +0.96 0.58227215742 -1.2084357833 +0.97 0.6227733871 -1.1957437779 +0.98 0.66105043859 -1.1823356271 +0.99 0.6970644809 -1.1681994044 +1.0 0.73078216308 -1.1533259198 +1.01 0.76217568426 -1.1377087564 +1.02 0.79122266757 -1.1213442058 +1.03 0.81790653051 -1.1042314518 +1.04 0.84221661359 -1.0863726296 +1.05 0.86414774871 -1.0677726042 +1.06 0.88370040536 -1.048439037 +1.07 0.90088046461 -1.0283822657 +1.08 0.91569937292 -1.0076153718 +1.09 0.92817465654 -0.98615441727 +1.1 0.93832889415 -0.96401792656 +1.11 0.9461899162 -0.94122697008 +1.12 0.9517900713 -0.91780480034 +1.13 0.95516808541 -0.8937777179 +1.14 0.95636695802 -0.86917403945 +1.15 0.95543424502 -0.84402421656 +1.16 0.95242155537 -0.81836058881 +1.17 0.94738611915 -0.79221806224 +1.18 0.94038861073 -0.76563307802 +1.19 0.93149314032 -0.73864360126 +1.2 0.9207680708 -0.71128944543 +1.21 0.90828544129 -0.68361195316 +1.22 0.89411988748 -0.65565351703 +1.23 0.87834887992 -0.62745766744 +1.24 0.86105341064 -0.5990692311 +1.25 0.84231617223 -0.57053359643 +1.26 0.82222172706 -0.54189679743 +1.27 0.80085735674 -0.51320561099 +1.28 0.77831122726 -0.48450694828 +1.29 0.75467272985 -0.45584799729 +1.3 0.73003271242 -0.42727593895 +1.31 0.70448226061 -0.39883777816 +1.32 0.6781130858 -0.37058037334 +1.33 0.65101693162 -0.34254982352 +1.34 0.62328542089 -0.31479192232 +1.35 0.59500981379 -0.28735157546 +1.36 0.56628028474 -0.26027246962 +1.37 0.53718687691 -0.23359805009 +1.38 0.50781737223 -0.20736952858 +1.39 0.47825865123 -0.18162739142 +1.4 0.44859623165 -0.15641087651 +1.41 0.41891237101 -0.13175663087 +1.42 0.38928923477 -0.10770120711 +1.43 0.35980439328 -0.084277653647 +1.44 0.33053421696 -0.061518184098 +1.45 0.30155239027 -0.03945302065 +1.46 0.27292797429 -0.0181091649 +1.47 0.24473036233 0.002486186983 +1.48 0.2170211898 0.022311130325 +1.49 0.18986298266 0.041343142397 +1.5 0.16331164708 0.059564125665 +1.51 0.13742115 0.076957214701 +1.52 0.11224181642 0.093507941835 +1.53 0.087818451049 0.10920533268 +1.54 0.064195549675 0.12403864261 +1.55 0.041408563173 0.13800275831 +1.56 0.019495861764 0.15109077964 +1.57 -0.0015158417959 0.16330294016 +1.58 -0.021596235849 0.17463728157 +1.59 -0.040725475422 0.18509813619 +1.6 -0.058881960784 0.19468876976 +1.61 -0.076052534895 0.20341742456 +1.62 -0.092224070602 0.21129232349 +1.63 -0.10739003129 0.21832549573 +1.64 -0.12154529981 0.2245297272 +1.65 -0.13468985191 0.22992068159 +1.66 -0.14682597759 0.23451524024 +1.67 -0.15795999363 0.23833250484 +1.68 -0.16810089293 0.24139293702 +1.69 -0.17726108092 0.24371882479 +1.7 -0.18545548467 0.24533367189 +1.71 -0.19270225049 0.24626263687 +1.72 -0.19902144307 0.24653172825 +1.73 -0.20443649328 0.2461685876 +1.74 -0.20897181658 0.24520121549 +1.75 -0.21265555109 0.24365927067 +1.76 -0.21551572366 0.24157228058 +1.77 -0.21758447021 0.23897137241 +1.78 -0.21889277302 0.23588720289 +1.79 -0.21947591783 0.23235173626 +1.8 -0.21936729817 0.22839644157 +1.81 -0.21860435564 0.224053418 +1.82 -0.2172225127 0.21935470487 +1.83 -0.21526018396 0.21433189433 +1.84 -0.21275469141 0.20901710885 +1.85 -0.2097443342 0.2034411581 +1.86 -0.20626747958 0.19763549074 +1.87 -0.20236211696 0.19163011691 +1.88 -0.19806642786 0.18545493817 +1.89 -0.19341811175 0.17913927614 +1.9 -0.18845384334 0.17271059028 +1.91 -0.18321119098 0.16619768388 +1.92 -0.17772431694 0.15962506074 +1.93 -0.17202997839 0.15302021981 +1.94 -0.1661601512 0.14640550171 +1.95 -0.16014881031 0.13980526812 +1.96 -0.15402698692 0.13324102441 +1.97 -0.14782425499 0.12673280543 +1.98 -0.14157131194 0.12030174533 +1.99 -0.13529270013 0.11396333406 +2.0 -0.12901767217 0.10773726718 +2.01 -0.12276761168 0.10163636831 +2.02 -0.11656673377 0.095675877908 +2.03 -0.1104363775 0.089868634417 +2.04 -0.10439399399 0.084224202689 +2.05 -0.098461395649 0.0787557483 +2.06 -0.09265062254 0.07346847772 +2.07 -0.086979074138 0.068371912909 +2.08 -0.081458820035 0.06347135213 +2.09 -0.076100125576 0.058770653954 +2.1 -0.070916200757 0.054275981844 +2.11 -0.065911720591 0.049986904382 +2.12 -0.061096352253 0.04590680729 +2.13 -0.05647481096 0.04203534513 +2.14 -0.052050288108 0.038371018518 +2.15 -0.047828987027 0.034914562031 +2.16 -0.043809718782 0.031661293642 +2.17 -0.039995100139 0.028609286026 +2.18 -0.036385130803 0.025754761958 +2.19 -0.032977565996 0.023092357681 +2.2 -0.029772938118 0.020618609458 +2.21 -0.02676722353 0.018327015582 +2.22 -0.023957902849 0.016212045303 +2.23 -0.021342381035 0.014268136652 +2.24 -0.018915988096 0.01248852304 +2.25 -0.016675937928 0.010867451055 +2.26 -0.014617052811 0.0093979619195 +2.27 -0.012733401274 0.0080727169653 +2.28 -0.011021551686 0.0068855480404 +2.29 -0.0094727420429 0.0058278549281 +2.3 -0.0080800306285 0.004891808646 +2.31 -0.0068367888466 0.0040696905805 +2.32 -0.0057335326398 0.0033529863328 +2.33 -0.0047622051973 0.002733387986 +2.34 -0.0039142269282 0.0022027527767 +2.35 -0.003180287976 0.001753168643 +2.36 -0.0025514070016 0.0013760738931 +2.37 -0.002018701345 0.0010641189905 +2.38 -0.0015733073204 0.00081018747083 +2.39 -0.0012057416961 0.000605822272 +2.4 -0.00090807593031 0.00044534423372 +2.41 -0.00067211676854 0.00032260120986 +2.42 -0.00048836884014 0.00022964110509 +2.43 -0.00035064527094 0.00016295096584 +2.44 -0.00025146290618 0.00011730333122 +2.45 -0.00018232534191 8.6257107481e-05 +2.46 -0.00013859768358 6.798715492e-05 +2.47 -0.00011459867708 5.9010018177e-05 +2.48 -0.00010538902877 5.5962206051e-05 +2.49 -0.0001014873045 5.4745829591e-05 +2.5 -9.0886540089e-05 4.9286426798e-05 +2.51 -5.7587161462e-05 3.1411431787e-05 +2.52 -2.3406234748e-05 1.2820803441e-05 +2.53 2.0554012075e-06 -1.1129389439e-06 +2.54 6.3698356898e-06 -3.4490775717e-06 +2.55 4.1835875739e-06 -2.265288898e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 0.16768810909 -0.09197895777 +0.02 0.33490177013 -0.18377178817 +0.03 0.50116814078 -0.27519297668 +0.04 0.66601758415 -0.36605823207 +0.05 0.82898525636 -0.45618509197 +0.06 0.98961267609 -0.54539352126 +0.07 1.1474492702 -0.63350650089 +0.08 1.3020538895 -0.7203506047 +0.09 1.4529962891 -0.80575656223 +0.1 1.5998585678 -0.88955980496 +0.11 1.7422365611 -0.97160099425 +0.12 1.8797411823 -1.0517265285 +0.13 2.0119997077 -1.129789028 +0.14 2.138657 -1.2056477949 +0.15 2.2593766658 -1.2791692479 +0.16 2.3738421434 -1.3502273279 +0.17 2.4817577163 -1.4187038753 +0.18 2.5828494491 -1.4844889757 +0.19 2.6768660426 -1.5474812743 +0.2 2.7635796037 -1.6075882562 +0.21 2.8427863308 -1.6647264933 +0.22 2.9143071076 -1.7188218556 +0.23 2.9779880072 -1.7698096863 +0.24 3.0337007024 -1.8176349412 +0.25 3.0813427856 -1.862252292 +0.26 3.1208379864 -1.9036261898 +0.27 3.1521363034 -1.9417308947 +0.28 3.1752140319 -1.9765504665 +0.29 3.1900737018 -2.0080787184 +0.3 3.1967439186 -2.0363191351 +0.31 3.1952791115 -2.0612847542 +0.32 3.1857591886 -2.0829980127 +0.33 3.1682891025 -2.1014905583 +0.34 3.1429983262 -2.1168030277 +0.35 3.110040245 -2.128984793 +0.36 3.0695914617 -2.1380936747 +0.37 3.0218510224 -2.1441956261 +0.38 2.9670395749 -2.1473643925 +0.39 2.9053984349 -2.1476811361 +0.4 2.8371885957 -2.1452340407 +0.41 2.7626896752 -2.1401178945 +0.42 2.6821987951 -2.1324336474 +0.43 2.596029406 -2.1222879509 +0.44 2.5045100614 -2.1097926797 +0.45 2.4079831454 -2.0950644377 +0.46 2.3068035597 -2.0782240514 +0.47 2.2013373814 -2.0593960544 +0.48 2.0919605029 -2.0387081682 +0.49 1.979057148 -2.0162907368 +0.5 1.8630185217 -1.9922762184 +0.51 1.7442414153 -1.9667986605 +0.52 1.6231266566 -1.9399931205 +0.53 1.500077649 -1.9119951227 +0.54 1.3754991997 -1.88294023 +0.55 1.2497955863 -1.852963328 +0.56 1.123369708 -1.8221983257 +0.57 0.99662116191 -1.7907774522 +0.58 0.86994536282 -1.7588309508 +0.59 0.74373162879 -1.7264863891 +0.6 0.61836264621 -1.6938684873 +0.61 0.49421220571 -1.6610983091 +0.62 0.37164494723 -1.628293199 +0.63 0.25101461062 -1.5955661733 +0.64 0.1326628874 -1.5630255372 +0.65 0.016918843032 -1.5307747113 +0.66 -0.095902538203 -1.4989117459 +0.67 -0.20550134835 -1.4675290072 +0.68 -0.31159326886 -1.4367130384 +0.69 -0.4139103595 -1.4065443242 +0.7 -0.51220225666 -1.377096921 +0.71 -0.60623655903 -1.3484383751 +0.72 -0.6957993 -1.3206296129 +0.73 -0.78069554004 -1.2937247949 +0.74 -0.86074977919 -1.267771236 +0.75 -0.93580620366 -1.2428093839 +0.76 -1.0057293808 -1.218872661 +0.77 -1.0704039576 -1.1959876322 +0.78 -1.1297349296 -1.1741739926 +0.79 -1.1836476649 -1.1534446391 +0.8 -1.23208785 -1.1338057713 +0.81 -1.2750213562 -1.1152570226 +0.82 -1.3124340302 -1.0977916176 +0.83 -1.34433141 -1.0813965584 +0.84 -1.3707383679 -1.0660528348 +0.85 -1.3916986843 -1.0517356601 +0.86 -1.4072745528 -1.0384147296 +0.87 -1.4175460222 -1.0260545009 +0.88 -1.422610376 -1.0146144944 +0.89 -1.4225814539 -1.0040496119 +0.9 -1.4175889193 -0.99431047237 +0.91 -1.4077774754 -0.98534376271 +0.92 -1.393306035 -0.97709260167 +0.93 -1.3743468472 -0.96949691479 +0.94 -1.3510845875 -0.96249381864 +0.95 -1.3237154565 -0.95601798287 +0.96 -1.2924462012 -0.9500020467 +0.97 -1.2574929566 -0.94437710703 +0.98 -1.2190803772 -0.93907301456 +0.99 -1.1774405576 -0.9340188113 +1.0 -1.1328119871 -0.92914312957 +1.01 -1.0854385025 -0.92437458501 +1.02 -1.0355685243 -0.91964203242 +1.03 -0.98345355948 -0.9148751682 +1.04 -0.92934684605 -0.91000502329 +1.05 -0.87350300645 -0.90496400678 +1.06 -0.816176798 -0.89968636628 +1.07 -0.75762253462 -0.89410834934 +1.08 -0.6980927773 -0.88816867824 +1.09 -0.63783592919 -0.88180942862 +1.1 -0.57709733472 -0.87497547403 +1.11 -0.51611779897 -0.86761500666 +1.12 -0.45513452401 -0.85967911403 +1.13 -0.39437481773 -0.8511240992 +1.14 -0.33406039561 -0.84190970478 +1.15 -0.27440564306 -0.83199970915 +1.16 -0.2156182574 -0.82136164295 +1.17 -0.15789253305 -0.80996907161 +1.18 -0.10141514283 -0.79779940144 +1.19 -0.046364491037 -0.78483408344 +1.2 0.0070936861676 -0.77106002408 +1.21 0.058806100624 -0.75646931936 +1.22 0.10862841944 -0.74105803457 +1.23 0.15642703646 -0.72482675268 +1.24 0.20208507927 -0.70778222023 +1.25 0.24549451563 -0.68993480428 +1.26 0.28655709783 -0.67129881979 +1.27 0.32519346013 -0.65189485473 +1.28 0.3613324373 -0.63174663121 +1.29 0.39491300311 -0.61088161585 +1.3 0.42589280427 -0.58933283509 +1.31 0.45423664439 -0.56713591443 +1.32 0.47992110013 -0.54433022226 +1.33 0.50293883528 -0.52095932209 +1.34 0.52328858456 -0.49706895242 +1.35 0.54098326189 -0.47270841667 +1.36 0.55604726384 -0.44792957482 +1.37 0.56851053426 -0.42278632121 +1.38 0.5784190736 -0.3973351547 +1.39 0.58582409631 -0.37163397592 +1.4 0.59078619814 -0.34574254453 +1.41 0.59337646951 -0.3197213658 +1.42 0.59367015608 -0.29363258185 +1.43 0.59175335557 -0.26753813652 +1.44 0.58771624712 -0.24150074456 +1.45 0.58165595284 -0.21558336285 +1.46 0.57367486997 -0.18984748079 +1.47 0.56387989108 -0.16435632354 +1.48 0.55238206952 -0.13916913159 +1.49 0.53929613871 -0.11434699408 +1.5 0.52473924056 -0.089947318262 +1.51 0.50883077179 -0.066027001308 +1.52 0.49169241327 -0.042641156566 +1.53 0.47344482644 -0.019841140122 +1.54 0.45421261948 0.0023209806635 +1.55 0.43411462436 0.023800386214 +1.56 0.41327662113 0.044548868159 +1.57 0.39181369387 0.064528338728 +1.58 0.36984863239 0.083696839073 +1.59 0.34749294808 0.10202192594 +1.6 0.32486335081 0.11946916202 +1.61 0.30206634054 0.13601212183 +1.62 0.27921056742 0.15162430019 +1.63 0.25639594087 0.16628574014 +1.64 0.23372158101 0.17997799414 +1.65 0.21127927552 0.19268795322 +1.66 0.18915759431 0.20440513617 +1.67 0.16743876844 0.21512355823 +1.68 0.1462002202 0.22484043033 +1.69 0.12551333682 0.23355703571 +1.7 0.10544417695 0.24127785752 +1.71 0.08605219679 0.24801143198 +1.72 0.067392201978 0.25376898991 +1.73 0.049510885271 0.25856612142 +1.74 0.032452166492 0.26242022156 +1.75 0.016249456911 0.26535352678 +1.76 0.0009363277765 0.26738900334 +1.77 -0.01346722955 0.26855489619 +1.78 -0.026937386187 0.26887918915 +1.79 -0.039467262404 0.26839525362 +1.8 -0.051042203966 0.26713561709 +1.81 -0.061667545688 0.26513764513 +1.82 -0.071337628583 0.26243806272 +1.83 -0.08006745591 0.25907689464 +1.84 -0.087863054815 0.25509471266 +1.85 -0.094744724104 0.25053311675 +1.86 -0.10073011925 0.24543531389 +1.87 -0.10584404869 0.23984427835 +1.88 -0.11011403507 0.23380415827 +1.89 -0.11356907255 0.22735934704 +1.9 -0.11624417545 0.22055306109 +1.91 -0.11817265478 0.21343130087 +1.92 -0.11939406617 0.20603495539 +1.93 -0.11994678428 0.1984103598 +1.94 -0.1198720834 0.19059676551 +1.95 -0.11921240203 0.18263758346 +1.96 -0.11801032113 0.17457234326 +1.97 -0.1163093863 0.1664389658 +1.98 -0.11415448856 0.15827800515 +1.99 -0.11158739315 0.15012088135 +2.0 -0.10865483203 0.14200718854 +2.01 -0.10539681541 0.13396470695 +2.02 -0.10185745506 0.12602623509 +2.03 -0.098078174326 0.11822061558 +2.04 -0.094096518357 0.11057099092 +2.05 -0.089956724139 0.10310784468 +2.06 -0.085689851344 0.095846617351 +2.07 -0.081335739687 0.088811521611 +2.08 -0.07692635626 0.082018660843 +2.09 -0.072491428123 0.075481468962 +2.1 -0.068066323213 0.069218277483 +2.11 -0.063671603987 0.063233949849 +2.12 -0.059337660545 0.057541412683 +2.13 -0.055085519235 0.052145715315 +2.14 -0.050933564418 0.047049428304 +2.15 -0.046906930776 0.042260020905 +2.16 -0.043014644681 0.037772793136 +2.17 -0.039274785396 0.03358922043 +2.18 -0.035699058908 0.029706278232 +2.19 -0.032293597937 0.026116974923 +2.2 -0.029072509077 0.022818973327 +2.21 -0.026036085371 0.019801644234 +2.22 -0.023189532297 0.017056816878 +2.23 -0.020537674284 0.014575946706 +2.24 -0.018077146728 0.012346366913 +2.25 -0.015810412032 0.010357603026 +2.26 -0.01373392254 0.0085968309318 +2.27 -0.011842116593 0.0070502736984 +2.28 -0.010133845747 0.0057048888874 +2.29 -0.0085993951615 0.0045459798677 +2.3 -0.0072316775018 0.0035587606278 +2.31 -0.0060239032063 0.0027280843754 +2.32 -0.0049651302707 0.0020399454291 +2.33 -0.0040460304292 0.0014785514847 +2.34 -0.0032566099898 0.0010293505502 +2.35 -0.0025860125292 0.00067967964524 +2.36 -0.002022999236 0.00041341795836 +2.37 -0.0015572677849 0.00021897250312 +2.38 -0.0011785405003 8.5467448311e-05 +2.39 -0.00087447260941 -2.6406389151e-06 +2.4 -0.00063670211965 -5.2354589398e-05 +2.41 -0.00045599180494 -7.2310661646e-05 +2.42 -0.00031988217501 -7.6466931673e-05 +2.43 -0.00022336127653 -6.6408931605e-05 +2.44 -0.00015833856538 -4.8596801665e-05 +2.45 -0.00011438780266 -3.2809722084e-05 +2.46 -8.9293750204e-05 -1.6747173877e-05 +2.47 -7.7560959826e-05 -3.3623200012e-06 +2.48 -7.3416025735e-05 3.0092130857e-06 +2.49 -7.2127889582e-05 6.165594886e-06 +2.5 -6.5365569508e-05 6.6445226696e-06 +2.51 -4.1806181002e-05 4.9128311678e-06 +2.52 -1.7125758779e-05 2.2069704328e-06 +2.53 1.4939152703e-06 -1.4573388792e-07 +2.54 4.6480499279e-06 -4.5342490346e-07 +2.55 3.0647699647e-06 -2.98973343e-07 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 2 +0.0 0.0 0.0 +0.01 -0.0016302809637 0.00035794683369 +0.02 -0.0065163515418 0.0014296822768 +0.03 -0.014643908542 0.0032089002835 +0.04 -0.025989159095 0.005685121165 +0.05 -0.040518889195 0.0088437370407 +0.06 -0.058190559384 0.012666075205 +0.07 -0.07895242731 0.017129479149 +0.08 -0.10274369681 0.022207406906 +0.09 -0.12949469307 0.027869546304 +0.1 -0.15912706346 0.034081946665 +0.11 -0.1915540033 0.040807166393 +0.12 -0.22668050609 0.048004435845 +0.13 -0.26440363738 0.055629834819 +0.14 -0.30461283157 0.06363648392 +0.15 -0.34719021073 0.07197474901 +0.16 -0.39201092451 0.08059245791 +0.17 -0.43894351037 0.089435128454 +0.18 -0.48785027272 0.098446206939 +0.19 -0.53858768033 0.10756731604 +0.2 -0.59100678041 0.11673851104 +0.21 -0.64495362865 0.12589854357 +0.22 -0.70026973351 0.13498513144 +0.23 -0.75679251382 0.14393523369 +0.24 -0.81435576809 0.15268532967 +0.25 -0.87279015472 0.16117170107 +0.26 -0.93192368055 0.16933071533 +0.27 -0.99158219813 0.17709911015 +0.28 -1.0515899082 0.18441427678 +0.29 -1.1117698675 0.19121454188 +0.3 -1.1719445 0.19743944636 +0.31 -1.2319361091 0.20303002007 +0.32 -1.2915673908 0.20792905119 +0.33 -1.3506619462 0.21208134961 +0.34 -1.4090447891 0.21543400246 +0.35 -1.4665428517 0.21793662148 +0.36 -1.522985483 0.21954158083 +0.37 -1.5782049408 0.22020424436 +0.38 -1.632036874 0.21988318183 +0.39 -1.6843207957 0.21854037236 +0.4 -1.7349005444 0.21614139504 +0.41 -1.7836247317 0.21265560581 +0.42 -1.8303471759 0.20805629957 +0.43 -1.8749273204 0.20232085696 +0.44 -1.9172306348 0.19543087537 +0.45 -1.9571289972 0.18737228343 +0.46 -1.9945010581 0.17813543873 +0.47 -2.0292325838 0.16771520866 +0.48 -2.0612167791 0.15611103466 +0.49 -2.090354582 0.14332697147 +0.5 -2.1165549426 0.12937171885 +0.51 -2.1397350778 0.11425863393 +0.52 -2.1598206914 0.098005713701 +0.53 -2.176746171 0.080635565211 +0.54 -2.190454786 0.062175383691 +0.55 -2.2008987911 0.04265684497 +0.56 -2.2080395975 0.022116075258 +0.57 -2.211847815 0.00059350427024 +0.58 -2.2123033659 -0.02186620296 +0.59 -2.2093954581 -0.045214335243 +0.6 -2.203122684 -0.06939820269 +0.61 -2.1934928744 -0.09436140661 +0.62 -2.1805231747 -0.12004391211 +0.63 -2.1642398919 -0.14638230327 +0.64 -2.1446783791 -0.17330999434 +0.65 -2.1218829812 -0.20075738758 +0.66 -2.0959068035 -0.22865215777 +0.67 -2.066811508 -0.25691950291 +0.68 -2.0346671658 -0.28548234681 +0.69 -1.999552031 -0.31426159213 +0.7 -1.961552178 -0.34317645587 +0.71 -1.9207612656 -0.37214470648 +0.72 -1.8772802547 -0.40108292621 +0.73 -1.8312170584 -0.42990680852 +0.74 -1.7826861934 -0.45853144401 +0.75 -1.7318084432 -0.48687159166 +0.76 -1.6787103295 -0.51484205255 +0.77 -1.6235238527 -0.542357874 +0.78 -1.5663860216 -0.56933467063 +0.79 -1.5074384166 -0.59568891442 +0.8 -1.4468267401 -0.62133822183 +0.81 -1.3847003559 -0.64620163671 +0.82 -1.3212118192 -0.6701999082 +0.83 -1.2565163983 -0.69325576249 +0.84 -1.1907715897 -0.71529416746 +0.85 -1.1241366287 -0.73624258926 +0.86 -1.0567719967 -0.75603123994 +0.87 -0.98883892656 -0.77459331519 +0.88 -0.92049890866 -0.79186522139 +0.89 -0.85191319821 -0.80778679106 +0.9 -0.78324232642 -0.82230148608 +0.91 -0.71464561672 -0.83535658783 +0.92 -0.64628070784 -0.84690337375 +0.93 -0.57830308546 -0.85689727953 +0.94 -0.51086562408 -0.86529804664 +0.95 -0.4441181986 -0.8720698411 +0.96 -0.37820717285 -0.87718139491 +0.97 -0.31327485984 -0.88060612622 +0.98 -0.24945933646 -0.88232217166 +0.99 -0.18689397376 -0.88231247569 +1.0 -0.12570707039 -0.88056484165 +1.01 -0.066021505913 -0.87707196654 +1.02 -0.0079545268368 -0.871831461 +1.03 0.048382799934 -0.86484587151 +1.04 0.10288589917 -0.85612263048 +1.05 0.15545667375 -0.84567402721 +1.06 0.20600385579 -0.83351716716 +1.07 0.25444313475 -0.81967394231 +1.08 0.30069748155 -0.80417095395 +1.09 0.34469751688 -0.78703927055 +1.1 0.38638120612 -0.76831446513 +1.11 0.42569413044 -0.74803646205 +1.12 0.46258944515 -0.72624964296 +1.13 0.49702846809 -0.70300202094 +1.14 0.52898007647 -0.67834564529 +1.15 0.5584209087 -0.65233634601 +1.16 0.58533542208 -0.62503380899 +1.17 0.60971583811 -0.59650035482 +1.18 0.63156188005 -0.56680171908 +1.19 0.65088091394 -0.5360069337 +1.2 0.66768769801 -0.5041873814 +1.21 0.68200399062 -0.47141655935 +1.22 0.69385878215 -0.43777069314 +1.23 0.70328821897 -0.40332830942 +1.24 0.71033429279 -0.3681684753 +1.25 0.71504576268 -0.33237249494 +1.26 0.71747825821 -0.29602370728 +1.27 0.71769167806 -0.25920453432 +1.28 0.7157522691 -0.22199923787 +1.29 0.71173247372 -0.18449341844 +1.3 0.70570736789 -0.14677065047 +1.31 0.69775812741 -0.10891610311 +1.32 0.68797057633 -0.071014918121 +1.33 0.67643200525 -0.033149740959 +1.34 0.66323581364 0.0045952849381 +1.35 0.64847703464 0.042138299609 +1.36 0.63225221039 0.079399428431 +1.37 0.61466413529 0.1162974046 +1.38 0.59581231224 0.15275639672 +1.39 0.57580091001 0.18870011769 +1.4 0.55473560956 0.22405425572 +1.41 0.53271899406 0.25874913705 +1.42 0.50985999692 0.29271380992 +1.43 0.48626100505 0.32588396457 +1.44 0.462028045 0.35819554496 +1.45 0.43726595407 0.3895878196 +1.46 0.41207481124 0.42000499326 +1.47 0.38656057126 0.44939063297 +1.48 0.3608175272 0.47769672342 +1.49 0.33494686826 0.5048740466 +1.5 0.30904061714 0.53088029232 +1.51 0.28319158621 0.55567492895 +1.52 0.25748919177 0.57922141737 +1.53 0.2320165748 0.60148809531 +1.54 0.20685930393 0.62244468231 +1.55 0.18209023524 0.64206822004 +1.56 0.15778974576 0.66033516372 +1.57 0.13402102121 0.67723064065 +1.58 0.11085528354 0.69273914034 +1.59 0.088348538575 0.70685266968 +1.6 0.066561188568 0.71956401527 +1.61 0.045541575888 0.73087203775 +1.62 0.025339115934 0.74077747805 +1.63 0.0059938468033 0.74928603386 +1.64 -0.012455641987 0.75640594374 +1.65 -0.02997814 0.762149628 +1.66 -0.046545525399 0.76653231439 +1.67 -0.062135771965 0.76957291518 +1.68 -0.07673072274 0.77129311635 +1.69 -0.090317688661 0.77171796373 +1.7 -0.10288787787 0.77087507287 +1.71 -0.11443785977 0.76879513028 +1.72 -0.12496755668 0.76551117705 +1.73 -0.13448246389 0.76105890947 +1.74 -0.14299056152 0.75547633432 +1.75 -0.15050566682 0.74880338946 +1.76 -0.15704320029 0.74108264673 +1.77 -0.162624416 0.73235738719 +1.78 -0.167271634 0.72267441308 +1.79 -0.17101238302 0.71207933872 +1.8 -0.17387544024 0.70062292022 +1.81 -0.17589320453 0.68835205878 +1.82 -0.17710054196 0.67532110015 +1.83 -0.17753340761 0.66157787454 +1.84 -0.17723165939 0.64717812643 +1.85 -0.17623431712 0.63217172116 +1.86 -0.17458444823 0.61661402113 +1.87 -0.17232411255 0.60055706063 +1.88 -0.16949748901 0.58405433927 +1.89 -0.16614984949 0.56716024762 +1.9 -0.16232429048 0.54992499374 +1.91 -0.15806950473 0.53240543237 +1.92 -0.15342617165 0.51464805908 +1.93 -0.14844426834 0.49670968764 +1.94 -0.14316373324 0.47863572096 +1.95 -0.13763122805 0.46047840766 +1.96 -0.13188874746 0.44228507248 +1.97 -0.12597657543 0.42410101919 +1.98 -0.1199394249 0.40597590216 +1.99 -0.11381039894 0.38794793686 +2.0 -0.10763397714 0.37006616432 +2.01 -0.10144010235 0.35236584341 +2.02 -0.095265583264 0.33488859255 +2.03 -0.089142973563 0.31767206088 +2.04 -0.083098195882 0.30074770623 +2.05 -0.077166630832 0.28415568454 +2.06 -0.071365978984 0.26791978875 +2.07 -0.065724731008 0.25207376486 +2.08 -0.060262269553 0.23664279998 +2.09 -0.054995055424 0.22164931787 +2.1 -0.049945501387 0.2071213062 +2.11 -0.045120802665 0.19307206246 +2.12 -0.040537413162 0.17952387706 +2.13 -0.036203128407 0.16649020909 +2.14 -0.032122968947 0.15398166035 +2.15 -0.028307414412 0.1420144928 +2.16 -0.024753587768 0.13059030089 +2.17 -0.02146523645 0.11971812052 +2.18 -0.018441211133 0.10940089432 +2.19 -0.015675932017 0.099636041806 +2.2 -0.013168594079 0.090427492502 +2.21 -0.010909475495 0.081766015473 +2.22 -0.0088911546145 0.073646095504 +2.23 -0.00710559859 0.06606113076 +2.24 -0.0055403251186 0.058996127683 +2.25 -0.0041843627321 0.052439195792 +2.26 -0.00302493571 0.046375295857 +2.27 -0.002048258537 0.040787704768 +2.28 -0.0012396425611 0.03565823697 +2.29 -0.00058545857417 0.030970770784 +2.3 -7.0524374943e-05 0.026706982272 +2.31 0.00032128610825 0.022847262124 +2.32 0.00060267490947 0.019374509348 +2.33 0.00079004202243 0.016268357268 +2.34 0.00089784358158 0.013509078375 +2.35 0.00093745353246 0.011078294802 +2.36 0.00092553347338 0.0089539544753 +2.37 0.00087252062706 0.0071167327738 +2.38 0.00078791337712 0.0055471546788 +2.39 0.00068748885458 0.0042218314666 +2.4 0.00057611638219 0.0031221978739 +2.41 0.0004608042569 0.0022279088413 +2.42 0.00035518992628 0.0015146245011 +2.43 0.00025785924786 0.00096472501905 +2.44 0.00017329303283 0.00055694774566 +2.45 0.00010989139907 0.00026733139288 +2.46 6.1872491483e-05 7.829302357e-05 +2.47 2.953833815e-05 -3.334821152e-05 +2.48 1.4806544348e-05 -9.5806441567e-05 +2.49 9.6783992659e-06 -0.00011221658675 +2.5 8.5136479095e-06 -9.7376625643e-05 +2.51 4.2505513471e-06 -6.772287195e-05 +2.52 1.390094449e-06 -2.9382005001e-05 +2.53 -2.082365022e-07 2.20387025e-06 +2.54 -6.504514312e-07 6.8840503135e-06 +2.55 -4.3057462874e-07 4.5569849889e-06 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -7.0231214206E+00 +0.01 -7.0453717502E+00 +0.02 -7.1143468216E+00 +0.03 -7.2090176842E+00 +0.04 -7.3083553876E+00 +0.05 -7.3952429292E+00 +0.06 -7.4597953536E+00 +0.07 -7.4991297585E+00 +0.08 -7.5152902525E+00 +0.09 -7.5128744992E+00 +0.1 -7.4971693665E+00 +0.11 -7.4730064339E+00 +0.12 -7.4442385526E+00 +0.13 -7.4136363893E+00 +0.14 -7.3830123504E+00 +0.15 -7.3534314963E+00 +0.16 -7.3254268122E+00 +0.17 -7.2991803384E+00 +0.18 -7.2746597190E+00 +0.19 -7.2517133907E+00 +0.2 -7.2301330081E+00 +0.21 -7.2096920714E+00 +0.22 -7.1901684172E+00 +0.23 -7.1713562496E+00 +0.24 -7.1530717169E+00 +0.25 -7.1351547228E+00 +0.26 -7.1174685568E+00 +0.27 -7.0998985249E+00 +0.28 -7.0823500273E+00 +0.29 -7.0647465397E+00 +0.3 -7.0470276188E+00 +0.31 -7.0291470239E+00 +0.32 -7.0110709744E+00 +0.33 -6.9927765879E+00 +0.34 -6.9742504049E+00 +0.35 -6.9554870639E+00 +0.36 -6.9364880834E+00 +0.37 -6.9172607312E+00 +0.38 -6.8978169707E+00 +0.39 -6.8781725207E+00 +0.4 -6.8583459681E+00 +0.41 -6.8383579698E+00 +0.42 -6.8182305369E+00 +0.43 -6.7979863986E+00 +0.44 -6.7776484445E+00 +0.45 -6.7572392483E+00 +0.46 -6.7367806674E+00 +0.47 -6.7162935188E+00 +0.48 -6.6957973262E+00 +0.49 -6.6753101150E+00 +0.5 -6.6548482902E+00 +0.51 -6.6344265604E+00 +0.52 -6.6140578820E+00 +0.53 -6.5937534575E+00 +0.54 -6.5735227996E+00 +0.55 -6.5533737172E+00 +0.56 -6.5333124572E+00 +0.57 -6.5133437238E+00 +0.58 -6.4934708208E+00 +0.59 -6.4736956904E+00 +0.6 -6.4540190739E+00 +0.61 -6.4344405251E+00 +0.62 -6.4149585733E+00 +0.63 -6.3955707664E+00 +0.64 -6.3762737537E+00 +0.65 -6.3570633874E+00 +0.66 -6.3379347681E+00 +0.67 -6.3188823126E+00 +0.68 -6.2998998303E+00 +0.69 -6.2809805806E+00 +0.7 -6.2621173131E+00 +0.71 -6.2433023331E+00 +0.72 -6.2245275562E+00 +0.73 -6.2057845580E+00 +0.74 -6.1870646259E+00 +0.75 -6.1683588117E+00 +0.76 -6.1496579790E+00 +0.77 -6.1309528635E+00 +0.78 -6.1122341237E+00 +0.79 -6.0934923948E+00 +0.8 -6.0747183433E+00 +0.81 -6.0559027212E+00 +0.82 -6.0370364200E+00 +0.83 -6.0181105239E+00 +0.84 -5.9991163614E+00 +0.85 -5.9800455551E+00 +0.86 -5.9608900694E+00 +0.87 -5.9416422543E+00 +0.88 -5.9222948867E+00 +0.89 -5.9028412073E+00 +0.9 -5.8832749527E+00 +0.91 -5.8635903839E+00 +0.92 -5.8437823081E+00 +0.93 -5.8238460972E+00 +0.94 -5.8037776995E+00 +0.95 -5.7835736400E+00 +0.96 -5.7632310315E+00 +0.97 -5.7427475878E+00 +0.98 -5.7221215886E+00 +0.99 -5.7013518781E+00 +1.0 -5.6804378489E+00 +1.01 -5.6593794226E+00 +1.02 -5.6381770133E+00 +1.03 -5.6168315332E+00 +1.04 -5.5953443695E+00 +1.05 -5.5737173207E+00 +1.06 -5.5519525829E+00 +1.07 -5.5300527090E+00 +1.08 -5.5080205953E+00 +1.09 -5.4858594768E+00 +1.1 -5.4635728450E+00 +1.11 -5.4411644391E+00 +1.12 -5.4186381993E+00 +1.13 -5.3959983139E+00 +1.14 -5.3732491142E+00 +1.15 -5.3503950752E+00 +1.16 -5.3274407901E+00 +1.17 -5.3043909888E+00 +1.18 -5.2812504690E+00 +1.19 -5.2580240944E+00 +1.2 -5.2347167970E+00 +1.21 -5.2113335516E+00 +1.22 -5.1878793578E+00 +1.23 -5.1643592404E+00 +1.24 -5.1407782392E+00 +1.25 -5.1171413949E+00 +1.26 -5.0934537514E+00 +1.27 -5.0697203419E+00 +1.28 -5.0459461891E+00 +1.29 -5.0221363068E+00 +1.3 -4.9982956801E+00 +1.31 -4.9744292806E+00 +1.32 -4.9505420602E+00 +1.33 -4.9266389339E+00 +1.34 -4.9027248049E+00 +1.35 -4.8788045416E+00 +1.36 -4.8548829774E+00 +1.37 -4.8309649370E+00 +1.38 -4.8070551869E+00 +1.39 -4.7831584769E+00 +1.4 -4.7592795265E+00 +1.41 -4.7354229970E+00 +1.42 -4.7115935473E+00 +1.43 -4.6877957587E+00 +1.44 -4.6640341935E+00 +1.45 -4.6403133701E+00 +1.46 -4.6166377312E+00 +1.47 -4.5930117259E+00 +1.48 -4.5694396733E+00 +1.49 -4.5459259073E+00 +1.5 -4.5224746470E+00 +1.51 -4.4990900739E+00 +1.52 -4.4757763045E+00 +1.53 -4.4525373437E+00 +1.54 -4.4293771944E+00 +1.55 -4.4062996642E+00 +1.56 -4.3833086284E+00 +1.57 -4.3604076923E+00 +1.58 -4.3376005463E+00 +1.59 -4.3148906194E+00 +1.6 -4.2922813822E+00 +1.61 -4.2697760768E+00 +1.62 -4.2473779371E+00 +1.63 -4.2250900034E+00 +1.64 -4.2029152612E+00 +1.65 -4.1808565335E+00 +1.66 -4.1589165503E+00 +1.67 -4.1370979017E+00 +1.68 -4.1154030601E+00 +1.69 -4.0938343674E+00 +1.7 -4.0723940409E+00 +1.71 -4.0510841603E+00 +1.72 -4.0299066951E+00 +1.73 -4.0088634503E+00 +1.74 -3.9879561588E+00 +1.75 -3.9671863413E+00 +1.76 -3.9465555084E+00 +1.77 -3.9260648902E+00 +1.78 -3.9057157915E+00 +1.79 -3.8855091504E+00 +1.8 -3.8654460818E+00 +1.81 -3.8455272395E+00 +1.82 -3.8257535560E+00 +1.83 -3.8061254478E+00 +1.84 -3.7866435899E+00 +1.85 -3.7673082659E+00 +1.86 -3.7481198716E+00 +1.87 -3.7290785831E+00 +1.88 -3.7101845281E+00 +1.89 -3.6914377938E+00 +1.9 -3.6728382611E+00 +1.91 -3.6543859374E+00 +1.92 -3.6360805050E+00 +1.93 -3.6179218406E+00 +1.94 -3.5999095315E+00 +1.95 -3.5820432604E+00 +1.96 -3.5643225918E+00 +1.97 -3.5467470213E+00 +1.98 -3.5293160912E+00 +1.99 -3.5120291557E+00 +2.0 -3.4948856938E+00 +2.01 -3.4778850017E+00 +2.02 -3.4610264332E+00 +2.03 -3.4443092916E+00 +2.04 -3.4277328079E+00 +2.05 -3.4112962860E+00 +2.06 -3.3949988785E+00 +2.07 -3.3788398185E+00 +2.08 -3.3628182624E+00 +2.09 -3.3469333416E+00 +2.1 -3.3311842302E+00 +2.11 -3.3155699746E+00 +2.12 -3.3000897024E+00 +2.13 -3.2847424519E+00 +2.14 -3.2695272359E+00 +2.15 -3.2544431327E+00 +2.16 -3.2394890382E+00 +2.17 -3.2246639597E+00 +2.18 -3.2099668049E+00 +2.19 -3.1953963961E+00 +2.2 -3.1809517062E+00 +2.21 -3.1666314047E+00 +2.22 -3.1524342799E+00 +2.23 -3.1383591039E+00 +2.24 -3.1244044055E+00 +2.25 -3.1105688087E+00 +2.26 -3.0968509426E+00 +2.27 -3.0832494382E+00 +2.28 -3.0697627913E+00 +2.29 -3.0563898779E+00 +2.3 -3.0431294386E+00 +2.31 -3.0299801887E+00 +2.32 -3.0169410939E+00 +2.33 -3.0040109831E+00 +2.34 -2.9911887500E+00 +2.35 -2.9784733778E+00 +2.36 -2.9658637789E+00 +2.37 -2.9533589232E+00 +2.38 -2.9409577917E+00 +2.39 -2.9286593571E+00 +2.4 -2.9164625797E+00 +2.41 -2.9043664216E+00 +2.42 -2.8923698868E+00 +2.43 -2.8804718654E+00 +2.44 -2.8686712935E+00 +2.45 -2.8569671495E+00 +2.46 -2.8453582499E+00 +2.47 -2.8338434784E+00 +2.48 -2.8224217322E+00 +2.49 -2.8110918278E+00 +2.5 -2.7998525918E+00 +2.51 -2.7887028310E+00 +2.52 -2.7776415409E+00 +2.53 -2.7666676118E+00 +2.54 -2.7557799085E+00 +2.55 -2.7449775402E+00 +2.56 -2.7342595084E+00 +2.57 -2.7236248015E+00 +2.58 -2.7130724883E+00 +2.59 -2.7026015965E+00 +2.6 -2.6922111078E+00 +2.61 -2.6819001800E+00 +2.62 -2.6716679021E+00 +2.63 -2.6615133021E+00 +2.64 -2.6514355718E+00 +2.65 -2.6414338420E+00 +2.66 -2.6315071900E+00 +2.67 -2.6216548251E+00 +2.68 -2.6118759290E+00 +2.69 -2.6021696307E+00 +2.7 -2.5925351489E+00 +2.71 -2.5829717156E+00 +2.72 -2.5734785124E+00 +2.73 -2.5640547609E+00 +2.74 -2.5546997426E+00 +2.75 -2.5454126928E+00 +2.76 -2.5361928300E+00 +2.77 -2.5270394846E+00 +2.78 -2.5179519390E+00 +2.79 -2.5089294248E+00 +2.8 -2.4999712991E+00 +2.81 -2.4910768846E+00 +2.82 -2.4822454645E+00 +2.83 -2.4734763795E+00 +2.84 -2.4647690035E+00 +2.85 -2.4561226740E+00 +2.86 -2.4475367100E+00 +2.87 -2.4390105350E+00 +2.88 -2.4305435276E+00 +2.89 -2.4221350247E+00 +2.9 -2.4137844549E+00 +2.91 -2.4054912392E+00 +2.92 -2.3972547682E+00 +2.93 -2.3890744373E+00 +2.94 -2.3809497189E+00 +2.95 -2.3728800481E+00 +2.96 -2.3648648185E+00 +2.97 -2.3569035173E+00 +2.98 -2.3489956165E+00 +2.99 -2.3411405637E+00 +3.0 -2.3333378034E+00 +3.01 -2.3255868594E+00 +3.02 -2.3178872183E+00 +3.03 -2.3102383329E+00 +3.04 -2.3026397232E+00 +3.05 -2.2950909160E+00 +3.06 -2.2875914196E+00 +3.07 -2.2801407030E+00 +3.08 -2.2727383444E+00 +3.09 -2.2653838770E+00 +3.1 -2.2580768156E+00 +3.11 -2.2508166903E+00 +3.12 -2.2436030858E+00 +3.13 -2.2364355573E+00 +3.14 -2.2293136286E+00 +3.15 -2.2222368833E+00 +3.16 -2.2152049132E+00 +3.17 -2.2082172941E+00 +3.18 -2.2012735614E+00 +3.19 -2.1943733441E+00 +3.2 -2.1875162423E+00 +3.21 -2.1807018469E+00 +3.22 -2.1739297205E+00 +3.23 -2.1671995163E+00 +3.24 -2.1605108486E+00 +3.25 -2.1538633207E+00 +3.26 -2.1472565296E+00 +3.27 -2.1406901400E+00 +3.28 -2.1341637841E+00 +3.29 -2.1276770788E+00 +3.3 -2.1212296497E+00 +3.31 -2.1148211738E+00 +3.32 -2.1084513002E+00 +3.33 -2.1021196610E+00 +3.34 -2.0958259044E+00 +3.35 -2.0895697208E+00 +3.36 -2.0833507757E+00 +3.37 -2.0771687174E+00 +3.38 -2.0710232112E+00 +3.39 -2.0649139620E+00 +3.4 -2.0588406508E+00 +3.41 -2.0528029436E+00 +3.42 -2.0468005172E+00 +3.43 -2.0408330924E+00 +3.44 -2.0349003651E+00 +3.45 -2.0290020196E+00 +3.46 -2.0231377399E+00 +3.47 -2.0173072634E+00 +3.48 -2.0115103000E+00 +3.49 -2.0057465539E+00 +3.5 -2.0000157114E+00 +3.51 -1.9943175274E+00 +3.52 -1.9886517255E+00 +3.53 -1.9830180291E+00 +3.54 -1.9774161269E+00 +3.55 -1.9718457881E+00 +3.56 -1.9663067504E+00 +3.57 -1.9607987500E+00 +3.58 -1.9553214969E+00 +3.59 -1.9498747526E+00 +3.6 -1.9444582756E+00 +3.61 -1.9390718144E+00 +3.62 -1.9337151017E+00 +3.63 -1.9283878872E+00 +3.64 -1.9230899509E+00 +3.65 -1.9178210530E+00 +3.66 -1.9125809500E+00 +3.67 -1.9073693767E+00 +3.68 -1.9021861346E+00 +3.69 -1.8970309951E+00 +3.7 -1.8919037296E+00 +3.71 -1.8868040854E+00 +3.72 -1.8817318554E+00 +3.73 -1.8766868316E+00 +3.74 -1.8716687958E+00 +3.75 -1.8666775210E+00 +3.76 -1.8617127770E+00 +3.77 -1.8567743797E+00 +3.78 -1.8518621212E+00 +3.79 -1.8469757937E+00 +3.8 -1.8421151646E+00 +3.81 -1.8372800505E+00 +3.82 -1.8324702607E+00 +3.83 -1.8276855969E+00 +3.84 -1.8229258539E+00 +3.85 -1.8181908181E+00 +3.86 -1.8134803248E+00 +3.87 -1.8087941851E+00 +3.88 -1.8041322098E+00 +3.89 -1.7994941909E+00 +3.9 -1.7948799518E+00 +3.91 -1.7902893242E+00 +3.92 -1.7857221279E+00 +3.93 -1.7811781826E+00 +3.94 -1.7766572800E+00 +3.95 -1.7721592746E+00 +3.96 -1.7676839959E+00 +3.97 -1.7632312719E+00 +3.98 -1.7588009238E+00 +3.99 -1.7543927664E+00 +4.0 -1.7500066580E+00 +4.01 -1.7456424349E+00 +4.02 -1.7412999332E+00 +4.03 -1.7369789755E+00 +4.04 -1.7326793994E+00 +4.05 -1.7284010643E+00 +4.06 -1.7241438138E+00 +4.07 -1.7199074918E+00 +4.08 -1.7156919241E+00 +4.09 -1.7114969666E+00 +4.1 -1.7073224806E+00 +4.11 -1.7031683172E+00 +4.12 -1.6990343275E+00 +4.13 -1.6949203421E+00 +4.14 -1.6908262304E+00 +4.15 -1.6867518576E+00 +4.16 -1.6826970815E+00 +4.17 -1.6786617602E+00 +4.18 -1.6746457304E+00 +4.19 -1.6706488710E+00 +4.2 -1.6666710522E+00 +4.21 -1.6627121385E+00 +4.22 -1.6587719945E+00 +4.23 -1.6548504641E+00 +4.24 -1.6509474321E+00 +4.25 -1.6470627748E+00 +4.26 -1.6431963631E+00 +4.27 -1.6393480680E+00 +4.28 -1.6355177416E+00 +4.29 -1.6317052709E+00 +4.3 -1.6279105396E+00 +4.31 -1.6241334248E+00 +4.32 -1.6203738033E+00 +4.33 -1.6166315370E+00 +4.34 -1.6129065112E+00 +4.35 -1.6091986186E+00 +4.36 -1.6055077418E+00 +4.37 -1.6018337635E+00 +4.38 -1.5981765561E+00 +4.39 -1.5945360003E+00 +4.4 -1.5909119986E+00 +4.41 -1.5873044391E+00 +4.42 -1.5837132098E+00 +4.43 -1.5801381951E+00 +4.44 -1.5765792679E+00 +4.45 -1.5730363413E+00 +4.46 -1.5695093087E+00 +4.47 -1.5659980634E+00 +4.48 -1.5625024987E+00 +4.49 -1.5590224884E+00 +4.5 -1.5555579458E+00 +4.51 -1.5521087730E+00 +4.52 -1.5486748682E+00 +4.53 -1.5452561298E+00 +4.54 -1.5418524452E+00 +4.55 -1.5384637135E+00 +4.56 -1.5350898499E+00 +4.57 -1.5317307575E+00 +4.58 -1.5283863393E+00 +4.59 -1.5250564978E+00 +4.6 -1.5217411153E+00 +4.61 -1.5184401207E+00 +4.62 -1.5151534216E+00 +4.63 -1.5118809255E+00 +4.64 -1.5086225400E+00 +4.65 -1.5053781613E+00 +4.66 -1.5021476999E+00 +4.67 -1.4989310782E+00 +4.68 -1.4957282082E+00 +4.69 -1.4925390016E+00 +4.7 -1.4893633704E+00 +4.71 -1.4862012066E+00 +4.72 -1.4830524446E+00 +4.73 -1.4799170014E+00 +4.74 -1.4767947930E+00 +4.75 -1.4736857355E+00 +4.76 -1.4705897384E+00 +4.77 -1.4675067108E+00 +4.78 -1.4644365871E+00 +4.79 -1.4613792870E+00 +4.8 -1.4583347306E+00 +4.81 -1.4553028376E+00 +4.82 -1.4522835159E+00 +4.83 -1.4492766914E+00 +4.84 -1.4462822960E+00 +4.85 -1.4433002532E+00 +4.86 -1.4403304868E+00 +4.87 -1.4373729203E+00 +4.88 -1.4344274613E+00 +4.89 -1.4314940487E+00 +4.9 -1.4285726134E+00 +4.91 -1.4256630825E+00 +4.92 -1.4227653832E+00 +4.93 -1.4198794427E+00 +4.94 -1.4170051697E+00 +4.95 -1.4141425126E+00 +4.96 -1.4112914024E+00 +4.97 -1.4084517699E+00 +4.98 -1.4056235455E+00 +4.99 -1.4028066588E+00 +5.0 -1.4000010240E+00 +5.01 -1.3972065925E+00 +5.02 -1.3944232980E+00 +5.03 -1.3916510745E+00 +5.04 -1.3888898557E+00 +5.05 -1.3861395741E+00 +5.06 -1.3834001481E+00 +5.07 -1.3806715315E+00 +5.08 -1.3779536612E+00 +5.09 -1.3752464741E+00 +5.1 -1.3725499072E+00 +5.11 -1.3698638970E+00 +5.12 -1.3671883633E+00 +5.13 -1.3645232633E+00 +5.14 -1.3618685368E+00 +5.15 -1.3592241238E+00 +5.16 -1.3565899640E+00 +5.17 -1.3539659973E+00 +5.18 -1.3513521482E+00 +5.19 -1.3487483721E+00 +5.2 -1.3461546139E+00 +5.21 -1.3435708162E+00 +5.22 -1.3409969216E+00 +5.23 -1.3384328728E+00 +5.24 -1.3358786008E+00 +5.25 -1.3333340559E+00 +5.26 -1.3307991892E+00 +5.27 -1.3282739460E+00 +5.28 -1.3257582715E+00 +5.29 -1.3232521111E+00 +5.3 -1.3207554034E+00 +5.31 -1.3182680912E+00 +5.32 -1.3157901326E+00 +5.33 -1.3133214753E+00 +5.34 -1.3108620674E+00 +5.35 -1.3084118565E+00 +5.36 -1.3059707901E+00 +5.37 -1.3035388010E+00 +5.38 -1.3011158551E+00 +5.39 -1.2987019027E+00 +5.4 -1.2962968941E+00 +5.41 -1.2939007796E+00 +5.42 -1.2915135095E+00 +5.43 -1.2891350244E+00 +5.44 -1.2867652793E+00 +5.45 -1.2844042331E+00 +5.46 -1.2820518385E+00 +5.47 -1.2797080480E+00 +5.48 -1.2773728144E+00 +5.49 -1.2750460886E+00 +5.5 -1.2727278110E+00 +5.51 -1.2704179504E+00 +5.52 -1.2681164617E+00 +5.53 -1.2658232996E+00 +5.54 -1.2635384189E+00 +5.55 -1.2612617746E+00 +5.56 -1.2589933135E+00 +5.57 -1.2567329919E+00 +5.58 -1.2544807741E+00 +5.59 -1.2522366172E+00 +5.6 -1.2500004780E+00 +5.61 -1.2477723135E+00 +5.62 -1.2455520805E+00 +5.63 -1.2433397235E+00 +5.64 -1.2411352113E+00 +5.65 -1.2389385051E+00 +5.66 -1.2367495638E+00 +5.67 -1.2345683465E+00 +5.68 -1.2323948119E+00 +5.69 -1.2302289177E+00 +5.7 -1.2280706113E+00 +5.71 -1.2259198665E+00 +5.72 -1.2237766443E+00 +5.73 -1.2216409055E+00 +5.74 -1.2195126109E+00 +5.75 -1.2173917213E+00 +5.76 -1.2152781938E+00 +5.77 -1.2131719831E+00 +5.78 -1.2110730624E+00 +5.79 -1.2089813942E+00 +5.8 -1.2068969412E+00 +5.81 -1.2048196662E+00 +5.82 -1.2027495316E+00 +5.83 -1.2006864953E+00 +5.84 -1.1986305166E+00 +5.85 -1.1965815689E+00 +5.86 -1.1945396164E+00 +5.87 -1.1925046237E+00 +5.88 -1.1904765552E+00 +5.89 -1.1884553752E+00 +5.9 -1.1864410430E+00 +5.91 -1.1844335204E+00 +5.92 -1.1824327817E+00 +5.93 -1.1804387931E+00 +5.94 -1.1784515207E+00 +5.95 -1.1764709306E+00 +5.96 -1.1744969887E+00 +5.97 -1.1725296572E+00 +5.98 -1.1705688974E+00 +5.99 -1.1686146860E+00 +6.0 -1.1666669908E+00 +6.01 -1.1647257794E+00 diff --git a/src/Ions.cc b/src/Ions.cc index 11c42f64..ea69a870 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -34,13 +34,13 @@ Timer ions_setup_tm("ions::setup"); const double ang2bohr = 1.8897269; // const double rmax = 8.0; -std::map Ions::map_species_ - = { { "H", 1 }, { "D", 1 }, { "Li", 3 }, { "Be", 4 }, { "B", 5 }, - { "C", 6 }, { "N", 7 }, { "O", 8 }, { "F", 9 }, { "Na", 11 }, - { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, { "P", 15 }, { "S", 16 }, - { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, { "Cr", 24 }, { "Mn", 25 }, - { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, { "Cu", 29 }, { "Zn", 30 }, - { "Ga", 31 }, { "Ge", 32 }, { "La", 57 }, { "Au", 79 } }; +std::map Ions::map_species_ = { { "H", 1 }, { "D", 1 }, + { "Li", 3 }, { "Be", 4 }, { "B", 5 }, { "C", 6 }, { "N", 7 }, { "O", 8 }, + { "F", 9 }, { "Na", 11 }, { "Mg", 12 }, { "Al", 13 }, { "Si", 14 }, + { "P", 15 }, { "S", 16 }, { "Cl", 17 }, { "K", 19 }, { "Ca", 20 }, + { "Cr", 24 }, { "Mn", 25 }, { "Fe", 26 }, { "Co", 27 }, { "Ni", 28 }, + { "Cu", 29 }, { "Zn", 30 }, { "Ga", 31 }, { "Ge", 32 }, { "Br", 35 }, + { "La", 57 }, { "Au", 79 } }; int Ions::num_ions_ = -1; short Ions::max_num_proj_ = -1; From f298d43b5adbf4fcb88b5280af5e289180815034 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 11 Feb 2025 15:23:42 -0500 Subject: [PATCH 17/58] Clean up some HDFrestart functions (#303) --- src/HDFrestart.cc | 584 ++----------------------------------- src/HDFrestart.h | 13 +- src/Ions.cc | 27 +- src/LocalizationRegions.cc | 3 +- 4 files changed, 51 insertions(+), 576 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index ac856ff2..0322bfea 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1874,141 +1874,32 @@ void HDFrestart::printTimers(std::ostream& os) create_file_tm_.print(os); close_file_tm_.print(os); } -/* -int HDFrestart::writeRandomState(unsigned short int rand_state[3]) -{ - if( active_ ){ - // Create the data space for new datasets - hsize_t dims[1]={3}; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - if( dataspace_id<0 ){ - (*MPIdata::serr)<<"HDFrestart::writeRandomState(): H5Screate_simple -failed!!!"<& data) +int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - { - (*MPIdata::sout) << "HDFrestart::readAtomicNumbers()..." << std::endl; - } + (*MPIdata::sout) << "HDFrestart::readAtomicData()..." << std::endl; if (active_) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_numbers", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return 0; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_numbers", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -1; } int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- No numbers!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("No " + datasetname); return -1; } data.resize(dim); @@ -2017,17 +1908,13 @@ int HDFrestart::readAtomicNumbers(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -1; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart::readAtomicNumbers() --- H5Dclose failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -1; } } // if active_ @@ -2037,126 +1924,8 @@ int HDFrestart::readAtomicNumbers(std::vector& data) return 0; } -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicIDs()..." << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- No IDs!!!" << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } - } - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -// return -2 means failure -// return -1 means dataset does not exists, and could be from older MGmol -// version -int HDFrestart::readAtomicNLprojIDs(std::vector& data) -{ - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs()..." << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (!exists) return -1; - - hid_t dataset_id = H5Dopen2(file_id_, "/AtomicNLproj_IDs", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readAtomicNLprojIDs() --- " - "H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)(H5Dget_storage_size(dataset_id) / sizeof(int)); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- No IDs!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readAtomicNLprojIDs() --- H5Dclose failed!!!" - << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - return 0; -} - -int HDFrestart::readAtomicPositions(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { if (onpe0) (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; @@ -2165,26 +1934,21 @@ int HDFrestart::readAtomicPositions(std::vector& data) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Ionic_positions", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (!exists) return -1; // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Ionic_positions", H5P_DEFAULT); + hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dopen2 failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dopen2 failed for " + datasetname); return -2; } int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); if (dim == 0) { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- No positions!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("readAtomicData() --- No " + datasetname); return -2; } data.resize(dim); @@ -2193,15 +1957,13 @@ int HDFrestart::readAtomicPositions(std::vector& data) H5P_DEFAULT, &data[0]); if (status < 0) { - (*MPIdata::sout) - << "HDFrestart:readAtomicPositions() --- H5Dread failed!!!" - << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dread failed for " + datasetname); return -2; } status = H5Dclose(dataset_id); if (status < 0) { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; + MGMOL_HDFRESTART_FAIL("H5Dclose failed for " + datasetname); return -2; } } @@ -2218,59 +1980,12 @@ int HDFrestart::readOldCenterOnMesh(std::vector& data, int i) "points from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); + std::stringstream datasetstream; + datasetstream << "OldCenterOnMesh_" << i; - std::stringstream datasetstream; - datasetstream << "OldCenterOnMesh_" << i; + std::string datasetname = datasetstream.str(); - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenterOnMesh() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; + return readAtomicData(datasetname, data); } int HDFrestart::readOldCenter(std::vector& data, int i) @@ -2279,256 +1994,16 @@ int HDFrestart::readOldCenter(std::vector& data, int i) (*MPIdata::sout) << "Read old localization centers from hdf5 file" << std::endl; - if (active_) - { - assert(file_id_ >= 0); - - std::stringstream datasetstream; - datasetstream << "OldCenter_" << i; - - std::string datasetname = datasetstream.str(); - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- No old centers!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readOldCenter() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readGidsList(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read list of gids from hdf5 file" << std::endl; - - if (active_) - { - assert(file_id_ >= 0); - - std::string datasetname = "GidsList"; - - htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (!exists) return -1; - - // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); - if (dataset_id < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dopen2 failed!!!" - << std::endl; - return -2; - } - - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(int); - if (dim == 0) - { - if (onpe0) - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- No GidsList!!!" - << std::endl; - return -2; - } - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, &data[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart:readGidsList() --- H5Dread failed!!!" - << std::endl; - return -2; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -2; - } - } - - if (gather_data_x_) gatherDataXdir(data); - - return 0; -} - -int HDFrestart::readAtomicVelocities(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "Read atomic velocities from hdf5 file" - << std::endl; - - if (active_) - { - assert(file_id_ >= 0); + std::stringstream datasetstream; + datasetstream << "OldCenter_" << i; - htri_t exists = H5Lexists(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (exists) - { - - // Open an existing dataset - hid_t dataset_id - = H5Dopen2(file_id_, "/Ionic_velocities", H5P_DEFAULT); - if (dataset_id < 0) - { - std::cerr << "HDFrestart::readAtomicVelocities(), " - "H5Dopen failed->no velocities read" - << std::endl; - data.clear(); - return -1; - } - int dim = (int)H5Dget_storage_size(dataset_id) / sizeof(double); - data.resize(dim); - - herr_t status = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, - H5S_ALL, H5P_DEFAULT, &data[0]); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dread failed!!!"); - return -2; - } - - status = H5Dclose(dataset_id); - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Dclose failed!!!"); - return -2; - } - } - } + std::string datasetname = datasetstream.str(); - if (gather_data_x_) gatherDataXdir(data); - - return 0; + return readAtomicData(datasetname, data); } -int HDFrestart::readLockedAtomNames(std::vector& data) -{ - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames()..." << std::endl; - - std::vector buffer; - short name_length = 7; // default, value used before February 2016 - - if (active_) - { - assert(file_id_ >= 0); - - htri_t exists = H5Lexists(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (!exists) return 0; - - hid_t dataset_id = H5Dopen2(file_id_, "/LockedAtomsNames", H5P_DEFAULT); - if (dataset_id < 0) - { - if (onpe0) - (*MPIdata::sout) << "HDFrestart::readLockedAtomNames(), " - "H5Dopen failed->no locked atoms read" - << std::endl; - return -1; - } - - std::string attname("String_Length"); - htri_t existsA = H5Aexists(dataset_id, attname.c_str()); - if (existsA) - { - hid_t attribute_id = H5Aopen_name(dataset_id, attname.c_str()); - herr_t status = H5Aread(attribute_id, H5T_NATIVE_INT, &name_length); - // check validity of data just read - if (status < 0) - { - MGMOL_HDFRESTART_FAIL("H5Aread failed!!!"); - return -1; - } - } - - int dim = (int)H5Dget_storage_size(dataset_id) / name_length; - - if (onpe0) - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), dataset size=" << dim - << std::endl; - - if (dim == 0) return 0; - - buffer.resize(dim * name_length); - - // create type for std::strings of length IonData_MaxStrLength - hid_t strtype = H5Tcopy(H5T_C_S1); - H5Tset_size(strtype, name_length); - herr_t status = H5Dread( - dataset_id, strtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buffer[0]); - if (status < 0) - { - (*MPIdata::sout) - << "HDFrestart::readLockedAtomNames(), H5Dread failed!!!" - << std::endl; - return -1; - } - status = H5Dclose(dataset_id); - if (status < 0) - { - (*MPIdata::sout) << "H5Dclose failed!!!" << std::endl; - return -1; - } - } - - if (gather_data_x_) gatherDataXdir(buffer); - - data.clear(); - for (unsigned short i = 0; i < buffer.size(); i += name_length) - { - std::string t(&buffer[i], name_length); - assert(t.size() > 0); - - stripLeadingAndTrailingBlanks(t); - - assert(t.size() > 0); - data.push_back(t); - } - - return 0; -} - -int HDFrestart::readAtomicNames(std::vector& data) +int HDFrestart::readAtomicData( + std::string datasetname, std::vector& data) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -2541,11 +2016,12 @@ int HDFrestart::readAtomicNames(std::vector& data) { assert(file_id_ >= 0); - htri_t exists = H5Lexists(file_id_, "/Atomic_names", H5P_DEFAULT); + htri_t exists = H5Lexists(file_id_, datasetname.c_str(), H5P_DEFAULT); if (exists) { // Open the dataset - hid_t dataset_id = H5Dopen2(file_id_, "/Atomic_names", H5P_DEFAULT); + hid_t dataset_id + = H5Dopen2(file_id_, datasetname.c_str(), H5P_DEFAULT); if (dataset_id < 0) { MGMOL_HDFRESTART_FAIL("H5Dopen2 failed!!!"); @@ -2577,7 +2053,7 @@ int HDFrestart::readAtomicNames(std::vector& data) int dim = (int)H5Dget_storage_size(dataset_id) / name_length; if (dim == 0) { - if (onpe0) MGMOL_HDFRESTART_FAIL("No names!!!"); + MGMOL_HDFRESTART_FAIL("No names!!!"); return -1; } diff --git a/src/HDFrestart.h b/src/HDFrestart.h index 72c5c1f5..1ed2d080 100644 --- a/src/HDFrestart.h +++ b/src/HDFrestart.h @@ -260,19 +260,12 @@ class HDFrestart template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); - // int writeRandomState(unsigned short int rand_state[3]); - // int readRandomState(unsigned short* rand_state); - int readAtomicIDs(std::vector& data); - int readAtomicNLprojIDs(std::vector& data); - int readAtomicNumbers(std::vector& data); - int readAtomicNames(std::vector& data); - int readAtomicPositions(std::vector& data); - int readAtomicVelocities(std::vector& data); - int readLockedAtomNames(std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); + int readAtomicData(std::string datasetname, std::vector& data); int readRestartRandomStates(std::vector& data); int readOldCenter(std::vector& data, int i); int readOldCenterOnMesh(std::vector& data, int i); - int readGidsList(std::vector& data); void addDateToFilename(); void addMDTime2File(const float run_time); diff --git a/src/Ions.cc b/src/Ions.cc index ea69a870..913a0f55 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -700,13 +700,12 @@ void Ions::readLockedAtomNames(HDFrestart& h5f_file) if (dim == 0) return; std::vector data; - h5f_file.readLockedAtomNames(data); + std::string datasetname("/LockedAtomsNames"); + h5f_file.readAtomicData(datasetname, data); - for (std::vector::const_iterator i = data.begin(), - end = data.end(); - i != end; ++i) + for (auto& i : data) { - lockAtom(*i); + lockAtom(i); } } @@ -874,13 +873,17 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) setupListIonsBoundaries(rmax); std::vector at_numbers; - h5_file.readAtomicNumbers(at_numbers); + std::string datasetname("/Atomic_numbers"); + h5_file.readAtomicData(datasetname, at_numbers); std::vector at_indexes; - int nidxs = h5_file.readAtomicIDs(at_indexes); + std::string datasetname_indexes("/Atomic_IDs"); + int nidxs = h5_file.readAtomicData(datasetname_indexes, at_indexes); std::vector at_nlprojIds; - int npids = h5_file.readAtomicNLprojIDs(at_nlprojIds); + std::string datasetname_nlprojIds("/AtomicNLproj_IDs"); + int npids = h5_file.readAtomicData(datasetname_nlprojIds, at_nlprojIds); std::vector at_names; - h5_file.readAtomicNames(at_names); + std::string datasetname_names("/Atomic_names"); + h5_file.readAtomicData(datasetname_names, at_names); if (onpe0 && ct.verbose > 2) { std::cout << "HDF file: at nb=" << at_numbers.size() << std::endl; @@ -967,7 +970,8 @@ void Ions::readRestartPositions(HDFrestart& h5_file) (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; std::vector data; - h5_file.readAtomicPositions(data); + std::string datasetname("/Ionic_positions"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) @@ -1142,7 +1146,8 @@ void Ions::readRestartVelocities(HDFrestart& h5_file) << std::endl; std::vector data; - h5_file.readAtomicVelocities(data); + std::string datasetname("/Ionic_velocities"); + h5_file.readAtomicData(datasetname, data); int i = 0; for (auto& ion : local_ions_) diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 99cda54e..60f28301 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -1726,7 +1726,8 @@ void LocalizationRegions::writeOldCenter(HDFrestart& h5f_file, int i) void LocalizationRegions::setupOldCenters(HDFrestart& h5_file) { vector gids; - h5_file.readGidsList(gids); + std::string datasetname("GidsList"); + h5_file.readAtomicData(datasetname, gids); map gids_map; for (unsigned int i = 0; i < gids.size(); i++) From c690bc5bb8998a1202ebc22a16ebb3ffb80346a6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Feb 2025 07:19:17 -0500 Subject: [PATCH 18/58] Fix and test restart single hdf5 file (#305) --- src/Control.h | 2 + src/HDFrestart.cc | 28 ++++++++--- src/Ions.cc | 20 ++++++-- src/restart.cc | 2 +- tests/CMakeLists.txt | 8 +++ tests/DMandEnergyAndForces/test.py | 3 ++ tests/HDF5single/h2o.xyz | 6 +++ tests/HDF5single/md.cfg | 34 +++++++++++++ tests/HDF5single/mgmol.cfg | 31 ++++++++++++ tests/HDF5single/test.py | 75 ++++++++++++++++++++++++++++ tests/RestartEnergyAndForces/test.py | 5 +- tests/WFEnergyAndForces/test.py | 3 ++ 12 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 tests/HDF5single/h2o.xyz create mode 100644 tests/HDF5single/md.cfg create mode 100644 tests/HDF5single/mgmol.cfg create mode 100755 tests/HDF5single/test.py diff --git a/src/Control.h b/src/Control.h index b3401c1d..bd4a7d96 100644 --- a/src/Control.h +++ b/src/Control.h @@ -371,6 +371,8 @@ class Control // 10 or larger means CG, otherwise MG V-cycles bool MGPoissonSolver() { return (diel_flag_ / 10 == 0); } + bool LangevinThermostat() { return (thermostat_type == 1); } + // // data // diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 0322bfea..0fac518e 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -457,7 +457,8 @@ HDFrestart::HDFrestart(const std::string& filename, const pb::PEenv& pes, verbosity_ = 0; closed_ = false; - //(*MPIdata::sout)<<"HDFrestart::HDFrestart(), filename="<& data) // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), -1), data.end()); + } + return 0; } @@ -1928,7 +1934,7 @@ int HDFrestart::readAtomicData( std::string datasetname, std::vector& data) { if (onpe0) - (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + (*MPIdata::sout) << "Read atomic data from hdf5 file" << std::endl; if (active_) { @@ -1968,6 +1974,10 @@ int HDFrestart::readAtomicData( } } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); + } if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2094,6 +2104,11 @@ int HDFrestart::readAtomicData( data.push_back(t); } + if (useHdf5p()) + { + data.erase(std::remove(data.begin(), data.end(), ""), data.end()); + } + return 0; } @@ -2128,7 +2143,6 @@ int HDFrestart::readRestartRandomStates(std::vector& data) dim = (int)H5Dget_storage_size(dataset_id) / sizeof(unsigned short); } - if (dim > 0) { data.resize(dim); @@ -2148,12 +2162,10 @@ int HDFrestart::readRestartRandomStates(std::vector& data) } if (!data.empty()) - if (data[0] != data[0]) + if (std::isnan(data[0])) { MGMOL_HDFRESTART_FAIL( - "ERROR: HDFrestart::readRestartRandomStates() " - "--- data[0]=" - << data[0]); + "readRestartRandomStates() is NaN"); return -2; } } diff --git a/src/Ions.cc b/src/Ions.cc index 913a0f55..e93d82ea 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -911,8 +911,10 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); - mmpi.allreduce(&num_ions_, 1, MPI_SUM); - + if (!h5_file.useHdf5p()) + { + mmpi.allreduce(&num_ions_, 1, MPI_SUM); + } if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ @@ -947,9 +949,21 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) } readRestartPositions(h5_file); readRestartVelocities(h5_file); - readRestartRandomStates(h5_file); + if (ct.LangevinThermostat()) readRestartRandomStates(h5_file); readLockedAtomNames(h5_file); + // remove atoms from local list if not local + for (std::vector::iterator it = local_ions_.begin(); + it != local_ions_.end();) + { + double p[3]; + (*it)->getPosition(p); + if (!inLocalIons(p[0], p[1], p[2])) + it = local_ions_.erase(it); + else + ++it; + } + // rescale all velocities by factor specified in input rescaleVelocities(ct.VelocityScalingFactor()); diff --git a/src/restart.cc b/src/restart.cc index 2fa76ecb..247cca06 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -117,7 +117,7 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicIDs(h5f_file); ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); - ions.writeRandomStates(h5f_file); + if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b29c1aad..baf1c451 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -504,6 +504,14 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/DMandEnergyAndForces/test.py b/tests/DMandEnergyAndForces/test.py index c36f0767..b21e6e72 100755 --- a/tests/DMandEnergyAndForces/test.py +++ b/tests/DMandEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test DMandEnergyAndForces...") @@ -33,6 +34,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: diff --git a/tests/HDF5single/h2o.xyz b/tests/HDF5single/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/HDF5single/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg new file mode 100644 index 00000000..1ff2adab --- /dev/null +++ b/tests/HDF5single/md.cfg @@ -0,0 +1,34 @@ +verbosity=3 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +input_type=single_file +output_level=4 +output_filename=WF_MD +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg new file mode 100644 index 00000000..4dba942a --- /dev/null +++ b/tests/HDF5single/mgmol.cfg @@ -0,0 +1,31 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF +output_type=single_file +[Coloring] +scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py new file mode 100755 index 00000000..080ee0ba --- /dev/null +++ b/tests/HDF5single/test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-5] +input1 = sys.argv[nargs-4] +input2 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +os.remove('WF') + +print("Check energy conservation...") +tol = 1.e-4 +energy = 0. +count = 0 +for line in lines: + if line.count(b'Total') and line.count(b'Energy'): + print(line) + count=count+1 + words=line.split() + + energy=eval(words[2]) + if count==1: + first_energy=energy + + if count>1 and abs(energy-first_energy)>tol: + print("ERROR Energy = {} != {}".format(energy,first_energy)) + sys.exit(1) + +if count<4: + print("ERROR needs 4 energy values for checking conservation!") + sys.exit(1) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index 349434e8..b62d39f8 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test RestartEnergyAndForces...") @@ -54,14 +55,14 @@ ref_energy=energy break -#sys.exit(0) - #run test command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input2,coords) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + test_energy=1.e18 for line in lines: if line.count(b'%%'): diff --git a/tests/WFEnergyAndForces/test.py b/tests/WFEnergyAndForces/test.py index 45420ddf..6cac8c93 100755 --- a/tests/WFEnergyAndForces/test.py +++ b/tests/WFEnergyAndForces/test.py @@ -3,6 +3,7 @@ import os import subprocess import string +import shutil print("Test WFEnergyAndForces...") @@ -37,6 +38,8 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') +shutil.rmtree('WF') + #analyse output energies=[] for line in lines: From 9f1fc20a63eb19e229a9f502b30d3271b264a3fc Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Feb 2025 07:19:37 -0500 Subject: [PATCH 19/58] Save Hartree potential for write in restart file (#306) --- src/Potentials.cc | 16 ++++++++++++---- src/Potentials.h | 23 +++++++++++++++++------ src/md.cc | 3 +++ src/restart.cc | 8 ++++++++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/Potentials.cc b/src/Potentials.cc index bed005db..abe277f1 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -75,10 +75,13 @@ Potentials::Potentials() dv_.resize(size_); - memset(&vepsilon_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&vh_rho_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&vxc_rho_[0], 0, size_ * sizeof(POTDTYPE)); - memset(&v_ext_[0], 0, size_ * sizeof(POTDTYPE)); + vh_rho_backup_.resize(size_); + + memset(vepsilon_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vh_rho_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vxc_rho_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(v_ext_.data(), 0, size_ * sizeof(POTDTYPE)); + memset(vh_rho_backup_.data(), 0, size_ * sizeof(POTDTYPE)); #ifdef HAVE_TRICUBIC vext_tricubic_ = NULL; @@ -596,6 +599,11 @@ void Potentials::axpVcomp(POTDTYPE* v, const double alpha) LinearAlgebraUtils::MPaxpy(size_, alpha, &v_comp_[0], v); } +void Potentials::backupVh() +{ + memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); +} + void Potentials::initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp) { diff --git a/src/Potentials.h b/src/Potentials.h index 970cf95d..71819a75 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -76,6 +76,11 @@ class Potentials std::vector dv_; + /*! + * Backpup copy of Hartree potential to save previous state + */ + std::vector vh_rho_backup_; + int itindex_vxc_; int itindex_vh_; @@ -138,14 +143,15 @@ class Potentials double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } - POTDTYPE* vtot() { return &vtot_[0]; } - POTDTYPE* vh_rho() { return &vh_rho_[0]; } - RHODTYPE* rho_comp() { return &rho_comp_[0]; } + POTDTYPE* vtot() { return vtot_.data(); } + POTDTYPE* vh_rho() { return vh_rho_.data(); } + RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return &v_nuc_[0]; } - POTDTYPE* vext() { return &v_ext_[0]; } - POTDTYPE* vepsilon() { return &vepsilon_[0]; } + POTDTYPE* vnuc() { return v_nuc_.data(); } + POTDTYPE* vext() { return v_ext_.data(); } + POTDTYPE* vepsilon() { return vepsilon_.data(); } + POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); void axpVcomp(POTDTYPE* v, const double alpha); @@ -196,6 +202,11 @@ class Potentials void initBackground(Ions& ions); void addBackgroundToRhoComp(); + /*! + * Save current Hartree potential into backup array + */ + void backupVh(); + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/md.cc b/src/md.cc index b50618c8..dfb4dfe6 100644 --- a/src/md.cc +++ b/src/md.cc @@ -54,6 +54,9 @@ void MGmol::moveVnuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // save Hartree potential internally + pot.backupVh(); + // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); initNuc(ions); diff --git a/src/restart.cc b/src/restart.cc index 247cca06..30bdad73 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -141,6 +141,14 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, pot.vh_rho(), "Hartree", &ll[0], &origin[0]); if (ierr < 0) return ierr; + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + // Write if (ct.diel) { From 8a1cf0530f2a2246dc62fca1db5b145919330101 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 19 Feb 2025 17:11:48 -0500 Subject: [PATCH 20/58] Speed-up recently added tests (#309) * simply use a smaller domain/mesh --- tests/DMandEnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/EnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/RestartEnergyAndForces/h2o.xyz | 8 ++++---- tests/RestartEnergyAndForces/mgmol.cfg | 18 +++++++++--------- tests/RestartEnergyAndForces/restart.cfg | 18 +++++++++--------- 5 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tests/DMandEnergyAndForces/mgmol.cfg b/tests/DMandEnergyAndForces/mgmol.cfg index 5278f40b..e7712706 100644 --- a/tests/DMandEnergyAndForces/mgmol.cfg +++ b/tests/DMandEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=LDA FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/EnergyAndForces/mgmol.cfg b/tests/EnergyAndForces/mgmol.cfg index cf23889e..d1299db9 100644 --- a/tests/EnergyAndForces/mgmol.cfg +++ b/tests/EnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=1 xcFunctional=LDA FDtype=Mehrstellen [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-6. -oy=-6. -oz=-6. -lx=12. -ly=12. -lz=12. +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.N_ONCVPSP_LDA [Run] diff --git a/tests/RestartEnergyAndForces/h2o.xyz b/tests/RestartEnergyAndForces/h2o.xyz index cdc906f6..d5171c8b 100644 --- a/tests/RestartEnergyAndForces/h2o.xyz +++ b/tests/RestartEnergyAndForces/h2o.xyz @@ -1,6 +1,6 @@ 3 -https://pubchem.ncbi.nlm.nih.gov/compound/Water -O 2.5369 -0.1550 0.0 -H 3.0739 0.1550 0.0 -H 2.0000 0.1550 0.0 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 diff --git a/tests/RestartEnergyAndForces/mgmol.cfg b/tests/RestartEnergyAndForces/mgmol.cfg index e590f810..b96f3d9c 100644 --- a/tests/RestartEnergyAndForces/mgmol.cfg +++ b/tests/RestartEnergyAndForces/mgmol.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 diff --git a/tests/RestartEnergyAndForces/restart.cfg b/tests/RestartEnergyAndForces/restart.cfg index 99bc77d8..280778e9 100644 --- a/tests/RestartEnergyAndForces/restart.cfg +++ b/tests/RestartEnergyAndForces/restart.cfg @@ -2,16 +2,16 @@ verbosity=2 xcFunctional=PBE FDtype=4th [Mesh] -nx=64 -ny=64 -nz=64 +nx=48 +ny=48 +nz=48 [Domain] -ox=-3.4 -oy=-6.4 -oz=-6.4 -lx=12.8 -ly=12.8 -lz=12.8 +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. [Potentials] pseudopotential=pseudo.O_ONCV_PBE_SG15 pseudopotential=pseudo.H_ONCV_PBE_SG15 From 8a2687965d538179129ea9aed22d93129a7ac1c6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 11:23:03 -0500 Subject: [PATCH 21/58] Encapsulate some functions in Potentials (#310) * some functions were called at random places and confusing --- src/Electrostatic.cc | 4 +- src/Electrostatic.h | 2 +- src/Forces.cc | 3 +- src/MGmol.cc | 18 +--- src/Poisson.h | 5 +- src/Potentials.cc | 203 ++++++++++++++++++++++++++++++------------- src/Potentials.h | 25 +++--- src/restart.cc | 49 +---------- 8 files changed, 165 insertions(+), 144 deletions(-) diff --git a/src/Electrostatic.cc b/src/Electrostatic.cc index 6ac4fef3..aba09d07 100644 --- a/src/Electrostatic.cc +++ b/src/Electrostatic.cc @@ -65,7 +65,7 @@ Electrostatic::~Electrostatic() if (grhoc_ != nullptr) delete grhoc_; } -void Electrostatic::setupInitialVh(const POTDTYPE* const vh_init) +void Electrostatic::setupInitialVh(const std::vector& vh_init) { poisson_solver_->set_vh(vh_init); @@ -153,7 +153,7 @@ void Electrostatic::setupPB( // initialize vh with last trial solution pb::GridFunc gf_vh(*pbGrid_, bc_[0], bc_[1], bc_[2]); - gf_vh.assign(pot.vh_rho()); + gf_vh.assign((pot.vh_rho()).data()); poisson_solver_->set_vh(gf_vh); } diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 072ce0cc..9beee899 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -58,7 +58,7 @@ class Electrostatic template void computeVh(const pb::GridFunc& vhinit, const Ions& ions, Rho& rho, Potentials& pot); - void setupInitialVh(const POTDTYPE* const); + void setupInitialVh(const std::vector&); void setupInitialVh(const pb::GridFunc&); template void computeVhRho(Rho& rho); diff --git a/src/Forces.cc b/src/Forces.cc index 9ab19f6a..04e6ffd7 100644 --- a/src/Forces.cc +++ b/src/Forces.cc @@ -272,9 +272,10 @@ void Forces::get_loc_proj(RHODTYPE* rho, const int numpt = mymesh->numpt(); Potentials& pot = hamiltonian_->potential(); + const std::vector& vh_rho(pot.vh_rho()); for (int idx = 0; idx < numpt; idx++) { - const double vhrho = pot.vh_rho(idx); + const double vhrho = vh_rho[idx]; for (short dir = 0; dir < 3; dir++) { double* lproj = &(loc_proj[dir * NPTS]); diff --git a/src/MGmol.cc b/src/MGmol.cc index 681efacc..649f2c39 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -319,8 +319,6 @@ int MGmol::initial() mmpi.barrier(); if (ct.verbose > 0) current_orbitals_->printChromaticNumber(os_); - pot.initBackground(*ions_); - // Random initialization of the wavefunctions if (ct.restart_info <= 2) { @@ -834,23 +832,9 @@ void MGmol::initNuc(Ions& ions) Potentials& pot = hamiltonian_->potential(); + // initialize poentials based on ionic positions and their species pot.initialize(ions); - // Check compensating charges - double comp_rho = getCharge(pot.rho_comp()); - - if (onpe0 && ct.verbose > 1) - { - os_ << std::setprecision(8) << std::fixed - << " Charge of rhoc: " << comp_rho << std::endl; - } - -#if 1 - pot.rescaleRhoComp(); -#endif - - pot.addBackgroundToRhoComp(); - electrostat_->setupRhoc(pot.rho_comp()); if (onpe0 && ct.verbose > 3) os_ << " initNuc done" << std::endl; diff --git a/src/Poisson.h b/src/Poisson.h index 87107542..ba5daa39 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -71,7 +71,10 @@ class Poisson : public PoissonInterface virtual void set_rhod(pb::GridFunc* /*rhod*/){}; void set_vh(const pb::GridFunc& vh) { (*vh_) = vh; }; - void set_vh(const POTDTYPE* const vh) { vh_->assign(vh, 'd'); }; + void set_vh(const std::vector& vh) + { + vh_->assign(vh.data(), 'd'); + }; void resetVh() { vh_->resetData(); } void set_vepsilon(const POTDTYPE* const vepsilon) { diff --git a/src/Potentials.cc b/src/Potentials.cc index abe277f1..a0ff3f1a 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -25,7 +25,6 @@ #include "mputils.h" #include -using namespace std; // unit conversion factor Ha -> Ry const double ha2ry = 2.; @@ -92,7 +91,7 @@ void Potentials::initWithVnuc() { assert(size_ > 0); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::initWithVnuc()" << endl; + (*MPIdata::sout) << "Potentials::initWithVnuc()" << std::endl; itindex_vxc_ = 0; itindex_vh_ = 0; int ione = 1; @@ -122,7 +121,8 @@ double Potentials::min() const return vmin; } -void Potentials::evalNormDeltaVtotRho(const vector>& rho) +void Potentials::evalNormDeltaVtotRho( + const std::vector>& rho) { Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); @@ -141,14 +141,14 @@ void Potentials::evalNormDeltaVtotRho(const vector>& rho) mmpi.allreduce(&scf_dvrho_, 1, MPI_SUM); } -double Potentials::update(const vector>& rho) +double Potentials::update(const std::vector>& rho) { assert(itindex_vxc_ >= 0); assert(itindex_vh_ >= 0); assert(itindex_vxc_ == itindex_vh_); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::update(rho)" << endl; + (*MPIdata::sout) << "Potentials::update(rho)" << std::endl; int ione = 1; Mesh* mymesh = Mesh::instance(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -185,7 +185,7 @@ double Potentials::update(const vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; + std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; MPI_Abort(myPEenv.comm(), 2); } dvdot = sum; @@ -202,7 +202,7 @@ void Potentials::update(const double mix) assert(itindex_vxc_ == itindex_vh_); #ifdef DEBUG - if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << endl; + if (onpe0) (*MPIdata::sout) << "Potentials::update(mix)" << std::endl; #endif // int ione=1; double potmix = mix; @@ -210,13 +210,13 @@ void Potentials::update(const double mix) size_, potmix, &dv_[0], &vtot_[0]); } -double Potentials::delta_v(const vector>& rho) +double Potentials::delta_v(const std::vector>& rho) { assert(itindex_vxc_ == itindex_vh_); assert(size_ > 0); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::delta_v()" << endl; + (*MPIdata::sout) << "Potentials::delta_v()" << std::endl; int ione = 1; Mesh* mymesh = Mesh::instance(); @@ -253,7 +253,7 @@ double Potentials::delta_v(const vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - cout << "MPI_Allreduce double sum failed!!!" << endl; + std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; MPI_Abort(myPEenv.comm(), 2); } dvdot = sum; @@ -266,7 +266,7 @@ double Potentials::delta_v(const vector>& rho) } // in Ry -void Potentials::getVofRho(vector& vrho) const +void Potentials::getVofRho(std::vector& vrho) const { vrho.resize(size_); int ione = 1; @@ -283,7 +283,7 @@ void Potentials::getVofRho(vector& vrho) const // type: // 2->text // 3->binary -void Potentials::readExternalPot(const string filename, const char type) +void Potentials::readExternalPot(const std::string filename, const char type) { assert(type == 2 || type == 3); @@ -304,27 +304,27 @@ void Potentials::readExternalPot(const string filename, const char type) { from->seekg(0, ios::end); const int length = from->tellg(); - (*MPIdata::sout) << "Length file = " << length << endl; + (*MPIdata::sout) << "Length file = " << length << std::endl; from->seekg(0, ios::beg); if (length <= 0) { (*MPIdata::serr) - << "ERROR Potential: file length <=0!!!!" << endl; + << "ERROR Potential: file length <=0!!!!" << std::endl; mmpi.abort(); } } } if (!from) { - (*MPIdata::serr) << " Cannot open file " << filename << endl; + (*MPIdata::serr) << " Cannot open file " << filename << std::endl; mmpi.abort(); } if (onpe0) { (*MPIdata::sout) << "Potentials::read_ExternalPot(), filename=" - << filename << endl; - if (type == 2) (*MPIdata::sout) << "text file..." << endl; - if (type == 3) (*MPIdata::sout) << "binary file..." << endl; + << filename << std::endl; + if (type == 2) (*MPIdata::sout) << "text file..." << std::endl; + if (type == 3) (*MPIdata::sout) << "binary file..." << std::endl; } // read origin and end of cell (to check compatibility) @@ -348,29 +348,32 @@ void Potentials::readExternalPot(const string filename, const char type) if (onpe0) for (short d = 0; d < 3; d++) { - (*MPIdata::sout) << setprecision(8); + (*MPIdata::sout) << std::setprecision(8); if (fabs(origin[d] - mygrid.origin(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell origin in direction " - << d << endl; - (*MPIdata::serr) << "Potential origin=" << origin[d] << endl; - (*MPIdata::serr) << "MGmol origin=" << mygrid.origin(d) << endl; + << d << std::endl; + (*MPIdata::serr) + << "Potential origin=" << origin[d] << std::endl; + (*MPIdata::serr) + << "MGmol origin=" << mygrid.origin(d) << std::endl; (*MPIdata::serr) << "Difference=" << fabs(origin[d] - mygrid.origin(d)) - << endl; + << std::endl; mmpi.abort(); } if (fabs(ll[d] - mygrid.ll(d)) > 1.e-3) { (*MPIdata::serr) << "ERROR Potential: Incompatible cell " "dimension in direction " - << d << endl; - (*MPIdata::serr) << "Potential cell end=" << end[d] << endl; + << d << std::endl; + (*MPIdata::serr) + << "Potential cell end=" << end[d] << std::endl; (*MPIdata::serr) - << "Potential cell dimension=" << ll[d] << endl; + << "Potential cell dimension=" << ll[d] << std::endl; (*MPIdata::serr) - << "MGmol cell dimension=" << mygrid.ll(d) << endl; + << "MGmol cell dimension=" << mygrid.ll(d) << std::endl; mmpi.abort(); } } @@ -392,9 +395,9 @@ void Potentials::readExternalPot(const string filename, const char type) if (nxyz[i] != gdim_[i]) { (*MPIdata::serr) << "Potentials::read_ExternalPot(): dimension " - << i << " incompatible with Grid!!!" << endl; + << i << " incompatible with Grid!!!" << std::endl; (*MPIdata::serr) - << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << endl; + << "n=" << nxyz[i] << ", gdim_=" << gdim_[i] << std::endl; mmpi.abort(); } @@ -454,7 +457,7 @@ void Potentials::readExternalPot(const string filename, const char type) if (type == 3) { - vector tmp(dim_[2]); + std::vector tmp(dim_[2]); for (int i = 0; i < dim_[0]; i++) { // advance (start-file_index) positions @@ -512,7 +515,8 @@ void Potentials::getGradVext(const double r[3], double dfdr[3]) const vext_tricubic_->getGradient(r, dfdr, comm); } -void Potentials::getValVext(const vector& r, vector& val) const +void Potentials::getValVext( + const std::vector& r, std::vector& val) const { assert(vext_tricubic_ != NULL); @@ -522,21 +526,22 @@ void Potentials::getValVext(const vector& r, vector& val) const } #endif -void Potentials::readAll(vector& sp) +void Potentials::readAll(std::vector& sp) { assert(sp.size() <= pot_filenames_.size()); if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::readAll() for " << pot_types_.size() - << " potentials" << endl; + << " potentials" << std::endl; Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); double hmin = mygrid.hmin(); if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "hmin= " << hmin << endl; + (*MPIdata::sout) << "hmin= " << hmin << std::endl; - vector::const_iterator it_filename = pot_filenames_.begin(); - int isp = 0; + std::vector::const_iterator it_filename + = pot_filenames_.begin(); + int isp = 0; while (it_filename != pot_filenames_.end()) { if (pot_types_[isp] == 'n' || pot_types_[isp] == 's' @@ -556,7 +561,7 @@ void Potentials::readAll(vector& sp) #else (*MPIdata::sout) << "ERROR: cannot read external potential " - << " -> need to compile with Tricubic library" << endl; + << " -> need to compile with Tricubic library" << std::endl; #endif } it_filename++; @@ -594,11 +599,6 @@ void Potentials::axpVcompToVh(const double alpha) size_, alpha, &v_comp_[0], &vh_rho_[0]); } -void Potentials::axpVcomp(POTDTYPE* v, const double alpha) -{ - LinearAlgebraUtils::MPaxpy(size_, alpha, &v_comp_[0], v); -} - void Potentials::backupVh() { memcpy(vh_rho_backup_.data(), vh_rho_.data(), size_ * sizeof(POTDTYPE)); @@ -792,9 +792,12 @@ void Potentials::initialize(Ions& ions) const pb::Grid& mygrid = mymesh->grid(); const int numpt = mygrid.size(); - memset(&v_comp_[0], 0, numpt * sizeof(POTDTYPE)); - memset(&rho_comp_[0], 0, numpt * sizeof(RHODTYPE)); - memset(&v_nuc_[0], 0, numpt * sizeof(RHODTYPE)); + memset(v_comp_.data(), 0, numpt * sizeof(POTDTYPE)); + memset(rho_comp_.data(), 0, numpt * sizeof(RHODTYPE)); + memset(v_nuc_.data(), 0, numpt * sizeof(RHODTYPE)); + + // Count up the total ionic charge + ionic_charge_ = ions.computeIonicCharge(); char flag_filter = pot_type(0); @@ -805,6 +808,7 @@ void Potentials::initialize(Ions& ions) Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + // initialize rho_comp_, v_comp_, v_nuc_ if (flag_filter == 's') { const int sampleRate = 3; @@ -820,6 +824,13 @@ void Potentials::initialize(Ions& ions) initializeRadialDataOnMesh(position, sp); } } + + // rescale rho_comp_ due to finite mesh effects + rescaleRhoComp(); + + initBackground(); + + addBackgroundToRhoComp(); } void Potentials::rescaleRhoComp() @@ -831,24 +842,33 @@ void Potentials::rescaleRhoComp() const pb::Grid& mygrid = mymesh->grid(); // Check compensating charges - double comp_rho = getCharge(&rho_comp_[0]); + double comp_rho = getCharge(rho_comp_.data()); + if (onpe0 && ct.verbose > 1) + { + std::cout << std::setprecision(8) << std::fixed + << " Charge of rhoc: " << comp_rho << std::endl; + } if (onpe0 && ct.verbose > 1) { - cout << " Rescaling rhoc" << endl; + std::cout << " Rescaling rhoc" << std::endl; } + + // rescale rho_comp_ (initialized by sampling on mesh) + // so that its integral exactly matches ionic_charge_ if (ionic_charge_ > 0.) { const int numpt = mygrid.size(); double t = ionic_charge_ / comp_rho; - LinearAlgebraUtils::MPscal(numpt, t, &rho_comp_[0]); + LinearAlgebraUtils::MPscal( + numpt, t, rho_comp_.data()); // Check new compensating charges - comp_rho = getCharge(&rho_comp_[0]); + comp_rho = getCharge(rho_comp_.data()); } if (onpe0 && ct.verbose > 1) - cout << " Rescaled compensating charges: " << setprecision(8) << fixed - << comp_rho << endl; + std::cout << " Rescaled compensating charges: " << std::setprecision(8) + << std::fixed << comp_rho << std::endl; if (comp_rho < 0.) mmpi.abort(); } @@ -868,26 +888,23 @@ void Potentials::addBackgroundToRhoComp() { if (onpe0) { - cout << setprecision(12) << scientific - << "Add background charge " << background << " to rhoc " - << endl; + std::cout << std::setprecision(12) << std::scientific + << "Add background charge " << background + << " to rhoc " << std::endl; } for (int i = 0; i < numpt; i++) rho_comp_[i] += background; // Check new compensating charges - getCharge(&rho_comp_[0]); + getCharge(rho_comp_.data()); } } } -void Potentials::initBackground(Ions& ions) +void Potentials::initBackground() { Control& ct = *(Control::instance()); - // Count up the total ionic charge - ionic_charge_ = ions.computeIonicCharge(); - // calculation the compensating background charge // for charged supercell calculations background_charge_ = 0.; @@ -898,14 +915,76 @@ void Potentials::initBackground(Ions& ions) } if (onpe0 && ct.verbose > 0) { - cout << "N electrons= " << ct.getNel() << endl; - cout << "ionic charge= " << ionic_charge_ << endl; - cout << "background charge=" << background_charge_ << endl; + std::cout << "N electrons= " << ct.getNel() << std::endl; + std::cout << "ionic charge= " << ionic_charge_ << std::endl; + std::cout << "background charge=" << background_charge_ << std::endl; } if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; } +int Potentials::read(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + // Read total potential + h5f_file.read_1func_hdf5(vtot_.data(), "Vtotal"); + + // Read the hartree potential + h5f_file.read_1func_hdf5(vh_rho_.data(), "Hartree"); + + // Read dielectric potential + if (ct.diel) + { + h5f_file.read_1func_hdf5(vepsilon_.data(), "VDielectric"); + } + + return 0; +} + +int Potentials::write(HDFrestart& h5f_file) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + double ll[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + double origin[3] = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + + // Write total potential + int ierr + = h5f_file.write_1func_hdf5(vtot_.data(), "Vtotal", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + // Write the hartree potential + ierr = h5f_file.write_1func_hdf5( + vh_rho_.data(), "Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + { + // Write hartree potential before extrapolation + ierr = h5f_file.write_1func_hdf5( + vh_rho_backup_.data(), "Preceding_Hartree", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + } + + // Write + if (ct.diel) + { + ierr = h5f_file.write_1func_hdf5( + vepsilon_.data(), "VDielectric", &ll[0], &origin[0]); + } + if (ierr < 0) return ierr; + + // Write external potential + ierr = h5f_file.write_1func_hdf5(v_ext_.data(), "Vext", &ll[0], &origin[0]); + if (ierr < 0) return ierr; + + return ierr; +} + template void Potentials::setVxc( const double* const vxc, const int iterativeIndex); template void Potentials::setVxc( diff --git a/src/Potentials.h b/src/Potentials.h index 71819a75..cb0f8e05 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -10,6 +10,7 @@ #ifndef MGMOL_POTENTIALS_H #define MGMOL_POTENTIALS_H +#include "HDFrestart.h" #include "Rho.h" #include "TriCubic.h" @@ -100,6 +101,12 @@ class Potentials void initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp); + void rescaleRhoComp(); + + void addBackgroundToRhoComp(); + + void initBackground(); + public: Potentials(); @@ -139,27 +146,22 @@ class Potentials void turnOnDiel() { diel_ = true; } + int write(HDFrestart& h5f_file); + int read(HDFrestart& h5f_file); + int size() const { return size_; } double scf_dvrho(void) const { return scf_dvrho_; } double scf_dv(void) const { return scf_dv_; } POTDTYPE* vtot() { return vtot_.data(); } - POTDTYPE* vh_rho() { return vh_rho_.data(); } RHODTYPE* rho_comp() { return rho_comp_.data(); } const std::vector& vnuc() const { return v_nuc_; } - POTDTYPE* vnuc() { return v_nuc_.data(); } - POTDTYPE* vext() { return v_ext_.data(); } + const std::vector& vh_rho() const { return vh_rho_; } + POTDTYPE* vepsilon() { return vepsilon_.data(); } - POTDTYPE* vh_rho_backup() { return vh_rho_backup_.data(); } void axpVcompToVh(const double alpha); - void axpVcomp(POTDTYPE* v, const double alpha); - - POTDTYPE vtot(const int i) { return vtot_[i]; } - POTDTYPE vh_rho(const int i) { return vh_rho_[i]; } - POTDTYPE vxc_rho(const int i) { return vxc_rho_[i]; } - POTDTYPE vepsilon(const int i) { return vepsilon_[i]; } bool diel() const { return diel_; } @@ -198,9 +200,6 @@ class Potentials void setVh(const pb::GridFunc& vh, const int iterativeIndex); void initialize(Ions& ions); - void rescaleRhoComp(); - void initBackground(Ions& ions); - void addBackgroundToRhoComp(); /*! * Save current Hartree potential into backup array diff --git a/src/restart.cc b/src/restart.cc index 30bdad73..ed7f478c 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -43,27 +43,11 @@ int MGmol::read_rho_and_pot_hdf5( os_ << "Try to read density and potentials" << std::endl; Potentials& pot = hamiltonian_->potential(); + pot.read(file); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - POTDTYPE* tmp = new POTDTYPE[mygrid.size()]; - - // Read total potential - file.read_1func_hdf5(pot.vtot(), "Vtotal"); - - // Read the hartree potential - file.read_1func_hdf5(tmp, "Hartree"); - pot.setVh(tmp, 0); - - // Read dielectric potential - if (ct.diel) - { - file.read_1func_hdf5(pot.vepsilon(), "VDielectric"); - } // Read the Density rho.readRestart(file); - delete[] tmp; return 0; } @@ -131,41 +115,12 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, if (ct.out_restart_info > 1) { - // Write total potential - int ierr = h5f_file.write_1func_hdf5( - pot.vtot(), "Vtotal", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - // Write the hartree potential - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho(), "Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - - if (ct.AtomsDynamic() == AtomsDynamicType::MD) - { - // Write hartree potential before extrapolation - ierr = h5f_file.write_1func_hdf5( - pot.vh_rho_backup(), "Preceding_Hartree", &ll[0], &origin[0]); - if (ierr < 0) return ierr; - } - - // Write - if (ct.diel) - { - ierr = h5f_file.write_1func_hdf5( - pot.vepsilon(), "VDielectric", &ll[0], &origin[0]); - } - if (ierr < 0) return ierr; + int ierr = pot.write(h5f_file); // Write the Density ierr = h5f_file.write_1func_hdf5( &rho[0][0], "Density", &ll[0], &origin[0]); if (ierr < 0) return ierr; - - // Write external potential - ierr - = h5f_file.write_1func_hdf5(pot.vext(), "Vext", &ll[0], &origin[0]); - if (ierr < 0) return ierr; } // Write wavefunctions and old centers. From 59b79e75b3f538c1f2ec6a76dc1806fa4d4df059 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 12:54:11 -0500 Subject: [PATCH 22/58] Remove confusing 0 in naming restart files (#308) * use added integer only in case of fail/retry --- src/MGmol.cc | 1 - src/md.cc | 9 ++++++--- tests/MD_D72/test.py | 2 +- tests/MD_MVP/md.cfg | 2 +- tests/MD_MVP/test.py | 8 ++++---- tests/ShortSighted/test.py | 2 +- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 649f2c39..439f4fac 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1096,7 +1096,6 @@ void MGmol::dumpRestart() // create restart file std::string filename(std::string(ct.out_restart_file)); - if (ct.out_restart_file_naming_strategy) filename += "0"; HDFrestart h5restartfile( filename, myPEenv, gdim, ct.out_restart_file_type); diff --git a/src/md.cc b/src/md.cc index dfb4dfe6..dc2857eb 100644 --- a/src/md.cc +++ b/src/md.cc @@ -242,9 +242,12 @@ int MGmol::dumpMDrestartFile(OrbitalsType& orbitals, Ions& ions, std::string filename(std::string(ct.out_restart_file)); // add an integer corresponding to attempt number/count // to allow several attempts at creating and writing file - std::stringstream s; - s << count; - filename += s.str(); + if (count > 0) + { + std::stringstream s; + s << count; + filename += s.str(); + } HDFrestart h5file(filename, myPEenv, gdim, ct.out_restart_file_type); diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index 17b70de2..3f98b2bd 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') diff --git a/tests/MD_MVP/md.cfg b/tests/MD_MVP/md.cfg index 83d5aa4c..e30a8807 100644 --- a/tests/MD_MVP/md.cfg +++ b/tests/MD_MVP/md.cfg @@ -34,6 +34,6 @@ solver=MVP nb_inner_it=1 mixing=1. [Restart] -input_filename=wave.out +input_filename=snapshotMVP input_level=3 output_level=3 diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py index 489c9c8c..47b5462d 100755 --- a/tests/MD_MVP/test.py +++ b/tests/MD_MVP/test.py @@ -42,20 +42,20 @@ #run MD for i in range(2): - command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" + command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') print(restart_file) - os.rename(restart_file, 'wave.out') + os.rename(restart_file, 'snapshotMVP') #run MGmol command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) output2 = subprocess.check_output(command,shell=True) #remove used restart files - shutil.rmtree('wave.out') + shutil.rmtree('snapshotMVP') #analyse mgmol standard output lines=output2.split(b'\n') @@ -81,7 +81,7 @@ sys.exit(1) #remove last restart files -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') shutil.rmtree(restart_file) diff --git a/tests/ShortSighted/test.py b/tests/ShortSighted/test.py index 27da7f7d..68a617aa 100755 --- a/tests/ShortSighted/test.py +++ b/tests/ShortSighted/test.py @@ -42,7 +42,7 @@ print(line) #run MD -command = "ls -ld snapshot0* | awk '{ print $9 }' | tail -n1" +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" print(command) restart_file = subprocess.check_output(command,shell=True) restart_file=str(restart_file[:-1],'utf-8') From 16a818a43bae7906ff9b08f1765898c6b0569ae8 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 20 Feb 2025 18:48:12 -0500 Subject: [PATCH 23/58] Add functionalities for extra info in restart file (#312) --- src/HDFrestart.cc | 22 ++++++------ src/HDFrestart.h | 8 ++--- src/Ion.cc | 8 ++++- src/Ion.h | 16 +++++---- src/Ions.cc | 87 +++++++++++++++++++++++++++++++++++++++++++++-- src/Ions.h | 5 +++ src/restart.cc | 2 ++ 7 files changed, 124 insertions(+), 24 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 0fac518e..87982290 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1440,8 +1440,8 @@ int HDFrestart::read_1func_hdf5(T* vv, const std::string& datasetname) } template -int HDFrestart::write_1func_hdf5( - T* vv, const std::string& datasetname, double* ll, double* cell_origin) +int HDFrestart::write_1func_hdf5(const T* const vv, + const std::string& datasetname, double* ll, double* cell_origin) { assert(ll != nullptr); assert(cell_origin != nullptr); @@ -1652,7 +1652,7 @@ int HDFrestart::readData( } template -int HDFrestart::writeData(T* data, hid_t space_id, hid_t memspace, +int HDFrestart::writeData(const T* const data, hid_t space_id, hid_t memspace, hid_t dset_id, const short precision) { if (precision == 1) @@ -2017,7 +2017,8 @@ int HDFrestart::readAtomicData( { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) - (*MPIdata::sout) << "HDFrestart::readAtomicNames()..." << std::endl; + (*MPIdata::sout) << "HDFrestart::readAtomicData(), dataset = " + << datasetname << std::endl; std::vector buffer; short name_length = 7; // default, value used before February 2016 @@ -2095,10 +2096,9 @@ int HDFrestart::readAtomicData( { std::string t(&buffer[i], name_length); assert(t.size() > 0); - // cout<<"name="< - int writeData(T* vv, hid_t filespace, hid_t memspace, hid_t dset_id, - const short precision); + int writeData(const T* const vv, hid_t filespace, hid_t memspace, + hid_t dset_id, const short precision); template int readData(T* vv, hid_t memspace, hid_t dset_id, const short precision); diff --git a/src/Ion.cc b/src/Ion.cc index 33465635..90350477 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -294,12 +294,18 @@ void Ion::getIonData(IonData& idata) const } } +void Ion::resetPositionsToPrevious() +{ + for (int pos = 0; pos < 3; pos++) + position_[pos] = old_position_[pos]; +} + void Ion::setFromIonData(const IonData& data) { // random state setRandomState(data.rand_state[0], data.rand_state[1], data.rand_state[2]); // previous position - setOldPosition( + setPreviousPosition( data.old_position[0], data.old_position[1], data.old_position[2]); // velocity setVelocity(data.velocity[0], data.velocity[1], data.velocity[2]); diff --git a/src/Ion.h b/src/Ion.h index ae870820..871205a3 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -74,12 +74,6 @@ class Ion position_[i] += shift; } - void setOldPosition(const double x, const double y, const double z) - { - old_position_[0] = x; - old_position_[1] = y; - old_position_[2] = z; - } public: Ion(const Species& species, const std::string& name, const double crds[3], @@ -96,6 +90,13 @@ class Ion void init(const double crds[3], const double velocity[3], const bool lock); void setup(); + void setPreviousPosition(const double x, const double y, const double z) + { + old_position_[0] = x; + old_position_[1] = y; + old_position_[2] = z; + } + std::shared_ptr kbproj() { return kbproj_; } const std::shared_ptr kbproj() const { return kbproj_; } @@ -189,6 +190,9 @@ class Ion kbproj_->clear(); } + + void resetPositionsToPrevious(); + void shiftPositionXLBOMDTest(Vector3D shift) { for (short dir = 0; dir < 3; dir++) diff --git a/src/Ions.cc b/src/Ions.cc index e93d82ea..09b97d2e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -853,6 +853,42 @@ void Ions::writePositions(HDFrestart& h5f_file) } } +void Ions::writePreviousPositions(HDFrestart& h5f_file) +{ + Control& ct(*(Control::instance())); + + if (onpe0 && ct.verbose > 1) + { + (*MPIdata::sout) << "Ions::writePositions" << std::endl; + } + + std::vector data; + if (h5f_file.gatherDataX()) + { + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MPI_Comm comm = myPEenv.comm_x(); + + gatherPreviousPositions(data, 0, comm); + } + else + { + for (auto& ion : local_ions_) + { + data.push_back(ion->getPreviousPosition(0)); + data.push_back(ion->getPreviousPosition(1)); + data.push_back(ion->getPreviousPosition(2)); + } + } + + hid_t file_id = h5f_file.file_id(); + if (file_id >= 0) + { + std::string datasetname("/Ionic_previous_positions"); + writeData2d(h5f_file, datasetname, data, 3, 1.e32); + } +} + void Ions::initFromRestartFile(HDFrestart& h5_file) { assert(list_ions_.empty()); @@ -995,14 +1031,39 @@ void Ions::readRestartPositions(HDFrestart& h5_file) } } +void Ions::readRestartPreviousPositions(HDFrestart& h5_file) +{ + Control& ct = *(Control::instance()); + if (onpe0 && ct.verbose > 0) + (*MPIdata::sout) << "Read ionic positions from hdf5 file" << std::endl; + + std::vector data; + std::string datasetname("/Ionic_previous_positions"); + h5_file.readAtomicData(datasetname, data); + assert(data.size() == 3 * local_ions_.size()); + + int i = 0; + for (auto& ion : local_ions_) + { + ion->setPreviousPosition(data[3 * i], data[3 * i + 1], data[3 * i + 2]); + i++; + } +} + +void Ions::resetPositionsToPrevious() +{ + for (auto& ion : local_ions_) + { + ion->resetPositionsToPrevious(); + } +} + void Ions::writeVelocities(HDFrestart& h5f_file) { Control& ct(*(Control::instance())); if (onpe0 && ct.verbose > 1) - { (*MPIdata::sout) << "Ions::writeVelocities" << std::endl; - } std::vector data; if (h5f_file.gatherDataX()) @@ -2535,6 +2596,28 @@ void Ions::gatherPositions( if (mype == root) positions = data; } +void Ions::gatherPreviousPositions( + std::vector& positions, const int root, const MPI_Comm comm) const +{ + std::vector local_positions; + + for (auto& ion : local_ions_) + { + local_positions.push_back(ion->getPreviousPosition(0)); + local_positions.push_back(ion->getPreviousPosition(1)); + local_positions.push_back(ion->getPreviousPosition(2)); + } + + // gather data to PE root + std::vector data; + mgmol_tools::gatherV(local_positions, data, root, comm); + + int mype = 0; + MPI_Comm_rank(comm, &mype); + positions.clear(); + if (mype == root) positions = data; +} + void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { diff --git a/src/Ions.h b/src/Ions.h index ae4f6adb..b03972d1 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -147,6 +147,8 @@ class Ions const MPI_Comm comm) const; void gatherPositions(std::vector& positions, const int root, const MPI_Comm comm) const; + void gatherPreviousPositions(std::vector& positions, const int root, + const MPI_Comm comm) const; void gatherLockedNames(std::vector& names, const int root, const MPI_Comm comm) const; void gatherIndexes( @@ -198,6 +200,7 @@ class Ions ion++; } } + void resetPositionsToPrevious(); void removeMassCenterMotion(); bool hasNLprojectors() @@ -237,6 +240,7 @@ class Ions double kinetic_E(void) const; void writePositions(HDFrestart& h5f_file); + void writePreviousPositions(HDFrestart& h5f_file); void writeVelocities(HDFrestart& h5f_file); void writeRandomStates(HDFrestart& h5f_file); void writeForces(HDFrestart& h5f_file); @@ -350,6 +354,7 @@ class Ions void addIonToList(const Species& sp, const std::string& name, const double crds[3], const double velocity[3], const bool lock); + void readRestartPreviousPositions(HDFrestart& h5_file); // void checkUnicityLocalIons(); }; diff --git a/src/restart.cc b/src/restart.cc index ed7f478c..532043af 100644 --- a/src/restart.cc +++ b/src/restart.cc @@ -102,6 +102,8 @@ int MGmol::write_hdf5(HDFrestart& h5f_file, ions.writeAtomicNLprojIDs(h5f_file); ions.writePositions(h5f_file); if (ct.LangevinThermostat()) ions.writeRandomStates(h5f_file); + if (ct.AtomsDynamic() == AtomsDynamicType::MD) + ions.writePreviousPositions(h5f_file); ions.writeVelocities(h5f_file); ions.writeForces(h5f_file); From 77bd87ef256623ee5d95866a75f601417e559ca7 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 21 Feb 2025 12:51:24 -0500 Subject: [PATCH 24/58] MPI abort (#313) * Fix error code used by MPI_Abort --- src/BlockVector.cc | 2 +- src/Control.cc | 18 ++++++++--------- src/Control.h | 2 +- src/DistMatrix/BlacsContext.cc | 10 +++++----- src/DistMatrix/DistMatrix.h | 2 +- src/DistanceConstraint.cc | 2 +- src/ExtendedGridOrbitals.cc | 2 +- src/IonicAlgorithm.cc | 2 +- src/Ions.cc | 3 +-- src/LDAFunctional.cc | 3 +-- src/LocalizationRegions.cc | 20 +++++-------------- src/LocalizationRegions.h | 6 +++--- src/MDfiles.cc | 11 +++++----- src/MGmol.cc | 15 ++++++-------- src/MGmol.h | 1 - src/NonOrthoDMStrategy.cc | 2 +- src/PBEFunctional.cc | 3 +-- src/Potentials.cc | 8 ++++---- src/ProjectedMatricesSparse.cc | 2 +- src/main.cc | 2 +- src/md.cc | 7 +++---- src/mgmol_memory.cc | 2 +- src/mgmol_run.cc | 2 +- src/pb/Lap.h | 2 +- src/pb/PEenv.cc | 12 +++++------ src/pb/PEenv.h | 6 +++--- src/radial/RadialMeshFunction.cc | 4 ++-- src/setup.cc | 4 ++-- src/sparse_linear_algebra/DataDistribution.cc | 6 +++--- src/tools.cc | 6 +++--- src/tools/MGmol_MPI.cc | 6 +++--- src/tools/MGmol_MPI.h | 2 +- src/tools/Vector3D.cc | 2 +- 33 files changed, 80 insertions(+), 97 deletions(-) diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 40271750..9a411ebe 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -209,7 +209,7 @@ void BlockVector::allocate_storage() std::cerr << "ERROR BlockVector: trying to use allocation " << size_storage_ << " bigger than initialy preallocated " << allocated_size_storage_ << "!!!" << std::endl; - ct.global_exit(0); + ct.global_exit(); } storage_ = class_storage_[my_allocation_]; assert(class_storage_.size() > 0); diff --git a/src/Control.cc b/src/Control.cc index d266aa2e..33fd93c4 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -495,7 +495,7 @@ void Control::sync(void) if (mpirc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } }; @@ -1152,7 +1152,7 @@ void Control::setTolEigenvalueGram(const float tol) << threshold_eigenvalue_gram_ << std::endl; } -void Control::global_exit(int i) { MPI_Abort(comm_global_, i); } +void Control::global_exit() { MPI_Abort(comm_global_, EXIT_FAILURE); } void Control::setSpecies(Potentials& pot) { @@ -1342,7 +1342,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::serr) << "ERROR in Control::setOptions: Invalid restart dump type" << std::endl; - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); } (*MPIdata::sout) << "Output restart file: " << out_restart_file @@ -1549,14 +1549,14 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else { std::cerr << "ERROR: Spread Penalty needs a type" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (spread_penalty_target_ <= 0.) { (*MPIdata::sout) << "Invalid value for Spread Penalty target: " << spread_penalty_target_ << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } @@ -1599,7 +1599,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) { (*MPIdata::sout) << "Invalid value for Thermostat.type: " << thermostat_type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } tkel = vm["Thermostat.temperature"].as(); @@ -1608,7 +1608,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.temperature: " << tkel << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } thtime = vm["Thermostat.relax_time"].as(); if (thtime < 0.) @@ -1616,7 +1616,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.relax_time: " << thtime << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } if (str.compare("SCALING") == 0) @@ -1627,7 +1627,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) (*MPIdata::sout) << "Invalid value for Thermostat.width: " << thwidth << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } } } diff --git a/src/Control.h b/src/Control.h index bd4a7d96..9481cc24 100644 --- a/src/Control.h +++ b/src/Control.h @@ -346,7 +346,7 @@ class Control return pair_mlwf_distance_threshold_; } - void global_exit(int i); + void global_exit(); bool Mehrstellen() const { return (lap_type == 0 || lap_type == 10); } diff --git a/src/DistMatrix/BlacsContext.cc b/src/DistMatrix/BlacsContext.cc index e87cb7dd..7610a3cb 100644 --- a/src/DistMatrix/BlacsContext.cc +++ b/src/DistMatrix/BlacsContext.cc @@ -191,7 +191,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: type = " << type << " is an incorrect parameter" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } size_ = nprow_ * npcol_; @@ -222,7 +222,7 @@ BlacsContext::BlacsContext( { std::cerr << " nprocs_=" << nprocs_ << std::endl; std::cerr << " BlacsContext nprow*npcol > nprocs_" << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); @@ -252,7 +252,7 @@ BlacsContext::BlacsContext( { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << " in " << __FILE__ << ":" << __LINE__ << std::endl; - MPI_Abort(comm_global, 0); + MPI_Abort(comm_global, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -296,7 +296,7 @@ BlacsContext::BlacsContext(BlacsContext& bc, const int irow, const int icol, { std::cerr << " BlacsContext::BlacsContext: invalid parameters" << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } int* pmap = new int[nprow * npcol]; // build pmap @@ -350,7 +350,7 @@ BlacsContext::BlacsContext(const BlacsContext& bc, const char type) std::cerr << " BlacsContext::BlacsContext: row/col incorrect parameter: " << type << std::endl; - MPI_Abort(comm_global_, 0); + MPI_Abort(comm_global_, EXIT_FAILURE); } ictxt_ = Csys2blacs_handle(comm_global_); diff --git a/src/DistMatrix/DistMatrix.h b/src/DistMatrix/DistMatrix.h index f56df2b4..175fbd9b 100644 --- a/src/DistMatrix/DistMatrix.h +++ b/src/DistMatrix/DistMatrix.h @@ -28,7 +28,7 @@ std::cerr << "ERROR in file " << __FILE__ << " at line " << __LINE__ \ << std::endl; \ std::cerr << "Error Message: " << X << std::endl; \ - MPI_Abort(comm_global_, 2); + MPI_Abort(comm_global_, EXIT_FAILURE); #endif diff --git a/src/DistanceConstraint.cc b/src/DistanceConstraint.cc index d5175d7e..6d046f3b 100644 --- a/src/DistanceConstraint.cc +++ b/src/DistanceConstraint.cc @@ -99,7 +99,7 @@ bool DistanceConstraint::enforce(void) { cerr << "mype=" << mype << ", tau1p_[0]=" << tau1p_[0] << endl; cerr << "mype=" << mype << ", tau2p_[0]=" << tau2p_[0] << endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } if (locally_owned_) (*MPIdata::sout) << setprecision(8) << "DistanceConstraint, d=" << d diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 19c0d26b..07face7c 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -1239,7 +1239,7 @@ double ExtendedGridOrbitals::dotProduct( "dot product type" << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(2); + ct.global_exit(); } dot_product_tm_.stop(); diff --git a/src/IonicAlgorithm.cc b/src/IonicAlgorithm.cc index 37a077e4..e804077a 100644 --- a/src/IonicAlgorithm.cc +++ b/src/IonicAlgorithm.cc @@ -57,7 +57,7 @@ void IonicAlgorithm::init(HDFrestart* h5f_file) if (ct.restart_info > 0) { int status = stepper_->init(*h5f_file); - if (status < 0) ct.global_exit(2); + if (status < 0) ct.global_exit(); // if restart data for lbfgs found if (status == 0) diff --git a/src/Ions.cc b/src/Ions.cc index 09b97d2e..9af0a64e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1961,7 +1961,6 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) double velocity[3] = { 0., 0., 0. }; MGmol_MPI& mmpi(*(MGmol_MPI::instance())); - Control& ct(*(Control::instance())); short movable = 0; std::string query; @@ -2005,7 +2004,7 @@ int Ions::read1atom(std::ifstream* tfile, const bool cell_relative) { std::cerr << "ERROR: Invalid name read in input file: " << name_read << std::endl; - ct.global_exit(2); + mmpi.abort(); } short dummy; ss >> dummy; // not used anymore (was species index) diff --git a/src/LDAFunctional.cc b/src/LDAFunctional.cc index 4b60b51e..89f5ee5f 100644 --- a/src/LDAFunctional.cc +++ b/src/LDAFunctional.cc @@ -120,8 +120,7 @@ double LDAFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = (POTDTYPE)sum; return exc; diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 60f28301..f0f96412 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -492,7 +492,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'nglobal_'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (nglobal_ == 0) return; @@ -522,7 +522,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'centers'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } vector::iterator it = all_regions_.begin(); int i = 0; @@ -554,7 +554,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in " "MPI_Bcast of 'radius'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } it = all_regions_.begin(); i = 0; @@ -642,7 +642,7 @@ void LocalizationRegions::bcastLRs() cerr << "ERROR!!!! LocalizationRegions::bcast(), Failure in MPI_Bcast " "of 'volume'!!!" << endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } if (ct.verbose > 0) printWithTimeStamp("LocalizationRegions::bcast() done...", cout); @@ -915,19 +915,9 @@ void LocalizationRegions::setupLocalRegionsFromOverlapRegions() cerr << "ERROR in distribution of localization centers: count=" << count << endl; cerr << "global number of regions=" << nglobal_ << endl; - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } -#if 0 - //if(onpe0) - if( count!=nglobal_ ) - { - cerr<<"ERROR in distribution of localization centers: count="<::write_header() if (ct.isLocMode() && ct.verbose > 3) lrs_->printAllRegions(os_); } -template -void MGmol::global_exit(int i) -{ - MPI_Abort(comm_, i); -} - template void MGmol::check_anisotropy() { @@ -748,7 +742,8 @@ void MGmol::check_anisotropy() << ", hmin=" << mygrid.hmin() << std::endl; (*MPIdata::serr) << "init: Anisotropy too large: " << mygrid.anisotropy() << std::endl; - global_exit(2); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.abort(); } } @@ -1046,7 +1041,8 @@ void MGmol::setup() total_tm_.start(); setup_tm_.start(); - Control& ct = *(Control::instance()); + Control& ct = *(Control::instance()); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (ct.verbose > 0) printWithTimeStamp("MGmol::setup()...", os_); @@ -1064,7 +1060,8 @@ void MGmol::setup() #else int ierr = initial(); #endif - if (ierr < 0) global_exit(0); + + if (ierr < 0) mmpi.abort(); // Write header to stdout write_header(); diff --git a/src/MGmol.h b/src/MGmol.h index 4d82dd64..bccbc7af 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -221,7 +221,6 @@ class MGmol : public MGmolInterface void initNuc(Ions& ions); void initKBR(); - void global_exit(int i); void printEigAndOcc(); int readCoordinates(std::ifstream* tfile, const bool cell_relative); diff --git a/src/NonOrthoDMStrategy.cc b/src/NonOrthoDMStrategy.cc index 5452592f..3835b039 100644 --- a/src/NonOrthoDMStrategy.cc +++ b/src/NonOrthoDMStrategy.cc @@ -46,7 +46,7 @@ int NonOrthoDMStrategy::update(OrbitalsType& orbitals) { std::cerr << "NonOrthoDMStrategy, Invalid mixing value: " << mix_ << std::endl; - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); } if (mmpi.PE0() && ct.verbose > 2) diff --git a/src/PBEFunctional.cc b/src/PBEFunctional.cc index 4a878c3c..29151a14 100644 --- a/src/PBEFunctional.cc +++ b/src/PBEFunctional.cc @@ -167,8 +167,7 @@ double PBEFunctional::computeRhoDotExc() const if (rc != MPI_SUCCESS) { (*MPIdata::sout) << "MPI_Allreduce double sum failed!!!" << std::endl; - Control& ct = *(Control::instance()); - ct.global_exit(2); + mmpi.abort(); } exc = sum; return exc; diff --git a/src/Potentials.cc b/src/Potentials.cc index a0ff3f1a..ebe42320 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -185,8 +185,8 @@ double Potentials::update(const std::vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + MPI_Abort(myPEenv.comm(), EXIT_FAILURE); } dvdot = sum; @@ -253,8 +253,8 @@ double Potentials::delta_v(const std::vector>& rho) = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); if (rc != MPI_SUCCESS) { - std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), 2); + std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; + MPI_Abort(myPEenv.comm(), EXIT_FAILURE); } dvdot = sum; diff --git a/src/ProjectedMatricesSparse.cc b/src/ProjectedMatricesSparse.cc index 8b328b1c..15ad6d6c 100644 --- a/src/ProjectedMatricesSparse.cc +++ b/src/ProjectedMatricesSparse.cc @@ -354,7 +354,7 @@ double ProjectedMatricesSparse::dotProductSimple( << std::endl; MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Abort(mmpi.commSameSpin(), 0); + MPI_Abort(mmpi.commSameSpin(), EXIT_FAILURE); return -1.; } diff --git a/src/main.cc b/src/main.cc index 3fcbf791..3c6830a0 100644 --- a/src/main.cc +++ b/src/main.cc @@ -45,7 +45,7 @@ int main(int argc, char** argv) if (mpirc != MPI_SUCCESS) { std::cerr << "MPI Initialization failed!!!" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } MPI_Comm comm = MPI_COMM_WORLD; diff --git a/src/md.cc b/src/md.cc index dc2857eb..8ec20074 100644 --- a/src/md.cc +++ b/src/md.cc @@ -690,11 +690,10 @@ void MGmol::loadRestartFile(const std::string filename) if (ierr < 0) { if (onpe0) - (*MPIdata::serr) - << "loadRestartFile: failed to read the restart file." - << std::endl; + std::cerr << "loadRestartFile: failed to read the restart file." + << std::endl; - global_exit(0); + mmpi.abort(); } if (!ct.fullyOccupied()) { diff --git a/src/mgmol_memory.cc b/src/mgmol_memory.cc index a1360c99..b1be12d6 100644 --- a/src/mgmol_memory.cc +++ b/src/mgmol_memory.cc @@ -62,7 +62,7 @@ void addTrack(long addr, long asize) if (nPos > MAXNUMALLOCATIONS) { printf("ERROR: Not enough memory slots!!!"); - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } // if(onpe0)printf("nPos=%d, addr=%ld, size=%ld\n",nPos,addr,asize); } diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index a5de1ad5..468d61bd 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -96,7 +96,7 @@ int mgmol_check() { std::cerr << "Code should be called with " << myPEenv.n_mpi_tasks() << " MPI tasks only" << std::endl; - ct.global_exit(2); + ct.global_exit(); } assert(ct.getMGlevels() >= -1); diff --git a/src/pb/Lap.h b/src/pb/Lap.h index 33d90706..460e36c1 100644 --- a/src/pb/Lap.h +++ b/src/pb/Lap.h @@ -35,7 +35,7 @@ class Lap : public FDoper virtual void applyWithPot(GridFunc&, const double* const, T*) { std::cerr << "ERROR: Lap::applyWithPot() not implemented" << std::endl; - MPI_Abort(MPI_COMM_WORLD, 0); + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } std::string name() const { return name_; } diff --git a/src/pb/PEenv.cc b/src/pb/PEenv.cc index d5ff78f3..3e2fb777 100644 --- a/src/pb/PEenv.cc +++ b/src/pb/PEenv.cc @@ -197,7 +197,7 @@ PEenv::~PEenv() if (mpirc != MPI_SUCCESS) { std::cerr << "MPI_Comm_free failed!" << std::endl; - MPI_Abort(comm_, 2); + MPI_Abort(comm_, EXIT_FAILURE); } } if (cart_comm_ != MPI_COMM_NULL) MPI_Comm_free(&cart_comm_); @@ -277,7 +277,7 @@ void PEenv::task2xyz() if (rc != MPI_SUCCESS) { std::cerr << " error in MPI_Cart_coords()!!!" << std::endl; - MPI_Abort(comm_, 1); + MPI_Abort(comm_, EXIT_FAILURE); } #else mytask_dir_[2] = mytask_ % n_mpi_tasks_dir_[2]; @@ -650,7 +650,7 @@ void PEenv::split_comm(const int nx, const int ny, const int nz, const int bias) { std::cerr << "MPI_Comm_split failed!, my color_=" << color_ << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } MPI_Comm_size(comm_active_, &n_mpi_tasks_); #ifndef NDEBUG @@ -696,7 +696,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Recv() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } else if (ip == mytask_) @@ -708,7 +708,7 @@ void PEenv::printPEnames(std::ostream& os) const { std::cerr << "PEenv::printPEnames, MPI_Send() failed!!!" << std::endl; - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } } if (mytask_ == 0) @@ -717,7 +717,7 @@ void PEenv::printPEnames(std::ostream& os) const if (mytask_ == 0) os << std::endl; } -void PEenv::globalExit() const { MPI_Abort(comm_, 2); } +void PEenv::globalExit() const { MPI_Abort(comm_, EXIT_FAILURE); } void PEenv::bcast(int* val, const int n) const { diff --git a/src/pb/PEenv.h b/src/pb/PEenv.h index bc4751a5..9782ddb9 100644 --- a/src/pb/PEenv.h +++ b/src/pb/PEenv.h @@ -166,7 +166,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxXdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -181,7 +181,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxYdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } @@ -196,7 +196,7 @@ class PEenv { std::cerr << "ERROR in PEenv::maxZdir()" << std::endl; sleep(5); - MPI_Abort(comm_, 0); + MPI_Abort(comm_, EXIT_FAILURE); } delete[] sendbuf; } diff --git a/src/radial/RadialMeshFunction.cc b/src/radial/RadialMeshFunction.cc index 499e482a..79b32518 100644 --- a/src/radial/RadialMeshFunction.cc +++ b/src/radial/RadialMeshFunction.cc @@ -98,7 +98,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int i = 0; i < nn[1]; i++) @@ -110,7 +110,7 @@ void RadialMeshFunction::bcast(MPI_Comm comm, const int root) { (*MPIdata::sout) << "RadialMeshFunction::bcast() failed!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } } } diff --git a/src/setup.cc b/src/setup.cc index 9963bfc5..2ff6d79e 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -140,7 +140,7 @@ int MGmol::setupLRsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { @@ -173,7 +173,7 @@ int MGmol::setupConstraintsFromInput(const std::string filename) if (!tfile->is_open()) { std::cerr << " Unable to open file " << filename << std::endl; - global_exit(0); + mmpi.abort(); } else { diff --git a/src/sparse_linear_algebra/DataDistribution.cc b/src/sparse_linear_algebra/DataDistribution.cc index d4b26d93..67bb1319 100644 --- a/src/sparse_linear_algebra/DataDistribution.cc +++ b/src/sparse_linear_algebra/DataDistribution.cc @@ -288,7 +288,7 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, std::cout << "ERROR: " << name_ << ", dir=" << dir << ", remote_size=" << remote_size << ", bsiz=" << bsiz << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } // string stamp="DataDistribution ("+name_+"), buffer size checked..."; // printWithTimeStamp(stamp,cout); @@ -302,14 +302,14 @@ void DataDistribution::distributeLocalDataWithCommOvlp(const int nsteps, if (mpircv != MPI_SUCCESS) { std::cout << "ERROR in MPI_Irecv, code=" << mpircv << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } int mpisnd = MPI_Isend(packed_buffer.sendBuffer(), siz, MPI_CHAR, dest, 0, cart_comm_, &request[1]); if (mpisnd != MPI_SUCCESS) { std::cout << "ERROR in MPI_Isend, code=" << mpisnd << std::endl; - MPI_Abort(cart_comm_, 0); + MPI_Abort(cart_comm_, EXIT_FAILURE); } /* wait to complete communication */ MPI_Waitall(2, request, MPI_STATUSES_IGNORE); diff --git a/src/tools.cc b/src/tools.cc index 95790c59..aff34933 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -29,7 +29,7 @@ void noMoreMemory() std::cerr << "Unable to satisfy request for memory for MPI task " << mype << std::endl; Control& ct = *(Control::instance()); - ct.global_exit(3); + ct.global_exit(); } // an atom name should start with a capital letter and end with a number @@ -229,12 +229,12 @@ void printWithTimeStamp(const std::string& string2print, std::ostream& os) if( mpierr!=MPI_SUCCESS ) { cerr << " Error in MPI!!!" << std::endl; - MPI_Abort(mmpi.commGlobal(),1); + MPI_Abort(mmpi.commGlobal(),EXIT_FAILURE); } if( r!=mmpi.size()*s && onpe0 ) { cerr << " Error in barrier: "<& vv, MPI_Comm comm) std::cerr << "ERROR!!!! bcastvv3d(), Failure in MPI_Bcast of 'radii_'!!!" << std::endl; - MPI_Abort(comm, 0); + MPI_Abort(comm, EXIT_FAILURE); } for (int j = 0; j < n; j++) for (short i = 0; i < 3; i++) From 8b84de49791659a4d874c617f71a920fb08e9138 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 24 Feb 2025 15:03:24 -0500 Subject: [PATCH 25/58] Rho and VH restart (#311) * enable restart with consistent rho and VHartree --------- Co-authored-by: Seung Whan Chung --- src/Electrostatic.h | 3 + src/MGmol.h | 4 +- src/Potentials.cc | 6 + src/Potentials.h | 2 + src/md.cc | 13 ++ tests/CMakeLists.txt | 15 ++ tests/RhoVhRestart/h2o.xyz | 6 + tests/RhoVhRestart/md.cfg | 30 ++++ tests/RhoVhRestart/mgmol.cfg | 28 ++++ tests/RhoVhRestart/restart.cfg | 25 +++ tests/RhoVhRestart/test.py | 79 +++++++++ tests/RhoVhRestart/testRhoVhRestart.cc | 220 +++++++++++++++++++++++++ 12 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 tests/RhoVhRestart/h2o.xyz create mode 100644 tests/RhoVhRestart/md.cfg create mode 100644 tests/RhoVhRestart/mgmol.cfg create mode 100644 tests/RhoVhRestart/restart.cfg create mode 100755 tests/RhoVhRestart/test.py create mode 100644 tests/RhoVhRestart/testRhoVhRestart.cc diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 9beee899..06a9e17c 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -47,6 +47,9 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + pb::GridFunc* getRhoc() { return grhoc_; } + Poisson* getPoissonSolver() { return poisson_solver_; } + void setup(const short max_sweeps); void setupPB(const double e0, const double rho0, const double drho0, Potentials& pot); diff --git a/src/MGmol.h b/src/MGmol.h index bccbc7af..b18465a1 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -150,7 +150,6 @@ class MGmol : public MGmolInterface void initialMasks(); int setupLRsFromInput(const std::string filename); - void setup(); int setupLRs(const std::string input_file) override; int setupFromInput(const std::string input_file) override; int setupConstraintsFromInput(const std::string input_file) override; @@ -177,12 +176,15 @@ class MGmol : public MGmolInterface ~MGmol() override; + void setup(); + /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } std::shared_ptr> getHamiltonian() { return hamiltonian_; } + std::shared_ptr> getRho() { return rho_; } void run() override; diff --git a/src/Potentials.cc b/src/Potentials.cc index ebe42320..96375c6d 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -939,6 +939,12 @@ int Potentials::read(HDFrestart& h5f_file) h5f_file.read_1func_hdf5(vepsilon_.data(), "VDielectric"); } + std::string datasetname("Preceding_Hartree"); + if (h5f_file.checkDataExists(datasetname)) + { + h5f_file.read_1func_hdf5(vh_rho_backup_.data(), datasetname); + } + return 0; } diff --git a/src/Potentials.h b/src/Potentials.h index cb0f8e05..8e9b763c 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -206,6 +206,8 @@ class Potentials */ void backupVh(); + void resetVhRho2Backup() { vh_rho_ = vh_rho_backup_; } + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/md.cc b/src/md.cc index 8ec20074..b1ffb619 100644 --- a/src/md.cc +++ b/src/md.cc @@ -702,6 +702,19 @@ void MGmol::loadRestartFile(const std::string filename) ierr = proj_matrices_->readWFDM(h5file); } + if (h5file.checkDataExists("Preceding_Hartree")) + { + ions_->readRestartPreviousPositions(h5file); + ions_->resetPositionsToPrevious(); + ions_->setup(); + + Potentials& pot = hamiltonian_->potential(); + pot.initialize(*ions_); + if (onpe0) std::cout << "Reset VhRho to backup..." << std::endl; + pot.resetVhRho2Backup(); + electrostat_->setupRhoc(pot.rho_comp()); + } + ierr = h5file.close(); mmpi.allreduce(&ierr, 1, MPI_MIN); if (ierr < 0) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index baf1c451..838ce6a4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -250,6 +250,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) +add_executable(testRhoVhRestart + ${CMAKE_SOURCE_DIR}/tests/RhoVhRestart/testRhoVhRestart.cc) add_executable(testEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/EnergyAndForces/testEnergyAndForces.cc) add_executable(testWFEnergyAndForces @@ -382,6 +384,17 @@ add_test(NAME testRestartEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testRhoVhRestart + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_BINARY_DIR}/testRhoVhRestart + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) + if(${MAGMA_FOUND}) add_test(NAME testOpenmpOffload @@ -577,6 +590,7 @@ target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) +target_include_directories(testRhoVhRestart PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} @@ -589,6 +603,7 @@ target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) +target_link_libraries(testRhoVhRestart mgmol_src) if(${MAGMA_FOUND}) target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} diff --git a/tests/RhoVhRestart/h2o.xyz b/tests/RhoVhRestart/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/RhoVhRestart/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/RhoVhRestart/md.cfg b/tests/RhoVhRestart/md.cfg new file mode 100644 index 00000000..2b8a378b --- /dev/null +++ b/tests/RhoVhRestart/md.cfg @@ -0,0 +1,30 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=5 +dt=40. +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF +output_level=4 +output_filename=WF_MD diff --git a/tests/RhoVhRestart/mgmol.cfg b/tests/RhoVhRestart/mgmol.cfg new file mode 100644 index 00000000..eee7f11c --- /dev/null +++ b/tests/RhoVhRestart/mgmol.cfg @@ -0,0 +1,28 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=120 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=1.5 +[Restart] +output_level=4 +output_filename=WF diff --git a/tests/RhoVhRestart/restart.cfg b/tests/RhoVhRestart/restart.cfg new file mode 100644 index 00000000..20f0293a --- /dev/null +++ b/tests/RhoVhRestart/restart.cfg @@ -0,0 +1,25 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=24 +atol=1.e-8 +[Restart] +input_level=4 +input_filename=WF_MD diff --git a/tests/RhoVhRestart/test.py b/tests/RhoVhRestart/test.py new file mode 100755 index 00000000..a34b962f --- /dev/null +++ b/tests/RhoVhRestart/test.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test test_rho_restart...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-7): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +mgmol_exe = sys.argv[nargs-7] +test_exe = sys.argv[nargs-6] +input1 = sys.argv[nargs-5] +input2 = sys.argv[nargs-4] +input3 = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +dst1 = 'pseudo.H_ONCV_PBE_SG15' +src1 = sys.argv[-1] + '/' + dst1 + +dst2 = 'pseudo.O_ONCV_PBE_SG15' +src2 = sys.argv[-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) + +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run mgmol to generate initial ground state +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input1,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("Initial quench failed to complete!") + sys.exit(1) + +#run MD +command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') + +flag=0 +for line in lines: + if line.count(b'Run ended'): + flag=1 + +if flag==0: + print("MD failed to complete!") + sys.exit(1) + +#run test +command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input3,coords) +print("Run command: {}".format(command)) +output = subprocess.check_output(command,shell=True) +lines=output.split(b'\n') +for line in lines: + print(line) + +print("Test SUCCESSFUL!") +sys.exit(0) diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc new file mode 100644 index 00000000..09da3320 --- /dev/null +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -0,0 +1,220 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "Electrostatic.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Poisson.h" +#include "Potentials.h" +#include "mgmol_run.h" + +#include +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +template +int testRhoRestart(MGmolInterface* mgmol_) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + std::shared_ptr> rho = mgmol->getRho(); + + /* save density from the restart file to elsewhere */ + std::vector rho0(rho->rho_[0].size()); + rho0 = rho->rho_[0]; + + /* recompute rho from the orbital */ + rho->update(*mgmol->getOrbitals()); + + /* check if the recomputed density is the same */ + for (int d = 0; d < (int)rho0.size(); d++) + { + double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); + if (error > 1e-10) + { + printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, + rho->rho_[0][d], d, rho0[d]); + std::cerr << "Density is inconsistent!!!" << std::endl; + return -1; + } + } + if (rank == 0) std::cout << "Density is consistent..." << std::endl; + + return 0; +} + +template +int testPotRestart(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + + MGmol* mgmol = static_cast*>(mgmol_); + Potentials& pot = mgmol->getHamiltonian()->potential(); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + std::shared_ptr> rho = mgmol->getRho(); + + /* GridFunc initialization inputs */ + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = ct.bcPoisson[d]; + + /* save potential from the restart file to elsewhere */ + pb::GridFunc vh0_gf(mygrid, bc[0], bc[1], bc[2]); + vh0_gf.assign((pot.vh_rho()).data(), 'd'); + double n = vh0_gf.norm2(); + std::cout << "Norm2 of vh = " << n << std::endl; + + std::vector vh0(pot.size()); + const std::vector& d_vhrho(pot.vh_rho()); + for (int d = 0; d < (int)vh0.size(); d++) + vh0[d] = d_vhrho[d]; + + /* recompute potential */ + pb::GridFunc grho(mygrid, bc[0], bc[1], bc[2]); + grho.assign(&rho->rho_[0][0]); + pb::GridFunc* grhoc = mgmol->electrostat_->getRhoc(); + + poisson->solve(grho, *grhoc); + const pb::GridFunc& vh(poisson->vh()); + + pb::GridFunc error_gf(vh0_gf); + error_gf -= vh; + + double rel_error = error_gf.norm2() / vh0_gf.norm2(); + if (rank == 0) + { + printf("FOM potential relative error: %.3e\n", rel_error); + } + if (rel_error > 1e-9) + { + if (rank == 0) + { + std::cerr << "Potential is inconsistent!!!" << std::endl; + } + return -1; + } + if (rank == 0) std::cout << "Potential is consistent..." << std::endl; + + return 0; +} + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + int status = 0; + + // Enter main scope + { + MGmolInterface* mgmol = new MGmol(global_comm, + *MPIdata::sout, input_filename, lrs_filename, constraints_filename); + + mgmol->setup(); + + /* load a restart file */ + MGmol* mgmol_ext + = dynamic_cast*>(mgmol); + mgmol_ext->loadRestartFile(ct.restart_file); + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testRhoRestart..." << std::endl; + status = testRhoRestart(mgmol); + if (status < 0) return status; + + if (MPIdata::onpe0) + std::cout << "=============================" << std::endl; + if (MPIdata::onpe0) std::cout << "testPotRestart..." << std::endl; + status = testPotRestart(mgmol); + if (status < 0) return status; + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} From 8003d70f5758f3cef4d74724f2b7d3a04afe7964 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Mar 2025 21:51:29 -0500 Subject: [PATCH 26/58] Strenghten testIons (#315) --- tests/testIons.cc | 76 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/tests/testIons.cc b/tests/testIons.cc index 30b2ad0c..6bfdd74a 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -41,7 +41,7 @@ int main(int argc, char** argv) // read species info from pseudopotential file std::string file_path = argv[1]; std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); - std::cout << "Potential = " << filename << std::endl; + if (myrank == 0) std::cout << "Potential = " << filename << std::endl; sp.read_1species(filename); sp.set_dim_nl(h[0]); @@ -80,13 +80,71 @@ int main(int argc, char** argv) ions.setup(); - std::vector& new_local_ions(ions.local_ions()); + // verify sum of local ions adds up to total number of ions + { + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + } + MPI_Barrier(MPI_COMM_WORLD); - int nlocal = new_local_ions.size(); - std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + // verify some functionalities of class Ions + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + ions.getAtomicNumbers(anumbers); + if (myrank == 0) + { + int i = 0; + for (auto& position : positions) + { + std::cout << position; + if (i % 3 == 2) + std::cout << std::endl; + else + std::cout << " "; + i++; + } + } + MPI_Barrier(MPI_COMM_WORLD); + + // swap x and z + for (size_t i = 0; i < positions.size() - 2; i++) + { + double x = positions[i]; + double z = positions[i + 2]; + positions[i] = z; + positions[i + 2] = x; + } + + ions.setPositions(positions, anumbers); + } - int ntotal = 0; - MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + MPI_Barrier(MPI_COMM_WORLD); + { + std::vector& new_local_ions(ions.local_ions()); + + int nlocal = new_local_ions.size(); + std::cout << "PE " << myrank << ", nlocal = " << nlocal << std::endl; + + int ntotal = 0; + MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + return 1; + } + } mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { @@ -94,11 +152,5 @@ int main(int argc, char** argv) return 1; } - if (ntotal != na) - { - std::cout << "ntotal = " << ntotal << std::endl; - return 1; - } - return 0; } From 4efb34c5e2e2283043faa4f8b612e195570806bd Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 3 Mar 2025 21:52:11 -0500 Subject: [PATCH 27/58] More clean up in class Potentials (#316) * remove some unused functions * make use of class MGmol_MPI * change some function names for clarity * fix some minor bug in iterative indexes --- src/MGmol.cc | 9 +++--- src/Potentials.cc | 70 ++++++++++++++--------------------------------- src/Potentials.h | 15 +++------- 3 files changed, 29 insertions(+), 65 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index 8d87f1c1..c18e58c7 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -400,7 +400,6 @@ int MGmol::initial() current_orbitals_->setDataWithGhosts(); current_orbitals_->trade_boundaries(); - // if(ct.restart_info <= 1)pot.initWithVnuc(); // initialize matrices S and invB if (ct.numst > 0) @@ -1379,14 +1378,14 @@ void MGmol::update_pot(const Ions& ions) const bool flag_mixing = (fabs(ct.mix_pot - 1.) > 1.e-3); - // evaluate potential correction + // update total potential if (flag_mixing) { - pot.delta_v(rho_->rho_); - pot.update(ct.mix_pot); + pot.computeDeltaV(rho_->rho_); + pot.updateVtot(ct.mix_pot); } else - pot.update(rho_->rho_); + pot.updateVtot(rho_->rho_); } template diff --git a/src/Potentials.cc b/src/Potentials.cc index 96375c6d..4da9ab9a 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -87,37 +87,19 @@ Potentials::Potentials() #endif } -void Potentials::initWithVnuc() -{ - assert(size_ > 0); - if (verbosity_level_ > 2 && onpe0) - (*MPIdata::sout) << "Potentials::initWithVnuc()" << std::endl; - itindex_vxc_ = 0; - itindex_vh_ = 0; - int ione = 1; - Tcopy(&size_, &v_nuc_[0], &ione, &vtot_[0], &ione); - double one = 1.; - LinearAlgebraUtils::MPaxpy( - size_, one, &v_ext_[0], &vtot_[0]); - // factor ha2ry to get total potential in [Ry] for calculations - LinearAlgebraUtils::MPscal(size_, ha2ry, &vtot_[0]); -} - double Potentials::max() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmax = (*max_element(vtot_.begin(), vtot_.end())); - vmax = myPEenv.double_max_all(vmax); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmax = (*max_element(vtot_.begin(), vtot_.end())); + vmax = mmpi.allreduce(&vmax, 1, MPI_MAX); return vmax; } double Potentials::min() const { - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); - double vmin = -(*min_element(vtot_.begin(), vtot_.end())); - vmin = -myPEenv.double_max_all(vmin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + double vmin = -(*min_element(vtot_.begin(), vtot_.end())); + vmin = -mmpi.allreduce(&vmin, 1, MPI_MAX); return vmin; } @@ -141,7 +123,7 @@ void Potentials::evalNormDeltaVtotRho( mmpi.allreduce(&scf_dvrho_, 1, MPI_SUM); } -double Potentials::update(const std::vector>& rho) +double Potentials::updateVtot(const std::vector>& rho) { assert(itindex_vxc_ >= 0); assert(itindex_vh_ >= 0); @@ -149,9 +131,8 @@ double Potentials::update(const std::vector>& rho) if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::update(rho)" << std::endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -181,23 +162,22 @@ double Potentials::update(const std::vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), EXIT_FAILURE); + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; } -void Potentials::update(const double mix) +void Potentials::updateVtot(const double mix) { assert(itindex_vxc_ == itindex_vh_); @@ -210,7 +190,7 @@ void Potentials::update(const double mix) size_, potmix, &dv_[0], &vtot_[0]); } -double Potentials::delta_v(const std::vector>& rho) +double Potentials::computeDeltaV(const std::vector>& rho) { assert(itindex_vxc_ == itindex_vh_); assert(size_ > 0); @@ -218,9 +198,8 @@ double Potentials::delta_v(const std::vector>& rho) if (verbosity_level_ > 2 && onpe0) (*MPIdata::sout) << "Potentials::delta_v()" << std::endl; - int ione = 1; - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); + int ione = 1; + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); // save old potentials Tcopy(&size_, &vtot_[0], &ione, &vtot_old_[0], &ione); @@ -249,17 +228,16 @@ double Potentials::delta_v(const std::vector>& rho) = LinearAlgebraUtils::MPdot(size_, &dv_[0], &dv_[0]); double sum = 0.; - int rc - = MPI_Allreduce(&dvdot, &sum, 1, MPI_DOUBLE, MPI_SUM, myPEenv.comm()); + int rc = mmpi.allreduce(&dvdot, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { std::cerr << "MPI_Allreduce double sum failed!!!" << std::endl; - MPI_Abort(myPEenv.comm(), EXIT_FAILURE); + mmpi.abort(); } dvdot = sum; scf_dv_ = 0.5 * sqrt(dvdot); - const double gsize = (double)size_ * (double)myPEenv.n_mpi_tasks(); + const double gsize = (double)size_ * (double)mmpi.size(); scf_dv_ /= gsize; return scf_dv_; @@ -437,7 +415,6 @@ void Potentials::readExternalPot(const std::string filename, const char type) { assert(index < size_); (*from) >> v_ext_[index]; - //(*MPIdata::sout)<& vrho) const; /*! * evaluate potential correction associated with a new rho */ - double delta_v(const std::vector>& rho); + double computeDeltaV(const std::vector>& rho); /*! - * update potentials based on argument rho + * update total potential with updated components */ - double update(const std::vector>& rho); + double updateVtot(const std::vector>& rho); /*! * update potentials based on potential correction delta v and mixing * parameter */ - void update(const double mix); + void updateVtot(const double mix); double max() const; double min() const; @@ -196,7 +190,6 @@ class Potentials template void setVxc(const T* const vxc, const int iterativeIndex); - void setVh(const POTDTYPE* const vh, const int iterativeIndex); void setVh(const pb::GridFunc& vh, const int iterativeIndex); void initialize(Ions& ions); From 2f511b775505a127869107d8c90a30cb212ad591 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 14 Mar 2025 17:41:39 -0400 Subject: [PATCH 28/58] Misc code fixes (#317) * exposed by porting on new environment --- src/HDFrestart.cc | 6 ++++++ src/Ions.cc | 2 ++ src/tools/SymmetricMatrix.h | 1 + 3 files changed, 9 insertions(+) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index 87982290..e2bfa1ab 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -1922,10 +1922,12 @@ int HDFrestart::readAtomicData(std::string datasetname, std::vector& data) // send data to inactive PEs if (gather_data_x_) gatherDataXdir(data); +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), -1), data.end()); } +#endif return 0; } @@ -1974,10 +1976,12 @@ int HDFrestart::readAtomicData( } } +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), 1e+32), data.end()); } +#endif if (gather_data_x_) gatherDataXdir(data); return 0; @@ -2104,10 +2108,12 @@ int HDFrestart::readAtomicData( data.push_back(t); } +#ifdef MGMOL_USE_HDF5P if (useHdf5p()) { data.erase(std::remove(data.begin(), data.end(), ""), data.end()); } +#endif return 0; } diff --git a/src/Ions.cc b/src/Ions.cc index 9af0a64e..bc30417b 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -947,10 +947,12 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) assert(at_numbers.size() == at_nlprojIds.size()); num_ions_ = at_names.size(); +#ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } +#endif if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ diff --git a/src/tools/SymmetricMatrix.h b/src/tools/SymmetricMatrix.h index bd887fb3..b7aece1e 100644 --- a/src/tools/SymmetricMatrix.h +++ b/src/tools/SymmetricMatrix.h @@ -14,6 +14,7 @@ #include #include #include +#include #include // full lower triangular part of symmetric matrix with compact storage From 45a2c80cd6199da0643adf787d04831650e57d07 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 14 Mar 2025 17:42:00 -0400 Subject: [PATCH 29/58] Use unique restart filenames in test HDF5single (#318) --- tests/HDF5single/md.cfg | 4 ++-- tests/HDF5single/mgmol.cfg | 2 +- tests/HDF5single/test.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/HDF5single/md.cfg b/tests/HDF5single/md.cfg index 1ff2adab..8d6b21e2 100644 --- a/tests/HDF5single/md.cfg +++ b/tests/HDF5single/md.cfg @@ -25,10 +25,10 @@ max_steps=24 atol=1.e-8 [Restart] input_level=4 -input_filename=WF +input_filename=wf.h5 input_type=single_file output_level=4 -output_filename=WF_MD +output_filename=wf_md.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/mgmol.cfg b/tests/HDF5single/mgmol.cfg index 4dba942a..e8f01a29 100644 --- a/tests/HDF5single/mgmol.cfg +++ b/tests/HDF5single/mgmol.cfg @@ -25,7 +25,7 @@ initial_type=Random initial_width=1.5 [Restart] output_level=4 -output_filename=WF +output_filename=wf.h5 output_type=single_file [Coloring] scope=global diff --git a/tests/HDF5single/test.py b/tests/HDF5single/test.py index 080ee0ba..e2f0596a 100755 --- a/tests/HDF5single/test.py +++ b/tests/HDF5single/test.py @@ -47,7 +47,7 @@ output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') -os.remove('WF') +os.remove('wf.h5') print("Check energy conservation...") tol = 1.e-4 @@ -71,5 +71,7 @@ print("ERROR needs 4 energy values for checking conservation!") sys.exit(1) +os.remove('wf_md.h5') + print("Test SUCCESSFUL!") sys.exit(0) From b6cf162eac41663473659d44bd04c087d56e5fdc Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Sat, 15 Mar 2025 09:12:08 -0400 Subject: [PATCH 30/58] Swap ions in existing test (#319) * strengthen testing by swapping ions in testRestartEnergyAndForces --- src/MGmol.h | 5 +++ src/MGmolInterface.h | 3 ++ tests/RestartEnergyAndForces/test.py | 6 ++++ .../testRestartEnergyAndForces.cc | 32 +++++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/src/MGmol.h b/src/MGmol.h index b18465a1..350e8511 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -341,6 +341,11 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + void setPositions(const std::vector& positions, + const std::vector& atnumbers) + { + ions_->setPositions(positions, atnumbers); + } /* * simply dump current state diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9a9bf8a6..dc54a8e7 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -40,6 +40,9 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual void setPositions(const std::vector& positions, + const std::vector& atnumbers) + = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/tests/RestartEnergyAndForces/test.py b/tests/RestartEnergyAndForces/test.py index b62d39f8..585722f0 100755 --- a/tests/RestartEnergyAndForces/test.py +++ b/tests/RestartEnergyAndForces/test.py @@ -64,7 +64,13 @@ shutil.rmtree('WF') test_energy=1.e18 +l=-1 for line in lines: + if line.count(b'Positions'): + l=0 + if l>=0 and l<4: + print(line) + l=l+1 if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index e323afbc..3027ab55 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,6 +124,8 @@ int main(int argc, char** argv) } } + mgmol->setPositions(positions, anumbers); + Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -165,6 +167,36 @@ int main(int argc, char** argv) projmatrices->setDMuniform(ct.getNelSpin(), 0); projmatrices->printDM(std::cout); + // swap H and O to make sure order of atoms in list does not matter + double x = positions[0]; + double y = positions[1]; + double z = positions[2]; + positions[0] = positions[3]; + positions[1] = positions[4]; + positions[2] = positions[5]; + positions[3] = x; + positions[4] = y; + positions[5] = z; + short tmp = anumbers[0]; + anumbers[0] = anumbers[1]; + anumbers[1] = tmp; + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + mgmol->setPositions(positions, anumbers); + // // evaluate energy and forces with wavefunctions just read // From 7fbfe133c1ebd4f0932c9464e4e512ef8228d8d3 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 17 Mar 2025 13:44:30 -0400 Subject: [PATCH 31/58] Code clean up (#320) * use more const * use initNuc() instead of moveVnuc() * remove incorrect assert * use MGmol_MPI in more cases * setup g_kbpsi_ inside initNuc() * rename initNuc() into setupPotentials() --- src/HDFrestart.cc | 1 - src/KBPsiMatrixSparse.cc | 45 +++++++++++++++++++--------------------- src/KBPsiMatrixSparse.h | 15 ++++++-------- src/MGmol.cc | 18 ++++++++-------- src/MGmol.h | 2 +- src/md.cc | 2 +- src/quench.cc | 1 - 7 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/HDFrestart.cc b/src/HDFrestart.cc index e2bfa1ab..1b2c7fb7 100644 --- a/src/HDFrestart.cc +++ b/src/HDFrestart.cc @@ -2104,7 +2104,6 @@ int HDFrestart::readAtomicData( stripLeadingAndTrailingBlanks(t); // std::cout<<"stripped name="< 0); data.push_back(t); } diff --git a/src/KBPsiMatrixSparse.cc b/src/KBPsiMatrixSparse.cc index 604e2f8e..8f5b3d49 100644 --- a/src/KBPsiMatrixSparse.cc +++ b/src/KBPsiMatrixSparse.cc @@ -11,16 +11,15 @@ #include "Control.h" #include "ExtendedGridOrbitals.h" -#include "Ions.h" #include "LocGridOrbitals.h" #include "MGmol_MPI.h" #include "Mesh.h" #include "ProjectedMatrices.h" -#include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" #include "SquareSubMatrix2DistMatrix.h" #include + #define Ry2Ha 0.5; Timer KBPsiMatrixSparse::global_sum_tm_("KBPsiMatrixSparse::global_sum"); @@ -133,7 +132,7 @@ void KBPsiMatrixSparse::globalSumKBpsi() // Loop over the ions with projectors overlapping with local subdomain // and evaluate for some state. template -void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, T& orbitals, const int first_color, const int nb_colors, const bool flag) { assert(first_color >= 0); @@ -189,7 +188,7 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, // Loop over the ions if (gid != -1) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement( *ion, gid, iloc, ppsi + color * ldsize, flag); @@ -211,8 +210,8 @@ void KBPsiMatrixSparse::computeKBpsi(Ions& ions, T& orbitals, compute_kbpsi_tm_.stop(); } -void KBPsiMatrixSparse::computeKBpsi( - Ions& ions, pb::GridFunc* phi, const int istate, const bool flag) +void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, + pb::GridFunc* phi, const int istate, const bool flag) { assert(lapop_ != nullptr); compute_kbpsi_tm_.start(); @@ -238,7 +237,7 @@ void KBPsiMatrixSparse::computeKBpsi( for (int iloc = 0; iloc < subdivx; iloc++) { // Loop over the ions - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { computeLocalElement(*ion, istate, iloc, ppsi, flag); } @@ -251,7 +250,7 @@ void KBPsiMatrixSparse::computeKBpsi( void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) { - for (auto ion : ions.overlappingNL_ions()) + for (const auto& ion : ions.overlappingNL_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -266,8 +265,8 @@ void KBPsiMatrixSparse::scaleWithKBcoeff(const Ions& ions) // loop over states to multiply kbpsi_[st][gid] and kbBpsi_[st][gid] // by coeff - (*kbpsimat_).scaleRow(gid, coeff); - if (lapop_) (*kbBpsimat_).scaleRow(gid, coeff); + kbpsimat_->scaleRow(gid, coeff); + if (lapop_) kbBpsimat_->scaleRow(gid, coeff); } } } @@ -454,7 +453,7 @@ SquareSubMatrix KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, Aij); } @@ -473,7 +472,7 @@ void KBPsiMatrixSparse::computeHvnlMatrix( // Loop over ions centered on current PE only // (distribution of work AND Hvnlij contributions) - for (auto ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { computeHvnlMatrix((KBPsiMatrixSparse*)kbpsi2, *ion, mat); } @@ -538,14 +537,14 @@ void KBPsiMatrixSparse::getPsiKBPsiSym( { // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { getPsiKBPsiSym(*ion, sm); } } template -void KBPsiMatrixSparse::computeAll(Ions& ions, T& orbitals) +void KBPsiMatrixSparse::computeAll(const Ions& ions, T& orbitals) { assert(count_proj_subdomain_ == ions.countProjectorsSubdomain()); @@ -603,7 +602,7 @@ double KBPsiMatrixSparse::getEvnl( double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -618,10 +617,9 @@ double KBPsiMatrixSparse::getEvnl( /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -637,7 +635,7 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, double trace = 0.0; // loop over all the ions // parallelization over ions by including only those centered in subdomain - for (auto& ion : ions.local_ions()) + for (const auto& ion : ions.local_ions()) { std::vector gids; ion->getGidsNLprojs(gids); @@ -652,10 +650,9 @@ double KBPsiMatrixSparse::getEvnl(const Ions& ions, /* gather trace result */ MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - MPI_Comm comm = mmpi.commSpin(); double evnl = 0.0; - MPI_Allreduce(&trace, &evnl, 1, MPI_DOUBLE, MPI_SUM, comm); + mmpi.allreduce(&trace, &evnl, 1, MPI_SUM); return evnl * Ry2Ha; } @@ -736,12 +733,12 @@ double KBPsiMatrixSparse::getTraceDM( return trace; } -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, LocGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, LocGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, LocGridOrbitals&); -template void KBPsiMatrixSparse::computeKBpsi(Ions& ions, +template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, ExtendedGridOrbitals& orbitals, const int first_color, const int nb_colors, const bool flag); -template void KBPsiMatrixSparse::computeAll(Ions&, ExtendedGridOrbitals&); +template void KBPsiMatrixSparse::computeAll(const Ions&, ExtendedGridOrbitals&); diff --git a/src/KBPsiMatrixSparse.h b/src/KBPsiMatrixSparse.h index 91f6c583..073fbddc 100644 --- a/src/KBPsiMatrixSparse.h +++ b/src/KBPsiMatrixSparse.h @@ -12,7 +12,9 @@ #include "DataDistribution.h" #include "DensityMatrixSparse.h" +#include "Ions.h" #include "KBPsiMatrixInterface.h" +#include "ProjectedMatricesSparse.h" #include "SquareSubMatrix.h" #include "VariableSizeMatrix.h" @@ -23,11 +25,6 @@ #include #include -class Ions; -class Ion; -class ProjectedMatricesInterface; -class ProjectedMatricesSparse; - class KBPsiMatrixSparse : public KBPsiMatrixInterface { static Timer global_sum_tm_; @@ -79,8 +76,8 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface void getPsiKBPsiSym(const Ions& ions, VariableSizeMatrix& sm); void getPsiKBPsiSym(const Ion& ion, VariableSizeMatrix& sm); template - void computeKBpsi(Ions& ions, OrbitalsType& orbitals, const int first_color, - const int nb_colors, const bool flag); + void computeKBpsi(const Ions& ions, OrbitalsType& orbitals, + const int first_color, const int nb_colors, const bool flag); void clearData(); public: @@ -105,7 +102,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface double getEvnl( const Ions& ions, ProjectedMatrices* proj_matrices); void computeKBpsi( - Ions& ions, pb::GridFunc*, const int, const bool flag); + const Ions& ions, pb::GridFunc*, const int, const bool flag); double getValIonState(const int gid, const int st) const { return (*kbpsimat_).get_value(gid, st); @@ -127,7 +124,7 @@ class KBPsiMatrixSparse : public KBPsiMatrixInterface const Ions& ions, ProjectedMatricesInterface* proj_matrices) const; template - void computeAll(Ions& ions, T& orbitals); + void computeAll(const Ions& ions, T& orbitals); void setup(const Ions& ions); double getTraceDM( diff --git a/src/MGmol.cc b/src/MGmol.cc index c18e58c7..8d516b24 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -358,8 +358,8 @@ int MGmol::initial() } // Initialize the nuclear local potential and the compensating charges - if (ct.verbose > 0) printWithTimeStamp("initNuc()...", os_); - initNuc(*ions_); + if (ct.verbose > 0) printWithTimeStamp("setupPotentials()...", os_); + setupPotentials(*ions_); // initialize Rho if (ct.verbose > 0) printWithTimeStamp("Initialize Rho...", os_); @@ -415,9 +415,6 @@ int MGmol::initial() current_orbitals_->checkCond(100000., ct.AtomsMove()); } - if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); - if (ct.restart_info == 0) { if (ct.verbose > 0) printWithTimeStamp("update_pot...", os_); @@ -817,7 +814,7 @@ double get_trilinval(const double xc, const double yc, const double zc, #endif template -void MGmol::initNuc(Ions& ions) +void MGmol::setupPotentials(Ions& ions) { init_nuc_tm_.start(); @@ -829,9 +826,12 @@ void MGmol::initNuc(Ions& ions) // initialize poentials based on ionic positions and their species pot.initialize(ions); + if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); + g_kbpsi_->setup(*ions_); + electrostat_->setupRhoc(pot.rho_comp()); - if (onpe0 && ct.verbose > 3) os_ << " initNuc done" << std::endl; + if (onpe0 && ct.verbose > 3) os_ << " setupPotentials done" << std::endl; init_nuc_tm_.stop(); } @@ -1428,7 +1428,7 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); @@ -1450,7 +1450,7 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, ions_->setPositions(tau, atnumbers); - moveVnuc(*ions_); + setupPotentials(*ions_); // initialize electronic density rho_->update(*dorbitals); diff --git a/src/MGmol.h b/src/MGmol.h index 350e8511..12197dfd 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -220,7 +220,7 @@ class MGmol : public MGmolInterface void getAtomicNumbers(std::vector& an); - void initNuc(Ions& ions); + void setupPotentials(Ions& ions); void initKBR(); void printEigAndOcc(); diff --git a/src/md.cc b/src/md.cc index b1ffb619..edccdb17 100644 --- a/src/md.cc +++ b/src/md.cc @@ -59,7 +59,7 @@ void MGmol::moveVnuc(Ions& ions) // Update items that change when the ionic coordinates change pot.axpVcompToVh(1.); - initNuc(ions); + setupPotentials(ions); pot.axpVcompToVh(-1.); proj_matrices_->setHiterativeIndex(-1, -1); diff --git a/src/quench.cc b/src/quench.cc index 59a16d50..073798f6 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -545,7 +545,6 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, // get actual indexes of stored functions const std::vector>& gids(orbitals.getOverlappingGids()); - g_kbpsi_->setup(*ions_); electrostat_->setup(ct.vh_its); rho_->setup(ct.getOrthoType(), gids); From 14eb43ff48c5f0eaf6ccad0e819536dcc3e08ee6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 21 Mar 2025 07:54:45 -0400 Subject: [PATCH 32/58] Added functionalities to set local forces (#321) --- src/Ions.cc | 39 +++++++++++++++++++++++++++++++++++++++ src/Ions.h | 9 +++++++++ tests/testIons.cc | 45 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index bc30417b..10baad10 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1288,6 +1288,29 @@ void Ions::writeForces(HDFrestart& h5f_file) } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& names) +{ + assert(forces.size() == 3 * names.size()); + + // loop over global list of forces and atom names + std::vector::const_iterator s = names.begin(); + for (auto it = forces.begin(); it != forces.end(); it += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + if (ion->compareName(*s)) + { + ion->set_force(0, *it); + ion->set_force(1, *(it + 1)); + ion->set_force(2, *(it + 2)); + } + } + s++; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { @@ -2187,6 +2210,22 @@ void Ions::getLocalPositions(std::vector& tau) const } } +void Ions::getLocalNames(std::vector& names) const +{ + for (auto& ion : local_ions_) + { + names.push_back(ion->name()); + } +} + +void Ions::getNames(std::vector& names) const +{ + for (auto& ion : list_ions_) + { + names.push_back(ion->name()); + } +} + void Ions::getPositions(std::vector& tau) { std::vector tau_local(3 * local_ions_.size()); diff --git a/src/Ions.h b/src/Ions.h index b03972d1..33b8982b 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -285,11 +285,20 @@ class Ions const std::vector& tau, const std::vector& anumbers); void getLocalPositions(std::vector& tau) const; + void getLocalNames(std::vector& names) const; + void getNames(std::vector& names) const; void getPositions(std::vector& tau); void getAtomicNumbers(std::vector& atnumbers); void getForces(std::vector& forces); void getLocalForces(std::vector& tau) const; + + /*! + * set forces for ions in local_ions_ based on names matching + */ + void setLocalForces(const std::vector& forces, + const std::vector& names); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/tests/testIons.cc b/tests/testIons.cc index 6bfdd74a..e23e0e24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -8,6 +8,8 @@ int main(int argc, char** argv) { + int status = 0; + int mpirc = MPI_Init(&argc, &argv); MPI_Comm comm = MPI_COMM_WORLD; @@ -92,7 +94,7 @@ int main(int argc, char** argv) if (ntotal != na) { std::cout << "ntotal = " << ntotal << std::endl; - return 1; + status = 1; } } MPI_Barrier(MPI_COMM_WORLD); @@ -141,16 +143,49 @@ int main(int argc, char** argv) MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); if (ntotal != na) { - std::cout << "ntotal = " << ntotal << std::endl; - return 1; + std::cerr << "ntotal = " << ntotal << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + + // get the names of all the ions + std::vector names; + ions.getNames(names); + if (myrank == 0) + for (auto& name : names) + std::cout << "name = " << name << std::endl; + if (names.size() != na) + { + std::cerr << "Incorrect count of names..." << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + MPI_Barrier(MPI_COMM_WORLD); + + std::vector forces(3 * na); + // arbitrary value + const double fval = 1.12; + for (auto& f : forces) + f = fval; + ions.setLocalForces(forces, names); + + int nlocal = ions.getNumLocIons(); + std::vector lforces(3 * nlocal); + ions.getLocalForces(lforces); + for (auto& f : lforces) + { + if (std::abs(f - fval) > 1.e-14) + { + std::cerr << "f = " << f << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { std::cerr << "MPI Finalize failed!!!" << std::endl; - return 1; + status = 1; } - return 0; + return status; } From c61b48d2dabcdbe291b1545eeba338c56889cc40 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 24 Mar 2025 13:35:03 -0400 Subject: [PATCH 33/58] Update mixed precision code (#322) * enable cmake build with mixed precision * enable use of BLIS * isolate blas3 loop rewrite in separate files * fix misc issues with mixed precision code --- CMakeLists.txt | 15 +- scripts/build_ubuntu22_mixedp.sh | 32 ++ src/BlockVector.cc | 4 +- src/ExtendedGridOrbitals.cc | 29 +- src/LocGridOrbitals.cc | 14 +- src/MGmol.cc | 2 + src/Rho.cc | 2 +- src/global.h | 2 +- src/linear_algebra/CMakeLists.txt | 2 +- src/linear_algebra/MGmol_blas1.h | 14 +- src/linear_algebra/gemm_impl.cc | 251 ++++++++++++++ src/linear_algebra/gemm_impl.h | 14 + src/linear_algebra/mputils.cc | 550 ++++++++---------------------- src/linear_algebra/mputils.h | 23 +- src/linear_algebra/syrk_impl.cc | 227 ++++++++++++ src/linear_algebra/syrk_impl.h | 13 + src/mgmol_run.cc | 8 + tests/CMakeLists.txt | 30 ++ 18 files changed, 768 insertions(+), 464 deletions(-) create mode 100755 scripts/build_ubuntu22_mixedp.sh create mode 100644 src/linear_algebra/gemm_impl.cc create mode 100644 src/linear_algebra/gemm_impl.h create mode 100644 src/linear_algebra/syrk_impl.cc create mode 100644 src/linear_algebra/syrk_impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 753a455d..e931b5cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,12 @@ else() find_package(OpenMP) endif() +# Use mixed precision +set(MGMOL_WITH_MP FALSE CACHE BOOL "Compile with mixed precision") +if(MGMOL_WITH_MP) + add_definitions(-DMGMOL_USE_MIXEDP) +endif() + # HDF5 (required) set(MGMOL_USE_HDF5P TRUE CACHE BOOL "Use HDF5 parallel capability") if(HDF5_LIBRARIES) #user sets libraries explicitly @@ -100,9 +106,16 @@ if(NOT BLAS_LIBRARIES) message(FATAL_ERROR "Required blas library not found.") endif(NOT ${BLAS_FOUND}) else(NOT BLAS_LIBRARIES) - message(STATUS "BLAS_LIBARIES: ${BLAS_LIBRARIES}") + message(STATUS "BLAS_LIBARIES set to ${BLAS_LIBRARIES}") endif(NOT BLAS_LIBRARIES) +set(MGMOL_WITH_BLIS FALSE CACHE BOOL "Use BLIS library") +if(MGMOL_WITH_BLIS) + add_definitions(-DMGMOL_USE_BLIS) + message(STATUS "BLIS_LIBRARIES include: ${BLIS_INCLUDE_DIRS}") + include_directories("${BLIS_INCLUDE_DIRS}") +endif() + if(NOT LAPACK_LIBRARIES) find_package(LAPACK REQUIRED) message(STATUS "LAPACK_LIBARIES: ${LAPACK_LIBRARIES}") diff --git a/scripts/build_ubuntu22_mixedp.sh b/scripts/build_ubuntu22_mixedp.sh new file mode 100755 index 00000000..d85ab9a5 --- /dev/null +++ b/scripts/build_ubuntu22_mixedp.sh @@ -0,0 +1,32 @@ +#/bin/bash +MGMOL_ROOT=`pwd` + +INSTALL_DIR=${MGMOL_ROOT}/mgmol_install +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_COMPILER=mpiCC.openmpi \ + -DMGMOL_WITH_BLIS=ON \ + -DBLAS_LIBRARIES=/home/q8j/blis/lib/libblis.so \ + -DBLIS_INCLUDE_DIRS="/home/q8j/blis//include" \ + -DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/openblas-pthread/liblapack.so \ + -DCMAKE_Fortran_COMPILER=mpif77.openmpi \ + -DSCALAPACK_LIBRARY=/usr/lib/x86_64-linux-gnu/libscalapack-openmpi.so.2.1 \ + -DMPIEXEC_EXECUTABLE=/usr/bin/mpirun \ + -DMPIEXEC_NUMPROC_FLAG="-np" \ + -DMPIEXEC_PREFLAGS="--oversubscribe" \ + -DMGMOL_WITH_CLANG_FORMAT=ON \ + -DMGMOL_WITH_MP=ON \ + -DCMAKE_PREFIX_PATH=${HOME}/bin \ + -D CMAKE_CXX_FLAGS="-Wall" \ + .. + +# call make install +make -j 4 +make install diff --git a/src/BlockVector.cc b/src/BlockVector.cc index 9a411ebe..671ba90d 100644 --- a/src/BlockVector.cc +++ b/src/BlockVector.cc @@ -556,7 +556,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); @@ -586,7 +586,7 @@ template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); template void BlockVector::setDataWithGhosts( pb::GridFuncVector* data_wghosts); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template class BlockVector; template void BlockVector::assign( const pb::GridFuncVector& src); diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index 07face7c..3e887dad 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -926,10 +926,10 @@ void ExtendedGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(numst_, nf, loc_numpt_, 1., - orbitals_psi_host_view + iloc * loc_numpt_, lda_, - work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * numst_, - numst_); + LinearAlgebraUtils::MPgemmTN(numst_, nf, + loc_numpt_, 1., orbitals_psi_host_view + iloc * loc_numpt_, + lda_, work + iloc * loc_numpt_, lda_, 0., + ssiloc + icolor * numst_, numst_); } } @@ -970,7 +970,7 @@ void ExtendedGridOrbitals::getLocalOverlap( if (numst_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else ORBDTYPE* psi = block_vector_.vect(0); @@ -1041,21 +1041,12 @@ void ExtendedGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, const int lda = transpose ? ld : lda_; const int ldb = transpose ? lda_ : ld; -#ifdef USE_MP - // use temporary float data for matrix ss - LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); -#else - LocalMatrices& ssf(ss); -#endif for (short iloc = 0; iloc < subdivx_; iloc++) { - LinearAlgebraUtils::MPgemm('T', 'N', numst_, numst_, + LinearAlgebraUtils::MPgemmTN(numst_, numst_, loc_numpt_, 1., a + iloc * loc_numpt_, lda, b + +iloc * loc_numpt_, - ldb, 0., ssf.getRawPtr(iloc), ssf.m()); + ldb, 0., ss.getRawPtr(iloc), ss.m()); } -#ifdef USE_MP - ss.copy(ssf); -#endif ss.scal(grid_.vel()); } @@ -1709,9 +1700,9 @@ void ExtendedGridOrbitals::addDotWithNcol2Matrix( Apsi.getPsi(0, iloc), phi_size, phi_host_view); // TODO this can be done on the GPU - MPgemmTN(numst_, numst_, loc_numpt_, vel, - block_vector_host_view + iloc * loc_numpt_, lda_, phi_host_view, - lda_, 1., work.data(), numst_); + LinearAlgebraUtils::MPgemmTN(numst_, numst_, + loc_numpt_, vel, block_vector_host_view + iloc * loc_numpt_, lda_, + phi_host_view, lda_, 1., work.data(), numst_); MemorySpace::Memory::free_host_view( phi_host_view); diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index 4a1b9442..3d14e051 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -1448,8 +1448,8 @@ void LocGridOrbitals::computeMatB( MATDTYPE* ssiloc = ss.getRawPtr(iloc); // calculate nf columns of ssiloc - MPgemmTN(chromatic_number_, nf, loc_numpt_, 1., - orbitals_psi + iloc * loc_numpt_, lda_, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + nf, loc_numpt_, 1., orbitals_psi + iloc * loc_numpt_, lda_, work + iloc * loc_numpt_, lda_, 0., ssiloc + icolor * chromatic_number_, chromatic_number_); } @@ -1490,7 +1490,7 @@ void LocGridOrbitals::getLocalOverlap( if (chromatic_number_ != 0) { -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP getLocalOverlap(*this, ss); #else const ORBDTYPE* const psi = block_vector_.vect(0); @@ -1560,7 +1560,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, MemorySpace::Memory::copy_view_to_host( const_cast(b), b_size, b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP // use temporary float data for matrix ss LocalMatrices ssf(ss.nmat(), ss.m(), ss.n()); #else @@ -1575,7 +1575,7 @@ void LocGridOrbitals::computeLocalProduct(const ORBDTYPE* const array, a_host_view); MemorySpace::Memory::free_host_view( b_host_view); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP ss.copy(ssf); #endif @@ -2495,8 +2495,8 @@ void LocGridOrbitals::addDotWithNcol2Matrix( { MATDTYPE* ssiloc = ss.getRawPtr(iloc); - // TODO - MPgemmTN(chromatic_number_, chromatic_number_, loc_numpt_, vel, + LinearAlgebraUtils::MPgemmTN(chromatic_number_, + chromatic_number_, loc_numpt_, vel, block_vector_.vect(0) + iloc * loc_numpt_, lda_, Apsi.getPsi(0, iloc), lda_, 0., ssiloc, chromatic_number_); } diff --git a/src/MGmol.cc b/src/MGmol.cc index 8d516b24..4d69387d 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -87,6 +87,7 @@ extern Timer sgemm_tm; extern Timer dgemm_tm; extern Timer mpgemm_tm; extern Timer tttgemm_tm; +extern Timer bligemm_tm; extern Timer dsyrk_tm; extern Timer ssyrk_tm; extern Timer mpsyrk_tm; @@ -864,6 +865,7 @@ void MGmol::printTimers() dgemm_tm.print(os_); mpgemm_tm.print(os_); tttgemm_tm.print(os_); + bligemm_tm.print(os_); ssyrk_tm.print(os_); dsyrk_tm.print(os_); diff --git a/src/Rho.cc b/src/Rho.cc index 45a41f3e..e80e74fe 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -588,7 +588,7 @@ Rho::computeRho>( ExtendedGridOrbitals&, const dist_matrix::DistMatrix&); template void Rho::computeRho>( LocGridOrbitals&, const dist_matrix::DistMatrix&); -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP template double Rho::dotWithRho( const float* const func) const; #endif diff --git a/src/global.h b/src/global.h index de6c8559..a23e4e3b 100644 --- a/src/global.h +++ b/src/global.h @@ -15,7 +15,7 @@ //#include "mgmol_memory.h" -#ifdef USE_MP +#ifdef MGMOL_USE_MIXEDP typedef float ORBDTYPE; #else typedef double ORBDTYPE; diff --git a/src/linear_algebra/CMakeLists.txt b/src/linear_algebra/CMakeLists.txt index 8bd5ef2f..0178f539 100644 --- a/src/linear_algebra/CMakeLists.txt +++ b/src/linear_algebra/CMakeLists.txt @@ -1,4 +1,4 @@ -set(SOURCES mputils.cc) +set(SOURCES mputils.cc gemm_impl.cc syrk_impl.cc) add_library(mgmol_linear_algebra ${SOURCES}) target_link_libraries(mgmol_linear_algebra PUBLIC MPI::MPI_CXX) if(${MGMOL_WITH_MAGMA}) diff --git a/src/linear_algebra/MGmol_blas1.h b/src/linear_algebra/MGmol_blas1.h index 41204226..32431468 100644 --- a/src/linear_algebra/MGmol_blas1.h +++ b/src/linear_algebra/MGmol_blas1.h @@ -11,6 +11,9 @@ #define MGMOL_MYBLAS1_H #include "fc_mangle.h" +#ifdef MGMOL_USE_BLIS +#include +#endif #include #include @@ -18,6 +21,8 @@ #define MY_VERSION 0 #define EPSILON 1.e-12 +#ifndef MGMOL_USE_BLIS + #ifdef __cplusplus extern "C" { @@ -42,12 +47,17 @@ extern "C" float SNRM2(const int* const, const float* const, const int* const); int IDAMAX(const int* const, const double* const, const int* const); int ISAMAX(const int* const, const float* const, const int* const); - void DROT(int*, double*, int*, double*, int*, double*, double*); - void SROT(int*, float*, int*, float*, int*, float*, float*); + void DROT(const int*, double*, const int*, double*, const int*, + const double*, const double*); + void SROT(const int*, float*, const int*, float*, const int*, const float*, + const float*); + #ifdef __cplusplus } #endif +#endif // MGMOL_USE_BLIS + inline void my_daxpy( const int n, const double alpha, const double* const a, double* b) { diff --git a/src/linear_algebra/gemm_impl.cc b/src/linear_algebra/gemm_impl.cc new file mode 100644 index 00000000..3500410c --- /dev/null +++ b/src/linear_algebra/gemm_impl.cc @@ -0,0 +1,251 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "gemm_impl.h" +#include "Timer.h" +#include "mputils.h" + +#include +#include + +Timer mpgemm_tm("mpgemm"); +Timer tttgemm_tm("tttgemm"); + +using LAU_H = LinearAlgebraUtils; + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) +{ + tttgemm_tm.start(); + // std::cout<<"template MPgemm..."< buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const T2* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = static_cast(c[pos]) * beta; + const T1* __restrict__ ai = a + lda * i; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + T3* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + { + cj[i] += (T3)buff[i]; + } + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const T1* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (T3)sum; + } + } + } + } + + tttgemm_tm.stop(); +} + +// input/output in float, computation in double +template <> +void gemm_impl(const char transa, const char transb, + const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const float* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + mpgemm_tm.start(); + + if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; + + /* case transb == 'N' and transa == 'N' */ + if (transb == 'N' || transb == 'n') + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * j + l]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ bj = b + ldb * j; + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + double bc = (double)c[pos] * beta; + const float* __restrict__ ai = a + lda * i; + c[pos] = (float)(alpha * LAU_H::MPdot(k, ai, bj) + bc); + } + } + } + } + else /* transb == 'T'/'C' */ + { + if (transa == 'N' || transa == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(m); + for (int j = 0; j < n; j++) + { + std::fill(buff.begin(), buff.end(), 0); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = (double)(alpha * b[ldb * l + j]); + LAU_H::MPaxpy(m, mult, colL, buff.data()); + } + /* Update col j of of result matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(m, beta, cj); + for (int i = 0; i < m; i++) + cj[i] += (float)buff[i]; + } + } + else /* transa == 'T'/'C' */ + { + for (int j = 0; j < n; j++) + { + for (int i = 0; i < m; i++) + { + const int pos = ldc * j + i; + const float* ai = a + lda * i; + double sum = 0.; + for (int l = 0; l < k; l++) + { + sum += alpha * ai[l] * b[ldb * l + j]; + } + sum += (double)(beta * c[pos]); + c[pos] = (float)sum; + } + } + } + } + + mpgemm_tm.stop(); +} + +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const double* const a, const int lda, + const double* const b, const int ldb, const double beta, float* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, double* const c, + const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, + const int ldc); diff --git a/src/linear_algebra/gemm_impl.h b/src/linear_algebra/gemm_impl.h new file mode 100644 index 00000000..5140b143 --- /dev/null +++ b/src/linear_algebra/gemm_impl.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void gemm_impl(const char transa, const char transb, const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc); diff --git a/src/linear_algebra/mputils.cc b/src/linear_algebra/mputils.cc index 77b08689..487d1367 100644 --- a/src/linear_algebra/mputils.cc +++ b/src/linear_algebra/mputils.cc @@ -16,19 +16,27 @@ #include #endif +#include "MGmol_blas1.h" +#include "gemm_impl.h" +#include "syrk_impl.h" + +#ifdef MGMOL_USE_BLIS +#include +#else +#include "blas2_c.h" +#include "blas3_c.h" +#endif + #include #include #include Timer dgemm_tm("dgemm"); Timer sgemm_tm("sgemm"); -Timer mpgemm_tm("mpgemm"); -Timer tttgemm_tm("tttgemm"); +Timer bligemm_tm("bligemm"); Timer dsyrk_tm("dsyrk"); Timer ssyrk_tm("ssyrk"); -Timer mpsyrk_tm("mpsyrk"); -Timer tttsyrk_tm("tttsyrk"); Timer mpdot_tm("mpdot"); Timer ttdot_tm("ttdot"); @@ -304,106 +312,6 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, dsyrk_tm.stop(); } -template <> -void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, - const double alpha, const float* const a, const int lda, const double beta, - float* c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(c); - - mpsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += static_cast(buff[i]); - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const float* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] = static_cast( - alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - mpsyrk_tm.stop(); -} - template <> template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, @@ -413,97 +321,7 @@ void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, MemorySpace::assert_is_host_ptr(a); MemorySpace::assert_is_host_ptr(c); - tttsyrk_tm.start(); - - if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; - - /* case Trans == 'N' */ - if (trans == 'N' || trans == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(n); - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const int len = j + 1; - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = static_cast( - alpha * colL[j]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const int len = n - (j + 1); - std::fill(buff.begin(), buff.begin() + len, 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l + j; - /* get multiplier */ - double mult = static_cast( - alpha * colL[0]); // same as alpha * a[lda*l + j]; - LAU_H::MPaxpy(len, mult, colL, buff.data()); - } - /* Update col j of upper part of matrix C. */ - /* Get pointer to beginning of column j in C. */ - T2* cj = c + ldc * j + j; - LAU_H::MPscal(len, beta, cj); - for (int i = 0; i < len; i++) - cj[i] += (T2)buff[i]; - } - } - } - else /* Trans == 'T' or 'C' */ - { - if (uplo == 'U' || uplo == 'u') - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = 0; i < j; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - else /* uplo = 'L' or 'l' */ - { - for (int j = 0; j < n; j++) - { - const T1* __restrict__ aj = a + lda * j; - for (int i = j; i < n; i++) - { - const int pos = ldc * j + i; - const T1* __restrict__ ai = a + lda * i; - double bc = static_cast(c[pos]) * beta; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); - } - } - } - } - - tttsyrk_tm.stop(); + syrk_impl(uplo, trans, n, k, alpha, a, lda, beta, c, ldc); } // MemorySpace::Device @@ -725,103 +543,7 @@ void LAU_H::MPgemm(const char transa, const char transb, const int m, MemorySpace::assert_is_host_ptr(b); MemorySpace::assert_is_host_ptr(c); - tttgemm_tm.start(); - // if(onpe0)cout<<"template MPgemm..."< buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const T2* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = static_cast(c[pos]) * beta; - const T1* __restrict__ ai = a + lda * i; - c[pos] - = static_cast(alpha * LAU_H::MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0.); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const T1* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - T3* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - { - cj[i] += (T3)buff[i]; - } - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const T1* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (T3)sum; - } - } - } - } - - tttgemm_tm.stop(); + gemm_impl(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } // input/output in double, computation in double @@ -842,111 +564,6 @@ void LAU_H::MPgemm(const char transa, const char transb, dgemm_tm.stop(); } -// input/output in float, computation in double -template <> -template <> -void LAU_H::MPgemm(const char transa, const char transb, - const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc) -{ - MemorySpace::assert_is_host_ptr(a); - MemorySpace::assert_is_host_ptr(b); - MemorySpace::assert_is_host_ptr(c); - - mpgemm_tm.start(); - - if (beta == 1. && (alpha == 0. || m == 0 || n == 0 || k == 0)) return; - - /* case transb == 'N' and transa == 'N' */ - if (transb == 'N' || transb == 'n') - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * j + l]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - const float* __restrict__ bj = b + ldb * j; - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - double bc = (double)c[pos] * beta; - const float* __restrict__ ai = a + lda * i; - c[pos] = (float)(alpha * MPdot(k, ai, bj) + bc); - } - } - } - } - else /* transb == 'T'/'C' */ - { - if (transa == 'N' || transa == 'n') - { - /* buffer to hold accumulation in double */ - std::vector buff(m); - for (int j = 0; j < n; j++) - { - std::fill(buff.begin(), buff.end(), 0); - for (int l = 0; l < k; l++) - { - /* pointer to beginning of column l in matrix a */ - const float* colL = a + lda * l; - /* get multiplier */ - double mult = (double)(alpha * b[ldb * l + j]); - LAU_H::MPaxpy(m, mult, colL, buff.data()); - } - /* Update col j of of result matrix C. */ - /* Get pointer to beginning of column j in C. */ - float* cj = c + ldc * j; - LAU_H::MPscal(m, beta, cj); - for (int i = 0; i < m; i++) - cj[i] += (float)buff[i]; - } - } - else /* transa == 'T'/'C' */ - { - for (int j = 0; j < n; j++) - { - for (int i = 0; i < m; i++) - { - const int pos = ldc * j + i; - const float* ai = a + lda * i; - double sum = 0.; - for (int l = 0; l < k; l++) - { - sum += alpha * ai[l] * b[ldb * l + j]; - } - sum += (double)(beta * c[pos]); - c[pos] = (float)sum; - } - } - } - } - - mpgemm_tm.stop(); -} - // MemorySpace::Device #ifdef HAVE_MAGMA template <> @@ -1014,10 +631,6 @@ void LAU_D::MPgemm(const char transa, const char transb, const int m, } #endif -/////////////////////////////// -// MPgemmNN // -/////////////////////////////// - template template void LinearAlgebraUtils::MPgemmNN(const int m, const int n, @@ -1031,19 +644,122 @@ void LinearAlgebraUtils::MPgemmNN(const int m, const int n, transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } +template <> +template <> +void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, + const float* const a, const int lda, const double* const b, const int ldb, + const double beta, float* const c, const int ldc) +{ + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, k, const_cast(a), 1, lda, &A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, k, n, const_cast(b), 1, ldb, &B); + + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else + char transa = 'n'; + char transb = 'n'; + + LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif +} + // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc) { + // std::cout << "LAU_H::MPgemmTN" << std::endl; + MemorySpace::assert_is_host_ptr(a); + MemorySpace::assert_is_host_ptr(b); + MemorySpace::assert_is_host_ptr(c); + +#ifdef MGMOL_USE_BLIS + bligemm_tm.start(); + + // Create matrix objects + // When storing by columns, the row stride is 1 + // When storing by columns, the column stride is also sometimes called the + // leading dimension + obj_t A; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, m, const_cast(a), 1, lda, &A); + bli_obj_toggle_trans(&A); + + obj_t B; + bli_obj_create_with_attached_buffer( + BLIS_FLOAT, k, n, const_cast(b), 1, ldb, &B); + obj_t C; + bli_obj_create_with_attached_buffer( + BLIS_DOUBLE, m, n, const_cast(c), 1, ldc, &C); + + obj_t bli_alpha; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_alpha); + bli_setsc(alpha, 0., &bli_alpha); + + obj_t bli_beta; + bli_obj_create_1x1(BLIS_DOUBLE, &bli_beta); + bli_setsc(beta, 0., &bli_beta); + + // accumulate results in double precision + // dafault: precision of C + bli_obj_set_comp_prec(BLIS_DOUBLE_PREC, &C); + bli_gemm(&bli_alpha, &A, &B, &bli_beta, &C); + + // Clean up BLIS objects + bli_obj_free(&bli_alpha); + bli_obj_free(&bli_beta); + + bligemm_tm.stop(); +#else char transa = 't'; char transb = 'n'; LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +#endif } // input in float, computation in double -void MPgemmTN(const int m, const int n, const int k, const double alpha, +template <> +template <> +void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, float* const c, const int ldc) { @@ -1053,12 +769,12 @@ void MPgemmTN(const int m, const int n, const int k, const double alpha, LAU_H::MPgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); } -/////// additional calls ... may be removed later if unused - +template template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc) +void LinearAlgebraUtils::MPgemmTN(const int m, const int n, + const int k, const double alpha, const T1* const a, const int lda, + const T2* const b, const int ldb, const double beta, T3* const c, + const int ldc) { // if(onpe0)cout<<"template MPgemmNN..."<(const char transa, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc); -template void LAU_H::MPgemmNN(const int m, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double* const b, const int ldb, const double beta, float* const c, +template void LAU_H::MPgemm(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, const int ldc); + +// template void LAU_H::MPgemmNN(const int m, const int n, +// const int k, const double alpha, const float* const a, const int lda, +// const double* const b, const int ldb, const double beta, float* const c, +// const int ldc); template void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, @@ -1127,14 +849,18 @@ template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, double* __restrict__ yptr); template void LAU_H::MPaxpy(const int len, const double scal, const float* __restrict__ xptr, float* __restrict__ yptr); + template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, float* c, const int ldc); template void LAU_H::MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const float* const a, const int lda, const double beta, double* c, const int ldc); +template void LAU_H::MPsyrk(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, float* c, const int ldc); -template void MPgemmTN(const int m, const int n, +template void LAU_H::MPgemmTN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, const int ldc); diff --git a/src/linear_algebra/mputils.h b/src/linear_algebra/mputils.h index 17bd0b4b..fa838f17 100644 --- a/src/linear_algebra/mputils.h +++ b/src/linear_algebra/mputils.h @@ -11,8 +11,6 @@ #define MGMOL_MPUTILS_H #include "MGmol_blas1.h" -#include "blas2_c.h" -#include "blas3_c.h" #include "memory_space.h" /* scal */ @@ -100,6 +98,11 @@ struct LinearAlgebraUtils const double alpha, const T1* const a, const int lda, const T2* const b, const int ldb, const double beta, T3* const c, const int ldc); + template + static void MPgemmTN(const int m, const int n, const int k, + const double alpha, const T1* const a, const int lda, const T2* const b, + const int ldb, const double beta, T3* const c, const int ldc); + /* mixed-precision scalar times vector. Accumulates results * in double precision and stores as single precision. */ @@ -125,28 +128,12 @@ struct LinearAlgebraUtils static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const double* const a, const int lda, const double beta, double* c, const int ldc); - static void MPsyrk(const char uplo, const char trans, const int n, - const int k, const double alpha, const float* const a, const int lda, - const double beta, float* c, const int ldc); template static void MPsyrk(const char uplo, const char trans, const int n, const int k, const double alpha, const T1* const a, const int lda, const double beta, T2* c, const int ldc); }; -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, double* const c, const int ldc); - -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const float* const a, const int lda, const float* const b, const int ldb, - const double beta, float* const c, const int ldc); - -template -void MPgemmTN(const int m, const int n, const int k, const double alpha, - const T1* const a, const int lda, const T2* const b, const int ldb, - const double beta, T3* const c, const int ldc); - /* trsm */ void Ttrsm(const char, const char, const char, const char, const int, const int, const double, const double* const, const int, double* const, const int); diff --git a/src/linear_algebra/syrk_impl.cc b/src/linear_algebra/syrk_impl.cc new file mode 100644 index 00000000..e1a4833f --- /dev/null +++ b/src/linear_algebra/syrk_impl.cc @@ -0,0 +1,227 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Timer.h" +#include "mputils.h" + +#include "MGmol_blas1.h" +#include "syrk_impl.h" + +#include "blas3_c.h" + +#include +#include +#include + +Timer mpsyrk_tm("mpsyrk"); +Timer tttsyrk_tm("tttsyrk"); + +using LAU_H = LinearAlgebraUtils; + +template <> +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const float* const a, const int lda, const double beta, + float* c, const int ldc) +{ + mpsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const float* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + float* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += static_cast(buff[i]); + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const float* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const float* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] = static_cast( + alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + mpsyrk_tm.stop(); +} + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc) +{ + tttsyrk_tm.start(); + + if (beta == 1. && (alpha == 0. || n == 0 || k == 0)) return; + + /* case Trans == 'N' */ + if (trans == 'N' || trans == 'n') + { + /* buffer to hold accumulation in double */ + std::vector buff(n); + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const int len = j + 1; + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l; + /* get multiplier */ + double mult = static_cast( + alpha * colL[j]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const int len = n - (j + 1); + std::fill(buff.begin(), buff.begin() + len, 0.); + for (int l = 0; l < k; l++) + { + /* pointer to beginning of column l in matrix a */ + const T1* colL = a + lda * l + j; + /* get multiplier */ + double mult = static_cast( + alpha * colL[0]); // same as alpha * a[lda*l + j]; + LAU_H::MPaxpy(len, mult, colL, buff.data()); + } + /* Update col j of upper part of matrix C. */ + /* Get pointer to beginning of column j in C. */ + T2* cj = c + ldc * j + j; + LAU_H::MPscal(len, beta, cj); + for (int i = 0; i < len; i++) + cj[i] += (T2)buff[i]; + } + } + } + else /* Trans == 'T' or 'C' */ + { + if (uplo == 'U' || uplo == 'u') + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = 0; i < j; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + else /* uplo = 'L' or 'l' */ + { + for (int j = 0; j < n; j++) + { + const T1* __restrict__ aj = a + lda * j; + for (int i = j; i < n; i++) + { + const int pos = ldc * j + i; + const T1* __restrict__ ai = a + lda * i; + double bc = static_cast(c[pos]) * beta; + c[pos] + = static_cast(alpha * LAU_H::MPdot(k, ai, aj) + bc); + } + } + } + } + + tttsyrk_tm.stop(); +} + +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const double* const a, + const int lda, const double beta, float* c, const int ldc); +template void syrk_impl(const char uplo, const char trans, + const int n, const int k, const double alpha, const float* const a, + const int lda, const double beta, double* c, const int ldc); diff --git a/src/linear_algebra/syrk_impl.h b/src/linear_algebra/syrk_impl.h new file mode 100644 index 00000000..ab3786f4 --- /dev/null +++ b/src/linear_algebra/syrk_impl.h @@ -0,0 +1,13 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +template +void syrk_impl(const char uplo, const char trans, const int n, const int k, + const double alpha, const T1* const a, const int lda, const double beta, + T2* c, const int ldc); diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index 468d61bd..92390e83 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -20,6 +20,10 @@ #include #include +#ifdef MGMOL_USE_BLIS +#include +#endif + #ifdef _OPENMP #include #endif @@ -39,6 +43,10 @@ int mgmol_init(MPI_Comm comm) assert(mype > -1); MPIdata::onpe0 = (MPIdata::mype == 0); +#ifdef MGMOL_USE_BLIS + bli_init(); +#endif + #ifdef HAVE_MAGMA magma_int_t magmalog = magma_init(); if (magmalog == MAGMA_SUCCESS) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 838ce6a4..df774171 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,8 @@ add_executable(testDistVector ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -32,6 +34,8 @@ add_executable(testReplicated2DistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -43,6 +47,8 @@ add_executable(testDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -54,6 +60,8 @@ add_executable(testConditionDistMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -68,6 +76,8 @@ add_executable(testConditionDistMatrixPower ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -79,6 +89,8 @@ add_executable(testPower ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -93,6 +105,8 @@ add_executable(testPowerDistMatrix ${CMAKE_SOURCE_DIR}/src/local_matrices/LocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/local_matrices/SquareLocalMatrices.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrix.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc @@ -123,6 +137,8 @@ add_executable(testVariableSizeMatrix ${CMAKE_SOURCE_DIR}/src/sparse_linear_algebra/Table.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -138,6 +154,8 @@ add_executable(testTradeGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -153,6 +171,8 @@ add_executable(testSetGhostValues ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -172,6 +192,8 @@ add_executable(testLaph4 ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) @@ -189,6 +211,8 @@ add_executable(testBatchLaph4 ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -204,6 +228,8 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/pb/GridFuncVector.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc @@ -225,6 +251,8 @@ add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc @@ -244,6 +272,8 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/DistMatrix/DistMatrixTools.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc + ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc From e37983b9a11a5ffca035254f0348fcbc246a5a75 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 09:46:33 -0400 Subject: [PATCH 34/58] Clean up and fixes Ions (#326) * reset static index counter ion Ion when needed * simplify some loops * make Ions copy constructor private --- src/Ion.cc | 2 + src/Ion.h | 2 + src/Ions.cc | 109 +++++++++++++++++----------------------------- src/Ions.h | 31 +++++++------ tests/testIons.cc | 4 ++ 5 files changed, 66 insertions(+), 82 deletions(-) diff --git a/src/Ion.cc b/src/Ion.cc index 90350477..896a1961 100644 --- a/src/Ion.cc +++ b/src/Ion.cc @@ -24,6 +24,8 @@ unsigned short isqrt(unsigned value) static unsigned int _nlproj_gid = 0; static unsigned int _index = 0; +void Ion::resetIndexCount() { _index = 0; } + Ion::Ion(const Species& species, const std::string& name, const double crds[3], const double velocity[3], const bool lock) : name_(name), species_(species), index_(_index), nlproj_gid_(_nlproj_gid) diff --git a/src/Ion.h b/src/Ion.h index 871205a3..cf52db97 100644 --- a/src/Ion.h +++ b/src/Ion.h @@ -314,6 +314,8 @@ class Ion void getKBcoeffs(std::vector& coeffs); double energyDiff( Ion& ion, const double lattice[3], const short bc[3]) const; + + static void resetIndexCount(); }; #endif diff --git a/src/Ions.cc b/src/Ions.cc index 10baad10..f10fb834 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -9,7 +9,6 @@ #include "Ions.h" #include "Control.h" -#include "HDFrestart.h" #include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "MPIdata.h" @@ -20,13 +19,10 @@ #include "mgmol_mpi_tools.h" #include "tools.h" -#include - #include #include #include -#include -#include +#include Timer ions_setupInteractingIons_tm("ions_setupInteractingIons"); Timer ions_setup_tm("ions::setup"); @@ -187,6 +183,8 @@ void Ions::setup() ions_setup_tm.start(); + Ion::resetIndexCount(); + updateListIons(); //#ifndef NDEBUG @@ -394,7 +392,7 @@ void Ions::iiforce(const short bc[3]) std::vector::const_iterator ion1 = local_ions_.begin(); int ion1_index = 0; - ; + while (ion1 != local_ions_.end()) { const double z1 = (*ion1)->getZion(); @@ -1800,7 +1798,7 @@ int Ions::readAtomsFromXYZ( } int Ions::setAtoms( - const std::vector& crds, const std::vector& spec) + const std::vector& crds, const std::vector& anum) { const int natoms = crds.size() / 3; @@ -1813,7 +1811,7 @@ int Ions::setAtoms( while (it != species_.end()) { ++isp; - if (it->getAtomicNumber() == spec[ia]) + if (it->getAtomicNumber() == anum[ia]) { break; } @@ -1823,7 +1821,7 @@ int Ions::setAtoms( for (std::map::iterator itr = map_species_.begin(); itr != map_species_.end(); ++itr) { - if (itr->second == spec[ia]) + if (itr->second == anum[ia]) { spname = itr->first; break; @@ -1833,7 +1831,7 @@ int Ions::setAtoms( { (*MPIdata::serr) << "Ions::setAtoms() --- ERROR: unknown " "species for atomic number " - << spec[ia] << std::endl; + << anum[ia] << std::endl; return -1; } @@ -2212,6 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { + names.clear(); for (auto& ion : local_ions_) { names.push_back(ion->name()); @@ -2220,6 +2219,7 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { + names.clear(); for (auto& ion : list_ions_) { names.push_back(ion->name()); @@ -2513,7 +2513,6 @@ void Ions::gatherLockedNames( std::vector& names, const int root, const MPI_Comm comm) const { std::vector local_names; - for (auto& ion : local_ions_) { if (ion->locked()) local_names.push_back(ion->name()); @@ -2533,7 +2532,6 @@ void Ions::gatherIndexes( std::vector& indexes, const int root, const MPI_Comm comm) const { std::vector local_indexes; - for (auto& ion : local_ions_) { local_indexes.push_back(ion->index()); @@ -2553,7 +2551,6 @@ void Ions::gatherNLprojIds( std::vector& nlprojids, const int root, const MPI_Comm comm) const { std::vector local_nlprojids; - for (auto& ion : local_ions_) { local_nlprojids.push_back(ion->nlprojid()); @@ -2573,7 +2570,6 @@ void Ions::gatherAtomicNumbers( std::vector& atnumbers, const int root, const MPI_Comm comm) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2593,7 +2589,6 @@ void Ions::gatherRandStates(std::vector& rstates, const int root, const MPI_Comm comm) const { std::vector local_rstates; - for (auto& ion : local_ions_) { local_rstates.push_back(ion->randomState(0)); @@ -2615,10 +2610,8 @@ void Ions::gatherPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { - // get position of local ion double position[3]; ion->getPosition(&position[0]); local_positions.push_back(position[0]); @@ -2640,7 +2633,6 @@ void Ions::gatherPreviousPositions( std::vector& positions, const int root, const MPI_Comm comm) const { std::vector local_positions; - for (auto& ion : local_ions_) { local_positions.push_back(ion->getPreviousPosition(0)); @@ -2662,17 +2654,14 @@ void Ions::gatherForces( std::vector& forces, const int root, const MPI_Comm comm) const { std::vector local_forces; - for (auto& ion : local_ions_) { - // get position of local ion double force[3]; ion->getForce(&force[0]); local_forces.push_back(force[0]); local_forces.push_back(force[1]); local_forces.push_back(force[2]); } - // gather data to PE root std::vector data; mgmol_tools::gatherV(local_forces, data, root, comm); @@ -2687,7 +2676,6 @@ void Ions::gatherVelocities( std::vector& velocities, const int root, const MPI_Comm comm) const { std::vector local_velocities; - for (auto& ion : local_ions_) { local_velocities.push_back(ion->velocity(0)); @@ -2711,13 +2699,10 @@ void Ions::gatherPositions(std::vector& positions, const int root) const positions.resize(3 * num_ions_, 0.); - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local positions - const int index = (*ion)->index(); - (*ion)->getPosition(&positions[3 * index]); - ++ion; + const int index = ion->index(); + ion->getPosition(&positions[3 * index]); } // gather data to PE root @@ -2726,17 +2711,17 @@ void Ions::gatherPositions(std::vector& positions, const int root) const void Ions::gatherForces(std::vector& forces, const int root) const { + assert(num_ions_ > 0); + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); forces.resize(3 * num_ions_, 0.); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local forces - const int index = (*ion)->index(); - (*ion)->getForce(&forces[3 * index]); - ion++; + const int index = ion->index(); + assert(forces.size() >= 3 * index); + assert(index < num_ions_); + ion->getForce(&forces[3 * index]); } // gather data to PE root @@ -2749,15 +2734,10 @@ void Ions::gatherLockedData(std::vector& locked_data, const int root) const MGmol_MPI& mmpi(*(MGmol_MPI::instance())); locked_data.resize(num_ions_, 0); - - std::vector::const_iterator ion = local_ions_.begin(); - while (ion != local_ions().end()) + for (auto& ion : local_ions_) { - // get local ion index - const int index = (*ion)->index(); - if ((*ion)->locked()) locked_data[index] = 1; - - ++ion; + const int index = ion->index(); + if (ion->locked()) locked_data[index] = 1; } // gather data to PE root @@ -3214,11 +3194,9 @@ void Ions::updateTaupInteractingIons() void Ions::clearLists() { local_ions_.clear(); - std::vector::iterator ion = list_ions_.begin(); - while (ion != list_ions_.end()) + for (auto& ion : list_ions_) { - delete *ion; - ion++; + delete ion; } list_ions_.clear(); } @@ -3335,24 +3313,21 @@ void Ions::initStepperData() { clearStepperData(); - std::vector::iterator lion = local_ions_.begin(); - while (lion != local_ions_.end()) + for (auto& lion : local_ions_) { - local_names_.push_back((*lion)->name()); - atmove_.push_back(!(*lion)->locked()); - pmass_.push_back((*lion)->getMass()); - gids_.push_back((*lion)->index()); + local_names_.push_back(lion->name()); + atmove_.push_back(!lion->locked()); + pmass_.push_back(lion->getMass()); + gids_.push_back(lion->index()); for (short i = 0; i < 3; i++) { - taum_.push_back((*lion)->getPreviousPosition(i)); - tau0_.push_back((*lion)->position(i)); - fion_.push_back((*lion)->force(i)); - velocity_.push_back((*lion)->velocity(i)); - rand_states_.push_back((*lion)->randomState(i)); + taum_.push_back(lion->getPreviousPosition(i)); + tau0_.push_back(lion->position(i)); + fion_.push_back(lion->force(i)); + velocity_.push_back(lion->velocity(i)); + rand_states_.push_back(lion->randomState(i)); } - - lion++; } // initialize taup to enable computing velocities int size_tau = (int)tau0_.size(); @@ -3399,11 +3374,9 @@ void Ions::updateIons() void Ions::shiftIons(const Vector3D& shift) { // update local_ions data - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->shiftPositionXLBOMDTest(shift); - ion++; + ion->shiftPositionXLBOMDTest(shift); } // update various list of ions @@ -3421,12 +3394,10 @@ void Ions::rescaleVelocities(const double factor) std::cout << "Ions::rescaleVelocities() with factor " << factor << std::endl; } - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) - { - (*ion)->rescaleVelocity(factor); - ion++; + for (auto& ion : local_ions_) + { + ion->rescaleVelocity(factor); } } diff --git a/src/Ions.h b/src/Ions.h index 33b8982b..3420d725 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -10,20 +10,20 @@ #ifndef MGMOL_IONS_H #define MGMOL_IONS_H -#include -#include -#include -#include -#include - #include "DistributedIonicData.h" +#include "HDFrestart.h" #include "Ion.h" -#include "hdf5.h" -class HDFrestart; +#include +#include +#include class Ions { +private: + /*! + * map species to atomic numbers + */ static std::map map_species_; static int num_ions_; @@ -37,7 +37,7 @@ class Ions std::vector list_ions_; - /* + /*! * ions located in local sub-domain */ std::vector local_ions_; @@ -61,6 +61,13 @@ class Ions bool has_locked_atoms_; + /*! + * Prevent usage of copy constructor by making it private and + * non-implemented + */ + Ions(const Ions&); + void operator=(const Ions&); + void readRestartVelocities(HDFrestart& h5_file); void readRestartRandomStates(HDFrestart& h5_file); void readRestartPositions(HDFrestart& h5_file); @@ -193,11 +200,9 @@ class Ions std::vector& getGids() { return gids_; } void resetForces() { - std::vector::iterator ion = local_ions_.begin(); - while (ion != local_ions_.end()) + for (auto& ion : local_ions_) { - (*ion)->resetForce(); - ion++; + ion->resetForce(); } } void resetPositionsToPrevious(); diff --git a/tests/testIons.cc b/tests/testIons.cc index e23e0e24..39c96f4a 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -166,8 +166,12 @@ int main(int argc, char** argv) const double fval = 1.12; for (auto& f : forces) f = fval; + + ions.getNames(names); ions.setLocalForces(forces, names); + ions.printForcesGlobal(std::cout); + int nlocal = ions.getNumLocIons(); std::vector lforces(3 * nlocal); ions.getLocalForces(lforces); From 89630b3fba8e8f0c998bc53433c8e4c904274b2e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 13:14:19 -0400 Subject: [PATCH 35/58] Fixes for build without HDF5P (#324) --- src/Ions.cc | 3 ++- tests/CMakeLists.txt | 18 ++++++++++-------- tests/MVP/mvp.cfg | 1 - 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index f10fb834..ed3d54e1 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -947,10 +947,11 @@ void Ions::initFromRestartFile(HDFrestart& h5_file) num_ions_ = at_names.size(); #ifdef MGMOL_USE_HDF5P if (!h5_file.useHdf5p()) +#endif { mmpi.allreduce(&num_ions_, 1, MPI_SUM); } -#endif + if (onpe0 && ct.verbose > 0) { (*MPIdata::sout) << "Ions::setFromRestartFile(), read " << num_ions_ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index df774171..865ef354 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -547,14 +547,16 @@ add_test(NAME testMD_D72 ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -add_test(NAME testHDF5single - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz - ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +if(MGMOL_USE_HDF5P) + add_test(NAME testHDF5single + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/md.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/HDF5single/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +endif() add_test(NAME testMD_MVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/mvp.cfg b/tests/MVP/mvp.cfg index bb3ab52a..868e5703 100644 --- a/tests/MVP/mvp.cfg +++ b/tests/MVP/mvp.cfg @@ -32,4 +32,3 @@ solver=MVP nb_inner_it=2 [Restart] output_level=2 -output_type=single_file From e80193b2095ded2daef4c821961653f3df5dc574 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 13:14:37 -0400 Subject: [PATCH 36/58] Fix testRhoVhRestart (#325) --- tests/RhoVhRestart/testRhoVhRestart.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/RhoVhRestart/testRhoVhRestart.cc b/tests/RhoVhRestart/testRhoVhRestart.cc index 09da3320..ce40272d 100644 --- a/tests/RhoVhRestart/testRhoVhRestart.cc +++ b/tests/RhoVhRestart/testRhoVhRestart.cc @@ -46,8 +46,8 @@ int testRhoRestart(MGmolInterface* mgmol_) /* check if the recomputed density is the same */ for (int d = 0; d < (int)rho0.size(); d++) { - double error = abs(rho0[d] - rho->rho_[0][d]) / abs(rho0[d]); - if (error > 1e-10) + double error = abs(rho0[d] - rho->rho_[0][d]); + if (error > 1e-10 * abs(rho0[d])) { printf("rank %d, rho[%d]=%.15e, rho0[%d]=%.15e\n", rank, d, rho->rho_[0][d], d, rho0[d]); From bcad1127e11f4cedec5a21f774b661589b4eb8b1 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 26 Mar 2025 14:21:30 -0400 Subject: [PATCH 37/58] Fix a few more issues with class Ions (#328) --- src/Ions.cc | 29 +++++++++++------------------ src/Ions.h | 6 +++--- tests/testIons.cc | 31 +++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index ed3d54e1..3b128b0e 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -109,14 +109,12 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { - std::vector::const_iterator ion = ions.list_ions_.begin(); - while (ion != ions.list_ions_.end()) + for (const auto& ion : ions.list_ions_) { - Ion* newion = new Ion(**ion); + Ion* newion = new Ion(*ion); newion->shiftPosition(shift); newion->setup(); list_ions_.push_back(newion); - ion++; } for (short i = 0; i < 3; ++i) lattice_[i] = ions.lattice_[i]; @@ -2212,7 +2210,7 @@ void Ions::getLocalPositions(std::vector& tau) const void Ions::getLocalNames(std::vector& names) const { names.clear(); - for (auto& ion : local_ions_) + for (const auto& ion : local_ions_) { names.push_back(ion->name()); } @@ -2220,14 +2218,14 @@ void Ions::getLocalNames(std::vector& names) const void Ions::getNames(std::vector& names) const { - names.clear(); - for (auto& ion : list_ions_) - { - names.push_back(ion->name()); - } + std::vector local_names; + getLocalNames(local_names); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + mmpi.allGatherV(local_names, names); } -void Ions::getPositions(std::vector& tau) +void Ions::getPositions(std::vector& tau) const { std::vector tau_local(3 * local_ions_.size()); @@ -2237,10 +2235,9 @@ void Ions::getPositions(std::vector& tau) mmpi.allGatherV(tau_local, tau); } -void Ions::getAtomicNumbers(std::vector& atnumbers) +void Ions::getAtomicNumbers(std::vector& atnumbers) const { std::vector local_atnumbers; - for (auto& ion : local_ions_) { local_atnumbers.push_back(ion->atomic_number()); @@ -2250,15 +2247,11 @@ void Ions::getAtomicNumbers(std::vector& atnumbers) mmpi.allGatherV(local_atnumbers, atnumbers); } -void Ions::getForces(std::vector& forces) +void Ions::getForces(std::vector& forces) const { std::vector forces_local(3 * local_ions_.size()); - getLocalForces(forces_local); - int n = getNumIons(); - forces.resize(3 * n); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allGatherV(forces_local, forces); } diff --git a/src/Ions.h b/src/Ions.h index 3420d725..3d382eb7 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -292,10 +292,10 @@ class Ions void getLocalPositions(std::vector& tau) const; void getLocalNames(std::vector& names) const; void getNames(std::vector& names) const; - void getPositions(std::vector& tau); - void getAtomicNumbers(std::vector& atnumbers); + void getPositions(std::vector& tau) const; + void getAtomicNumbers(std::vector& atnumbers) const; - void getForces(std::vector& forces); + void getForces(std::vector& forces) const; void getLocalForces(std::vector& tau) const; /*! diff --git a/tests/testIons.cc b/tests/testIons.cc index 39c96f4a..6bea5d64 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -107,6 +107,7 @@ int main(int argc, char** argv) ions.getAtomicNumbers(anumbers); if (myrank == 0) { + std::cout << "Positions:" << std::endl; int i = 0; for (auto& position : positions) { @@ -121,7 +122,7 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); // swap x and z - for (size_t i = 0; i < positions.size() - 2; i++) + for (size_t i = 0; i < positions.size() - 2; i += 3) { double x = positions[i]; double z = positions[i + 2]; @@ -162,11 +163,13 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); std::vector forces(3 * na); - // arbitrary value - const double fval = 1.12; + // set forces to a different arbitrary value for each component + int i = 0; for (auto& f : forces) - f = fval; - + { + f = (double)i; + i++; + } ions.getNames(names); ions.setLocalForces(forces, names); @@ -177,13 +180,29 @@ int main(int argc, char** argv) ions.getLocalForces(lforces); for (auto& f : lforces) { - if (std::abs(f - fval) > 1.e-14) + if (std::fmod(f, 1.) > 1.e-14) { std::cerr << "f = " << f << std::endl; MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + ions.getForces(forces); + if (myrank == 0) + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < 1.e-14) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 + << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + } mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From a78e260d4e19416217cd68a1e72d0f4a422ec5a6 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Thu, 27 Mar 2025 14:46:11 -0400 Subject: [PATCH 38/58] Introduce new constructor for class Ions (#330) * modify various functions to use argument of type Ions instead of MGmol::ions_ --- src/ABPG.cc | 6 +- src/DFTsolver.cc | 4 +- src/DavidsonSolver.cc | 8 +- src/Energy.cc | 29 ++- src/Energy.h | 12 +- src/GrassmanLineMinimization.cc | 4 +- src/HamiltonianMVPSolver.cc | 14 +- src/Ions.cc | 193 ++++++++---------- src/Ions.h | 19 +- src/MGmol.cc | 55 ++--- src/MGmol.h | 8 +- src/MGmolInterface.h | 3 - src/MVPSolver.cc | 4 +- src/PolakRibiereSolver.cc | 8 +- src/lbfgsrlx.cc | 1 - src/quench.cc | 4 +- .../testRestartEnergyAndForces.cc | 4 - 17 files changed, 183 insertions(+), 193 deletions(-) diff --git a/src/ABPG.cc b/src/ABPG.cc index 62cb6ba1..107edc63 100644 --- a/src/ABPG.cc +++ b/src/ABPG.cc @@ -38,7 +38,7 @@ void ABPG::setup(T& orbitals) // // orthof=true: wants orthonormalized updated wave functions template -int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, +int ABPG::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool /*orthof*/, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -51,8 +51,8 @@ int ABPG::updateWF(T& orbitals, Ions& /*ions*/, const double precond_factor, T res("Residual", orbitals, false); const bool check_res = (atol > 0.); - double normRes = mgmol_strategy_->computeResidual( - orbitals, work_orbitals, res, (print_res || check_res), check_res); + double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, + ions, res, (print_res || check_res), check_res); if (normRes < atol && check_res) { abpg_nl_update_tm_.stop(); diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index 91c4a0f6..206fba91 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -196,8 +196,8 @@ double DFTsolver::evaluateEnergy( // Get the new total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - eks_history_[0] - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + eks_history_[0] = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); sum_eig_[1] = sum_eig_[0]; sum_eig_[0] = 2. * proj_matrices_->getEigSum(); // 2.*sum in [Ry] diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index 0dc88ef4..a049fde7 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -500,7 +500,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(projmatrices, (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, projmatrices, orbitals, printE, os_); + ts0, projmatrices, ions_, orbitals, printE, os_); retval = checkConvergence(e0, outer_it, ct.conv_tol); if (retval == 0 || (outer_it == ct.max_electronic_steps)) @@ -549,7 +549,7 @@ int DavidsonSolver::solve( ts0 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 1), os_); e0 = energy_->evaluateTotal( - ts0, proj_mat2N_.get(), orbitals, printE, os_); + ts0, proj_mat2N_.get(), ions_, orbitals, printE, os_); } // 2N x 2N target... @@ -621,8 +621,8 @@ int DavidsonSolver::solve( const double ts1 = evalEntropy(proj_mat2N_.get(), (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat2N_.get(), orbitals, ct.verbose - 1, os_); + const double e1 = energy_->evaluateTotal(ts1, proj_mat2N_.get(), + ions_, orbitals, ct.verbose - 1, os_); // line minimization beta = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); diff --git a/src/Energy.cc b/src/Energy.cc index 1cee6d67..f53dbc93 100644 --- a/src/Energy.cc +++ b/src/Energy.cc @@ -25,11 +25,10 @@ #define RY2HA 0.5 template -Energy::Energy(const pb::Grid& mygrid, const Ions& ions, - const Potentials& pot, const Electrostatic& es, const Rho& rho, - const XConGrid& xc, SpreadPenaltyInterface* spread_penalty) +Energy::Energy(const pb::Grid& mygrid, const Potentials& pot, + const Electrostatic& es, const Rho& rho, const XConGrid& xc, + SpreadPenaltyInterface* spread_penalty) : mygrid_(mygrid), - ions_(ions), pot_(pot), es_(es), rho_(rho), @@ -59,7 +58,7 @@ double Energy::getEVrhoRho() const } template -double Energy::evaluateEnergyIonsInVext() +double Energy::evaluateEnergyIonsInVext(Ions& ions) { double energy = 0.; @@ -69,12 +68,12 @@ double Energy::evaluateEnergyIonsInVext() //(*MPIdata::sout)<<"Energy::evaluateEnergyIonsInVext()"< positions; - positions.reserve(3 * ions_.local_ions().size()); + positions.reserve(3 * ions.local_ions().size()); // loop over ions int nions = 0; - std::vector::const_iterator ion = ions_.local_ions().begin(); - while (ion != ions_.local_ions().end()) + std::vector::const_iterator ion = ions.local_ions().begin(); + while (ion != ions.local_ions().end()) { (*ion)->getPosition(position); positions.push_back(position[0]); @@ -88,9 +87,9 @@ double Energy::evaluateEnergyIonsInVext() pot_.getValVext(positions, val); // loop over ions again - ion = ions_.local_ions().begin(); + ion = ions.local_ions().begin(); int ion_index = 0; - while (ion != ions_.local_ions().end()) + while (ion != ions.local_ions().end()) { const double z = (*ion)->getZion(); // int ion_index=(*ion)->index(); @@ -112,16 +111,16 @@ double Energy::evaluateEnergyIonsInVext() template double Energy::evaluateTotal(const double ts, // in [Ha] - ProjectedMatricesInterface* projmatrices, const T& phi, const int verbosity, - std::ostream& os) + ProjectedMatricesInterface* projmatrices, Ions& ions, const T& phi, + const int verbosity, std::ostream& os) { eval_te_tm_.start(); Control& ct = *(Control::instance()); - const double eself = ions_.energySelf(); - const double ediff = ions_.energyDiff(ct.bcPoisson); - const double eipot = evaluateEnergyIonsInVext(); + const double eself = ions.energySelf(); + const double ediff = ions.energyDiff(ct.bcPoisson); + const double eipot = evaluateEnergyIonsInVext(ions); const double eigsum = 0.5 * projmatrices->getExpectationH(); diff --git a/src/Energy.h b/src/Energy.h index e6cafd98..d55d4040 100644 --- a/src/Energy.h +++ b/src/Energy.h @@ -11,6 +11,7 @@ #define MGMOL_ENERGY_H #include "Grid.h" +#include "Ions.h" #include "Rho.h" #include "SpreadPenaltyInterface.h" #include "Timer.h" @@ -20,7 +21,6 @@ #include class Potentials; -class Ions; class Electrostatic; class ProjectedMatricesInterface; class XConGrid; @@ -29,7 +29,6 @@ template class Energy { const pb::Grid& mygrid_; - const Ions& ions_; const Potentials& pot_; const Electrostatic& es_; const Rho& rho_; @@ -45,16 +44,15 @@ class Energy double getEVrhoRho() const; public: - Energy(const pb::Grid&, const Ions&, const Potentials&, - const Electrostatic&, const Rho&, const XConGrid&, - SpreadPenaltyInterface*); + Energy(const pb::Grid&, const Potentials&, const Electrostatic&, + const Rho&, const XConGrid&, SpreadPenaltyInterface*); static Timer eval_te_tm() { return eval_te_tm_; } - double evaluateTotal(const double ts, ProjectedMatricesInterface*, + double evaluateTotal(const double ts, ProjectedMatricesInterface*, Ions&, const T& phi, const int, std::ostream&); - double evaluateEnergyIonsInVext(); + double evaluateEnergyIonsInVext(Ions&); void saveVofRho(); }; diff --git a/src/GrassmanLineMinimization.cc b/src/GrassmanLineMinimization.cc index a83d6b08..f642e0f4 100644 --- a/src/GrassmanLineMinimization.cc +++ b/src/GrassmanLineMinimization.cc @@ -29,7 +29,7 @@ Timer GrassmanLineMinimization::update_states_tm_("Grassman_update_states"); // // orthof=true: wants orthonormalized updated wave functions template -int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, +int GrassmanLineMinimization::updateWF(T& orbitals, Ions& ions, const double precond_factor, const bool orthof, T& work_orbitals, const bool accelerate, const bool print_res, const double atol) { @@ -61,7 +61,7 @@ int GrassmanLineMinimization::updateWF(T& orbitals, Ions& /*ions*/, // Update wavefunctions const bool check_res = (atol > 0.); double normRes = mgmol_strategy_->computeResidual(orbitals, work_orbitals, - *new_grad_, (print_res || check_res), check_res); + ions, *new_grad_, (print_res || check_res), check_res); if (normRes < atol && check_res) { nl_update_tm_.stop(); diff --git a/src/HamiltonianMVPSolver.cc b/src/HamiltonianMVPSolver.cc index 4bc017dc..f90dd228 100644 --- a/src/HamiltonianMVPSolver.cc +++ b/src/HamiltonianMVPSolver.cc @@ -160,8 +160,8 @@ int HamiltonianMVPSolver::solve( // compute energy at origin const int printE = (ct.verbose > 1) ? 1 : 0; - double e0 - = energy_->evaluateTotal(ts0, projmatrices, orbitals, printE, os_); + double e0 = energy_->evaluateTotal( + ts0, projmatrices, ions_, orbitals, printE, os_); // // compute energy at end for new H @@ -189,8 +189,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at end (beta=1.) - double e1 - = energy_->evaluateTotal(ts1, projmatrices, orbitals, printE, os_); + double e1 = energy_->evaluateTotal( + ts1, projmatrices, ions_, orbitals, printE, os_); // // evaluate energy at mid-point @@ -226,8 +226,8 @@ int HamiltonianMVPSolver::solve( projmatrices->setHB2H(); // compute energy at midpoint - double ei - = energy_->evaluateTotal(tsi, projmatrices, orbitals, printE, os_); + double ei = energy_->evaluateTotal( + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization double beta @@ -285,7 +285,7 @@ int HamiltonianMVPSolver::solve( // compute energy at end (beta=1.) ei = energy_->evaluateTotal( - tsi, projmatrices, orbitals, printE, os_); + tsi, projmatrices, ions_, orbitals, printE, os_); // line minimization beta = minQuadPolynomialFrom3values( diff --git a/src/Ions.cc b/src/Ions.cc index 3b128b0e..ef9d687c 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -70,8 +70,18 @@ void writeData2d(HDFrestart& h5f_file, std::string datasetname, } } -Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) +Ions::Ions(const double lat[3], const std::vector& sp) + : species_(sp), setup_(false), has_locked_atoms_(false) { + setupSubdomains(lat); +} + +void Ions::setupSubdomains(const double lat[3]) +{ + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + for (short i = 0; i < 3; i++) { assert(lat[i] > 0.); @@ -79,15 +89,10 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) { (*MPIdata::serr) << "Ions constructor: lattice[" << i << "]=" << lat[i] << "!!!" << std::endl; - exit(2); + mmpi.abort(); } lattice_[i] = lat[i]; } - setup_ = false; - has_locked_atoms_ = false; - - Mesh* mymesh = Mesh::instance(); - const pb::PEenv& myPEenv = mymesh->peenv(); for (short i = 0; i < 3; i++) div_lattice_[i] = lattice_[i] / (double)(myPEenv.n_mpi_task(i)); @@ -107,6 +112,20 @@ Ions::Ions(const double lat[3], const std::vector& sp) : species_(sp) MPI_Cart_shift(cart_comm_, dir, disp, &source_[dir], &dest_[dir]); } +Ions::Ions(const std::vector& p, const std::vector& anum, + const double lat[3], const std::vector& sp) + : species_(sp) +{ + setupSubdomains(lat); + + double rmax = getMaxListRadius(); + setupListIonsBoundaries(rmax); + + num_ions_ = setAtoms(p, anum); + + setup(); +} + Ions::Ions(const Ions& ions, const double shift[3]) : species_(ions.species_) { for (const auto& ion : ions.list_ions_) @@ -181,8 +200,6 @@ void Ions::setup() ions_setup_tm.start(); - Ion::resetIndexCount(); - updateListIons(); //#ifndef NDEBUG @@ -197,7 +214,7 @@ void Ions::setup() has_locked_atoms_ = hasLockedAtoms(); // initialize data for constraints - setupContraintsData(interacting_ions_); + setupContraintsData(); computeMaxNumProjs(); @@ -297,13 +314,13 @@ void Ions::setupInteractingIons() // setup arrays to be used in constraints enforcement // using references to local_ions and extra "dummy" data -void Ions::setupContraintsData(std::vector& ions_for_constraints) +void Ions::setupContraintsData() { Control& ct = *(Control::instance()); if (ct.verbose > 0) printWithTimeStamp("Ions::setupContraintsData()...", std::cout); - const int nnloc = ions_for_constraints.size() - local_ions_.size(); + const int nnloc = interacting_ions_.size() - local_ions_.size(); // std::cout<<"interacting_ions_.size()="<::finalEnergy() // Get the total energy const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] total_energy_ = energy_->evaluateTotal( - ts, proj_matrices_.get(), *current_orbitals_, 2, os_); + ts, proj_matrices_.get(), *ions_, *current_orbitals_, 2, os_); } template @@ -828,7 +827,7 @@ void MGmol::setupPotentials(Ions& ions) pot.initialize(ions); if (ct.verbose > 0) printWithTimeStamp("Setup kbpsi...", os_); - g_kbpsi_->setup(*ions_); + g_kbpsi_->setup(ions); electrostat_->setupRhoc(pot.rho_comp()); @@ -1170,8 +1169,8 @@ void MGmol::precond_mg(OrbitalsType& phi) template double MGmol::computeResidual(OrbitalsType& orbitals, - OrbitalsType& work_orbitals, OrbitalsType& res, const bool print_residual, - const bool norm_res) + OrbitalsType& work_orbitals, Ions& ions, OrbitalsType& res, + const bool print_residual, const bool norm_res) { assert(orbitals.getIterativeIndex() >= 0); @@ -1190,7 +1189,7 @@ double MGmol::computeResidual(OrbitalsType& orbitals, // get H*psi stored in work_orbitals.psi // and psi^T H psi in Hij - getHpsiAndTheta(*ions_, orbitals, work_orbitals); + getHpsiAndTheta(ions, orbitals, work_orbitals); double norm2Res = computeConstraintResidual( orbitals, work_orbitals, res, print_residual, norm_res); @@ -1428,17 +1427,22 @@ double MGmol::evaluateEnergyAndForces(Orbitals* orbitals, Control& ct = *(Control::instance()); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); double eks = 0.; OrbitalsType* dorbitals = dynamic_cast(orbitals); - quench(*dorbitals, *ions_, ct.max_electronic_steps, 20, eks); + quench(*dorbitals, ions, ct.max_electronic_steps, 20, eks); - force(*dorbitals, *ions_); + force(*dorbitals, ions); - ions_->getForces(forces); + ions.getForces(forces); return eks; } @@ -1450,37 +1454,42 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, { OrbitalsType* dorbitals = dynamic_cast(orbitals); - ions_->setPositions(tau, atnumbers); + // create a new temporary Ions object to be used for + // energy end forces calculation + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + Ions ions(tau, atnumbers, lattice, ions_->getSpecies()); - setupPotentials(*ions_); + setupPotentials(ions); // initialize electronic density rho_->update(*dorbitals); // initialize potential - update_pot(*ions_); + update_pot(ions); // initialize projected matrices - updateHmatrix(*dorbitals, *ions_); + updateHmatrix(*dorbitals, ions); proj_matrices_->updateThetaAndHB(); // compute DM std::shared_ptr> dm_strategy( DMStrategyFactory>::create(comm_, os_, *ions_, + dist_matrix::DistMatrix>::create(comm_, os_, ions, rho_.get(), energy_.get(), electrostat_.get(), this, proj_matrices_.get(), dorbitals)); dm_strategy->update(*dorbitals); // evaluate energy and forces - double ts = 0.; - double eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), *dorbitals, 2, os_); + double ts = 0.; + double eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, *dorbitals, 2, os_); - force(*dorbitals, *ions_); + force(*dorbitals, ions); - ions_->getForces(forces); + ions.getForces(forces); return eks; } diff --git a/src/MGmol.h b/src/MGmol.h index 12197dfd..c85e7b59 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -335,17 +335,13 @@ class MGmol : public MGmolInterface void precond_mg(OrbitalsType& orbitals); void setGamma(const pb::Lap& lapOper, const Potentials& pot); double computeResidual(OrbitalsType& orbitals, OrbitalsType& work_orbitals, - OrbitalsType& res, const bool print_residual, const bool norm_res); + Ions& ions, OrbitalsType& res, const bool print_residual, + const bool norm_res); void applyAOMMprojection(OrbitalsType&); void force(OrbitalsType& orbitals, Ions& ions) { forces_->force(orbitals, ions); } - void setPositions(const std::vector& positions, - const std::vector& atnumbers) - { - ions_->setPositions(positions, atnumbers); - } /* * simply dump current state diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index dc54a8e7..9a9bf8a6 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -40,9 +40,6 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; - virtual void setPositions(const std::vector& positions, - const std::vector& atnumbers) - = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 7ab9f665..6990e29d 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -256,7 +256,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts0 = evalEntropyMVP(current_proj_mat, (ct.verbose > 1), os_); const double e0 = energy_->evaluateTotal( - ts0, current_proj_mat, orbitals, printE, os_); + ts0, current_proj_mat, ions_, orbitals, printE, os_); MatrixType target("target", numst_, numst_); @@ -309,7 +309,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) const double ts1 = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, orbitals, ct.verbose - 1, os_); + ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); // line minimization const double beta diff --git a/src/PolakRibiereSolver.cc b/src/PolakRibiereSolver.cc index 6b5f88de..3f46a00e 100644 --- a/src/PolakRibiereSolver.cc +++ b/src/PolakRibiereSolver.cc @@ -243,9 +243,9 @@ double PolakRibiereSolver::evaluateEnergy( const OrbitalsType& orbitals, const bool print_flag) { // Get the new total energy - const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] - const double energy - = energy_->evaluateTotal(ts, proj_matrices_, orbitals, print_flag, os_); + const double ts = 0.5 * proj_matrices_->computeEntropy(); // in [Ha] + const double energy = energy_->evaluateTotal( + ts, proj_matrices_, ions_, orbitals, print_flag, os_); // Control& ct(*(Control::instance())); // if( ct.verbose>2 && onpe0 )os_<<"energy="<::solve(OrbitalsType& orbitals, // evaluate residuals, preconditioned residuals for current orbitals double normRes = mgmol_strategy_->computeResidual(orbitals, - work_orbitals, *r_k_, (print_res || ct.checkResidual()), + work_orbitals, ions_, *r_k_, (print_res || ct.checkResidual()), ct.checkResidual()); if (normRes < ct.conv_tol && ct.checkResidual()) { diff --git a/src/lbfgsrlx.cc b/src/lbfgsrlx.cc index 6698eef0..ce05c1a3 100644 --- a/src/lbfgsrlx.cc +++ b/src/lbfgsrlx.cc @@ -14,7 +14,6 @@ #include "Energy.h" #include "Ions.h" #include "LBFGS.h" -#include "LBFGS_IonicStepper.h" #include "LocalizationRegions.h" #include "MGmol.h" #include "MGmol_blas1.h" diff --git a/src/quench.cc b/src/quench.cc index 073798f6..6eed74bb 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -600,8 +600,8 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, << " TS [Ha] = " << ts << std::endl; } } - last_eks - = energy_->evaluateTotal(ts, proj_matrices_.get(), orbitals, 2, os_); + last_eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, orbitals, 2, os_); if (ct.computeCondGramMD()) { diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index 3027ab55..b1fb0094 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -124,8 +124,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - Mesh* mymesh = Mesh::instance(); const pb::Grid& mygrid = mymesh->grid(); const pb::PEenv& myPEenv = mymesh->peenv(); @@ -195,8 +193,6 @@ int main(int argc, char** argv) } } - mgmol->setPositions(positions, anumbers); - // // evaluate energy and forces with wavefunctions just read // From c4817b8a1028923d71458a69a95ee05cd9486826 Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Thu, 27 Mar 2025 16:44:32 -0700 Subject: [PATCH 39/58] Add cleanup and MGmolInterface (#331) --- src/MGmolInterface.h | 1 + tests/WFEnergyAndForces/testWFEnergyAndForces.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index 9a9bf8a6..ec913944 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -25,6 +25,7 @@ class MGmolInterface virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void setup() = 0; virtual void run() = 0; + virtual void cleanup() = 0; virtual double evaluateEnergyAndForces(const std::vector& tau, const std::vector& atnumbers, std::vector& forces) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index c9e28d48..64765c7a 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,7 @@ int main(int argc, char** argv) } } + mgmol->cleanup(); delete mgmol; } // close main scope From a5f6f3a9906c8e4cee1bfe7a7ce2197531ee8f83 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 28 Mar 2025 10:58:24 -0400 Subject: [PATCH 40/58] Add function to set local forces (#333) * based on matching coordinates to local ions --- src/Ions.cc | 33 +++++++++++++++++++++++++++ src/Ions.h | 6 +++++ tests/testIons.cc | 58 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index ef9d687c..462373a9 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1322,6 +1322,39 @@ void Ions::setLocalForces( } } +void Ions::setLocalForces( + const std::vector& forces, const std::vector& coords) +{ + assert(forces.size() == coords.size()); + + // tolerance can be pretty loose, as long as it does not allow mix up + // with coordinates of other atoms + const double tol = 1.e-2; + + // loop over global list of forces and coordinates + std::vector::const_iterator cit = coords.begin(); + for (auto fit = forces.begin(); fit != forces.end(); fit += 3) + { + // find possible matching ion + for (auto& ion : local_ions_) + { + double p[3]; + ion->getPosition(&p[0]); + double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) + + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + double d = std::sqrt(d2); + if (d < tol) + { + ion->set_force(0, *fit); + ion->set_force(1, *(fit + 1)); + ion->set_force(2, *(fit + 2)); + } + } + cit += 3; + } +} + // Writes out the postions of the ions and the current forces on them by root void Ions::printForcesGlobal(std::ostream& os, const int root) const { diff --git a/src/Ions.h b/src/Ions.h index f39e1fed..aaf48e79 100644 --- a/src/Ions.h +++ b/src/Ions.h @@ -308,6 +308,12 @@ class Ions void setLocalForces(const std::vector& forces, const std::vector& names); + /*! + * set forces for ions in local_ions_ based on coordinates matching + */ + void setLocalForces( + const std::vector& forces, const std::vector& coords); + void syncData(const std::vector& sp); // void syncNames(const int nions, std::vector& local_names, // std::vector& names); diff --git a/tests/testIons.cc b/tests/testIons.cc index 6bea5d64..33c3cd24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -6,6 +6,29 @@ #include +// check that all forces components have integer values larger than 0 +// and differ from each other +int checkForces(std::vector& forces) +{ + const double tol = 1.e-14; + + for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + std::cout << "f0 = " << *f0 << std::endl; + for (auto f1 = f0 + 1; f1 != forces.end(); f1++) + { + // make sure each force component is different + if (std::abs(*f0 - *f1) < tol || *f1 < tol || *f0 < tol) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 << std::endl; + return 1; + } + } + } + + return 0; +} + int main(int argc, char** argv) { int status = 0; @@ -164,7 +187,7 @@ int main(int argc, char** argv) std::vector forces(3 * na); // set forces to a different arbitrary value for each component - int i = 0; + int i = 1; for (auto& f : forces) { f = (double)i; @@ -189,20 +212,29 @@ int main(int argc, char** argv) ions.getForces(forces); if (myrank == 0) - for (auto f0 = forces.begin(); f0 != forces.end(); f0++) + { + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + + // test Ions::setLocalForces based on coordinates matching + { + std::vector positions; + std::vector anumbers; + ions.getPositions(positions); + + ions.setLocalForces(forces, positions); + + ions.printForcesGlobal(std::cout); + + ions.getForces(forces); + if (myrank == 0) { - std::cout << "f0 = " << *f0 << std::endl; - for (auto f1 = f0 + 1; f1 != forces.end(); f1++) - { - // make sure each force component is different - if (std::abs(*f0 - *f1) < 1.e-14) - { - std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 - << std::endl; - MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); - } - } + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } + } + mpirc = MPI_Finalize(); if (mpirc != MPI_SUCCESS) { From 4e82239a124bea0128562e627b15a82d07ec2bcb Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Sun, 30 Mar 2025 10:48:27 -0700 Subject: [PATCH 41/58] Fix setLocalForces (#335) --- src/Ions.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ions.cc b/src/Ions.cc index 462373a9..70ca5c36 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -1341,8 +1341,8 @@ void Ions::setLocalForces( double p[3]; ion->getPosition(&p[0]); double d2 = (p[0] - (*cit)) * (p[0] - (*cit)) - + (p[1] - (*(cit + 1))) * (p[0] - (*(cit + 1))) - + (p[2] - (*(cit + 2))) * (p[0] - (*(cit + 2))); + + (p[1] - (*(cit + 1))) * (p[1] - (*(cit + 1))) + + (p[2] - (*(cit + 2))) * (p[2] - (*(cit + 2))); double d = std::sqrt(d2); if (d < tol) { From f715dd9961afa5d2589be19e9423a4885a4f8ac9 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Apr 2025 15:36:16 -0400 Subject: [PATCH 42/58] Fix test WFEnergyAndForces (#336) * writing restart twice in same directory was failing --- tests/WFEnergyAndForces/testWFEnergyAndForces.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc index 64765c7a..f9039abe 100644 --- a/tests/WFEnergyAndForces/testWFEnergyAndForces.cc +++ b/tests/WFEnergyAndForces/testWFEnergyAndForces.cc @@ -190,6 +190,9 @@ int main(int argc, char** argv) } } + // rename output restart file + ct.out_restart_file = ct.out_restart_file + "1"; + mgmol->cleanup(); delete mgmol; From 89d502e776bd8150c8e127fdb9980741e73ab713 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 4 Apr 2025 16:20:50 -0400 Subject: [PATCH 43/58] Add mixing option in MVP (#337) --- src/Control.cc | 4 +- src/MVPSolver.cc | 127 +++++++++++++++++++++++------------------ src/MVPSolver.h | 3 +- src/MVP_DMStrategy.cc | 2 +- tests/CMakeLists.txt | 7 +++ tests/MVP/test.py | 2 + tests/MVPmix/h2o.xyz | 6 ++ tests/MVPmix/mgmol.cfg | 36 ++++++++++++ tests/MVPmix/test.py | 103 +++++++++++++++++++++++++++++++++ 9 files changed, 230 insertions(+), 60 deletions(-) create mode 100644 tests/MVPmix/h2o.xyz create mode 100644 tests/MVPmix/mgmol.cfg create mode 100644 tests/MVPmix/test.py diff --git a/src/Control.cc b/src/Control.cc index 33fd93c4..22de9b27 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -699,8 +699,8 @@ void Control::setDefaultValues() void Control::adjust() { - // change dm_mix default to 1. if not using Davidson - if (it_algo_type_ != 2 && dm_mix < 0.) dm_mix = 1.; + // change dm_mix default to 1. if not using Davidson or MVP + if ((it_algo_type_ != 2 && DM_solver_ != 1) && dm_mix < 0.) dm_mix = 1.; if (nel_ - 2 * numst == 0) { diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 6990e29d..770acdd0 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -47,13 +47,14 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm) + const short n_inner_steps, const double mixing, const bool use_old_dm) : comm_(comm), os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), ions_(ions), - numst_(numst) + numst_(numst), + mixing_(mixing) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -267,65 +268,79 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; - - double de0 = evaluateDerivative(dmInit, delta_dm, ts0); - - if (std::abs(de0) < tol_de0 && inner_it > 0) + double beta = 0.; + if (mixing_ > 0.) { - if (onpe0 && ct.verbose > 0) - std::cout << "MVP: de0 = " << de0 - << ", convergence achieved" << std::endl; - break; + beta = mixing_; + if (onpe0 && ct.verbose > 1) + { + os_ << "MVP with beta = " << beta << std::endl; + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << std::endl; + } } - - // - // evaluate free energy at beta=1 - // - if (onpe0 && ct.verbose > 2) - std::cout << "MVP --- Target energy..." << std::endl; - proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); - proj_mat_work_->computeOccupationsFromDM(); - if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); - const double nel = proj_mat_work_->getNel(); - if (onpe0 && ct.verbose > 1) - os_ << "MVP --- Number of electrons at beta=1 : " << nel - << std::endl; - - rho_->computeRho(orbitals, target); - - mgmol_strategy_->update_pot(vh_init, ions_); - - energy_->saveVofRho(); - - // update h11 + else { - h11 = h11_nl; - mgmol_strategy_->addHlocal2matrix(orbitals, orbitals, h11); - } - - proj_mat_work_->assignH(h11); - proj_mat_work_->setHB2H(); - - const double ts1 - = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); - const double e1 = energy_->evaluateTotal( - ts1, proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); + double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + + if (std::abs(de0) < tol_de0 && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + + // + // evaluate free energy at beta=1 + // + if (onpe0 && ct.verbose > 2) + std::cout << "MVP --- Target energy..." << std::endl; + proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); + proj_mat_work_->computeOccupationsFromDM(); + if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); + const double nel = proj_mat_work_->getNel(); + if (onpe0 && ct.verbose > 1) + os_ << "MVP --- Number of electrons at beta=1 : " << nel + << std::endl; + + rho_->computeRho(orbitals, target); + + mgmol_strategy_->update_pot(vh_init, ions_); + + energy_->saveVofRho(); + + // update h11 + { + h11 = h11_nl; + mgmol_strategy_->addHlocal2matrix( + orbitals, orbitals, h11); + } + + proj_mat_work_->assignH(h11); + proj_mat_work_->setHB2H(); + + const double ts1 + = evalEntropyMVP(proj_mat_work_, (ct.verbose > 2), os_); + const double e1 = energy_->evaluateTotal(ts1, + proj_mat_work_, ions_, orbitals, ct.verbose - 1, os_); + + // line minimization + beta + = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); + assert(!std::isnan(beta)); - // line minimization - const double beta - = minQuadPolynomial(e0, e1, de0, (ct.verbose > 2), os_); - assert(!std::isnan(beta)); - - if (onpe0 && ct.verbose > 0) - { - os_ << std::setprecision(12); - os_ << std::fixed << "MVP inner iteration " << inner_it - << ", E0=" << e0 << ", E1=" << e1; - os_ << std::scientific << ", E0'=" << de0 - << " -> beta=" << beta; - os_ << std::endl; + if (onpe0 && ct.verbose > 0) + { + os_ << std::setprecision(12); + os_ << std::fixed << "MVP inner iteration " << inner_it + << ", E0=" << e0 << ", E1=" << e1; + os_ << std::scientific << ", E0'=" << de0 + << " -> beta=" << beta; + os_ << std::endl; + } } - // update DM *work_ = dmInit; work_->axpy(beta, delta_dm); diff --git a/src/MVPSolver.h b/src/MVPSolver.h index 8e7a7c70..5558bcdd 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -33,6 +33,7 @@ class MVPSolver Ions& ions_; int numst_; + double mixing_; Rho* rho_; Energy* energy_; @@ -56,7 +57,7 @@ class MVPSolver Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const bool use_old_dm); + const short n_inner_steps, const double mixing, const bool use_old_dm); ~MVPSolver(); int solve(OrbitalsType& orbitals); diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 8712ae92..899bca46 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -54,7 +54,7 @@ int MVP_DMStrategy::update(OrbitalsType& orbitals) MVPSolver solver(comm_, os_, ions_, rho_, energy_, electrostat_, mgmol_strategy_, ct.numst, ct.occ_width, global_indexes_, - ct.dm_inner_steps, use_old_dm_); + ct.dm_inner_steps, ct.dm_mix, use_old_dm_); return solver.solve(orbitals); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 865ef354..d90b9ac0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -500,6 +500,13 @@ add_test(NAME testMVP ${CMAKE_CURRENT_SOURCE_DIR}/MVP/mvp.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MVP/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testMVPmix + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/MVPmix/h2o.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME BandGapN2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/BandGapN2/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} diff --git a/tests/MVP/test.py b/tests/MVP/test.py index bd5d708e..57d9522e 100644 --- a/tests/MVP/test.py +++ b/tests/MVP/test.py @@ -51,6 +51,8 @@ energies=[] print("Check forces are smaller than tol = {}".format(tol)) for line in lines: + if line.count(b'MVP'): + print(line) if line.count(b'%%'): print(line) words=line.split() diff --git a/tests/MVPmix/h2o.xyz b/tests/MVPmix/h2o.xyz new file mode 100644 index 00000000..d5171c8b --- /dev/null +++ b/tests/MVPmix/h2o.xyz @@ -0,0 +1,6 @@ +3 + +O 0.00 0.00 0.00 +H -0.76 0.59 0.00 +H 0.76 0.59 0.00 + diff --git a/tests/MVPmix/mgmol.cfg b/tests/MVPmix/mgmol.cfg new file mode 100644 index 00000000..10edc464 --- /dev/null +++ b/tests/MVPmix/mgmol.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=48 +ny=48 +nz=48 +[Domain] +ox=-4.5 +oy=-4.5 +oz=-4.5 +lx=9. +ly=9. +lz=9. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +solver=PSD +max_steps=100 +atol=1.e-8 +step_length=2. +ortho_freq=10 +[Orbitals] +initial_type=random +initial_width=1.5 +temperature=10. +nempty=5 +[Restart] +output_level=0 +[DensityMatrix] +solver=MVP +nb_inner_it=2 +mixing=0.5 diff --git a/tests/MVPmix/test.py b/tests/MVPmix/test.py new file mode 100644 index 00000000..8ffafdbb --- /dev/null +++ b/tests/MVPmix/test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string + +print("Test MVP solver with mixing coefficient...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-4): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-4] +inp = sys.argv[nargs-3] +coords = sys.argv[nargs-2] +print("coordinates file: %s"%coords) + +#create links to potentials files +cwd = os.getcwd() + +dst = 'pseudo.O_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +dst = 'pseudo.H_ONCV_PBE_SG15' +src = sys.argv[nargs-1] + '/' + dst +if not os.path.exists(cwd+'/'+dst): + print("Create link to %s"%dst) + os.symlink(src, dst) + +#run mgmol +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp,coords) +print("Run command: {}".format(command)) + +output = subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True) + +#analyse mgmol standard output +#make sure force is below tolerance +lines=output.split(b'\n') + +convergence=0 +for line in lines: + if line.count(b'DFTsolver:') and line.count(b'convergence'): + convergence=1 + break + +if convergence==0: + print("MVP Solver did not converge") + sys.exit(1) + +flag = 0 +eigenvalues=[] +energies=[] +ecount=0 +for line in lines: + if line.count(b'FERMI'): + flag = 0 + if flag==1: + words=line.split() + for w in words: + eigenvalues.append(eval(w)) + if line.count(b'Eigenvalues'): + flag = 1 + eigenvalues=[] + if line.count(b'%%'): + words=line.split() + e=words[5][0:-1] + print(e) + ecount=ecount+1 + energies.append(eval(e)) +print(energies) + +print(eigenvalues) +tol = 1.e-4 +eigenvalue0 = -0.916 +if abs(eigenvalues[0]-eigenvalue0)>tol: + print("Expected eigenvalue 0 to be {}".format(eigenvalue0)) + sys.exit(1) +eigenvalue8 = 0.219 +if abs(eigenvalues[8]-eigenvalue8)>tol: + print("Expected eigenvalue 8 to be {}".format(eigenvalue8)) + sys.exit(1) + +niterations = ecount +print("MVP solver ran for {} iterations".format(niterations)) +if niterations>180: + print("MVP test FAILED for taking too many iterations") + sys.exit(1) + +print("Check energy...") +last_energy = energies[-1] +print("Energy = {}".format(last_energy)) +if last_energy>-17.16269: + print("Last energy = {}".format(last_energy)) + sys.exit(1) + +print("Test PASSED") +sys.exit(0) From f198397436eb7b4b3bd7e6a72d0fd5f47348623c Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Wed, 9 Apr 2025 10:48:07 -0400 Subject: [PATCH 44/58] Update MVP with mixing (#339) * use tolerance to terminate iterations * tune verbosity --- src/Control.cc | 6 +++++- src/MVPSolver.cc | 34 +++++++++++++++++++--------------- src/MVPSolver.h | 8 +++++++- src/MVP_DMStrategy.cc | 2 +- src/read_config.cc | 4 +++- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 22de9b27..57e12336 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,6 +224,7 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; + os << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " @@ -439,7 +440,7 @@ void Control::sync(void) memset(&int_buffer[0], 0, size_int_buffer * sizeof(int)); } - const short size_float_buffer = 43; + const short size_float_buffer = 44; float* float_buffer = new float[size_float_buffer]; if (mype_ == 0) { @@ -485,6 +486,7 @@ void Control::sync(void) float_buffer[40] = threshold_eigenvalue_gram_quench_; float_buffer[41] = pair_mlwf_distance_threshold_; float_buffer[42] = e0_; + float_buffer[43] = dm_tol; } else { @@ -680,6 +682,7 @@ void Control::sync(void) threshold_eigenvalue_gram_quench_ = float_buffer[40]; pair_mlwf_distance_threshold_ = float_buffer[41]; e0_ = float_buffer[42]; + dm_tol = float_buffer[43]; max_electronic_steps_loose_ = max_electronic_steps; delete[] short_buffer; @@ -1720,6 +1723,7 @@ void Control::setOptions(const boost::program_options::variables_map& vm) lrs_extrapolation = 10; dm_mix = vm["DensityMatrix.mixing"].as(); + dm_tol = vm["DensityMatrix.tol"].as(); dm_inner_steps = vm["DensityMatrix.nb_inner_it"].as(); dm_use_old_ = vm["DensityMatrix.use_old"].as() ? 1 : 0; str = vm["DensityMatrix.algo"].as(); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 770acdd0..1c093e53 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -47,14 +47,16 @@ MVPSolver::MVPSolver(MPI_Comm comm, std::ostream& os, Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const double mixing, const bool use_old_dm) + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm) : comm_(comm), os_(os), n_inner_steps_(n_inner_steps), use_old_dm_(use_old_dm), ions_(ions), numst_(numst), - mixing_(mixing) + mixing_(mixing), + tol_de0_(tol_de0) { Control& ct = *(Control::instance()); if (onpe0 && ct.verbose > 0) @@ -208,7 +210,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) kbpsi.computeHvnlMatrix(&kbpsi, ions_, h11_nl); - const double tol_de0 = 1.e-12; for (int inner_it = 0; inner_it < n_inner_steps_; inner_it++) { if (onpe0 && ct.verbose > 1) @@ -268,13 +269,26 @@ int MVPSolver::solve(OrbitalsType& orbitals) MatrixType delta_dm("delta_dm", numst_, numst_); delta_dm = target; delta_dm -= dmInit; + + double de0 = evaluateDerivative(dmInit, delta_dm, ts0); + + // check for convergence + if (std::abs(de0) < tol_de0_ && inner_it > 0) + { + if (onpe0 && ct.verbose > 0) + std::cout << "MVP: de0 = " << de0 + << ", convergence achieved" << std::endl; + break; + } + double beta = 0.; if (mixing_ > 0.) { beta = mixing_; - if (onpe0 && ct.verbose > 1) + if (onpe0 && ct.verbose > 0) { - os_ << "MVP with beta = " << beta << std::endl; + if (ct.verbose > 1) + os_ << "MVP with beta = " << beta << std::endl; os_ << std::setprecision(12); os_ << std::fixed << "MVP inner iteration " << inner_it << ", E0=" << e0 << std::endl; @@ -282,16 +296,6 @@ int MVPSolver::solve(OrbitalsType& orbitals) } else { - double de0 = evaluateDerivative(dmInit, delta_dm, ts0); - - if (std::abs(de0) < tol_de0 && inner_it > 0) - { - if (onpe0 && ct.verbose > 0) - std::cout << "MVP: de0 = " << de0 - << ", convergence achieved" << std::endl; - break; - } - // // evaluate free energy at beta=1 // diff --git a/src/MVPSolver.h b/src/MVPSolver.h index 5558bcdd..77a6a345 100644 --- a/src/MVPSolver.h +++ b/src/MVPSolver.h @@ -35,6 +35,11 @@ class MVPSolver int numst_; double mixing_; + /*! + * tolerance on energy slope in inner iterations + */ + double tol_de0_; + Rho* rho_; Energy* energy_; Electrostatic* electrostat_; @@ -57,7 +62,8 @@ class MVPSolver Electrostatic* electrostat, MGmol* mgmol_strategy, const int numst, const double kbT, const std::vector>& global_indexes, - const short n_inner_steps, const double mixing, const bool use_old_dm); + const short n_inner_steps, const double mixing, const double tol_de0, + const bool use_old_dm); ~MVPSolver(); int solve(OrbitalsType& orbitals); diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 899bca46..3fb53dee 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -54,7 +54,7 @@ int MVP_DMStrategy::update(OrbitalsType& orbitals) MVPSolver solver(comm_, os_, ions_, rho_, energy_, electrostat_, mgmol_strategy_, ct.numst, ct.occ_width, global_indexes_, - ct.dm_inner_steps, ct.dm_mix, use_old_dm_); + ct.dm_inner_steps, ct.dm_mix, ct.dm_tol, use_old_dm_); return solver.solve(orbitals); } diff --git a/src/read_config.cc b/src/read_config.cc index 642b1b99..ed0e66e1 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -297,7 +297,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(0), "Flag for computing new centers from extrapolated orbitals.")( "DensityMatrix.mixing", po::value()->default_value(-1.), - "Mixing coefficient for Density Matrix")("DensityMatrix.solver", + "Mixing coefficient for Density Matrix")("DensityMatrix.tol", + po::value()->default_value(1.e-12), + "Tolerance for Density Matrix convergence")("DensityMatrix.solver", po::value()->default_value("Mixing"), "Algorithm for updating Density Matrix: Mixing, MVP, HMVP")( "DensityMatrix.nb_inner_it", po::value()->default_value(3), From 781d7e563a53fb65ca15b9e50813164a2f322f44 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 11 Apr 2025 16:32:34 -0400 Subject: [PATCH 45/58] Update DensityMatrix class (#342) * add extrapolation function * remove unused function * simplify use of iterative index --- src/DavidsonSolver.cc | 12 +-- src/DensityMatrix.cc | 81 +++++++++---------- src/DensityMatrix.h | 40 ++++----- src/DensityMatrixSparse.cc | 15 ++-- src/DensityMatrixSparse.h | 16 ++-- src/EigenDMStrategy.cc | 2 +- src/HamiltonianMVPSolver.cc | 22 ++--- src/MGmol.cc | 4 +- src/MVPSolver.cc | 12 +-- src/NonOrthoDMStrategy.cc | 6 +- src/ProjectedMatrices.cc | 52 ++++++------ src/ProjectedMatrices.h | 40 ++++----- src/ProjectedMatrices2N.cc | 4 +- src/ProjectedMatrices2N.h | 4 +- src/ProjectedMatricesInterface.h | 22 ++--- src/ProjectedMatricesSparse.h | 9 +-- .../testDMandEnergyAndForces.cc | 2 +- .../testRestartEnergyAndForces.cc | 2 +- tests/testDensityMatrix.cc | 4 +- 19 files changed, 154 insertions(+), 195 deletions(-) diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index a049fde7..c34b6396 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -206,8 +206,8 @@ double DavidsonSolver::evaluateDerivative( const double dbeta = 0.0001; *work2N_ = dm2Ninit; work2N_->axpy(dbeta, delta_dm); - // proj_mat2N_->setDM(*work2N_,orbitals.getIterativeIndex()); - proj_mat2N_->setDM(*work2N_, -1); + + proj_mat2N_->setDM(*work2N_); proj_mat2N_->computeOccupationsFromDM(); const double tsd0e = evalEntropy(proj_mat2N_.get(), false, os_); @@ -250,7 +250,7 @@ void DavidsonSolver::buildTarget2N_MVP( // if( onpe0 )os_<<"Compute X2N..."<setHB2H(); - proj_mat2N_->updateDM(orbitals_index); + proj_mat2N_->updateDM(); target = proj_mat2N_->dm(); @@ -584,7 +584,7 @@ int DavidsonSolver::solve( // if (mmpi.PE0() && ct.verbose > 2) os_ << "Target energy..." << std::endl; - proj_mat2N_->setDM(target, orbitals.getIterativeIndex()); + proj_mat2N_->setDM(target); proj_mat2N_->computeOccupationsFromDM(); double nel = proj_mat2N_->getNel(); if (mmpi.PE0() && ct.verbose > 2) @@ -643,7 +643,7 @@ int DavidsonSolver::solve( // update DM *work2N_ = dm2Ninit; work2N_->axpy(beta, delta_dm); - proj_mat2N_->setDM(*work2N_, orbitals.getIterativeIndex()); + proj_mat2N_->setDM(*work2N_); if (inner_it < ct.dm_inner_steps - 1) { @@ -809,7 +809,7 @@ int DavidsonSolver::solve( = dynamic_cast*>( orbitals.getProjMatrices()); assert(pmat); - pmat->buildDM(new_occ, orbitals.getIterativeIndex()); + pmat->buildDM(new_occ); if (retval == 0) break; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index 78d7008e..7be19aab 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -22,8 +22,6 @@ const double factor_kernel4dot = 10.; -#define PROCRUSTES 0 - #define MGMOL_DENSITYMATRIX_FAIL(X) \ { \ std::cerr << "DensityMatrix failure:" << std::endl; \ @@ -36,7 +34,7 @@ const double factor_kernel4dot = 10.; template DensityMatrix::DensityMatrix(const int ndim) : dim_(ndim), - orbitals_index_(-1), + update_index_(-1), occ_uptodate_(false), uniform_occ_(false), stripped_(false) @@ -66,8 +64,8 @@ DensityMatrix::~DensityMatrix() } template -void DensityMatrix::build(const MatrixType& zmat, - const std::vector& occ, const int new_orbitals_index) +void DensityMatrix::build( + const MatrixType& zmat, const std::vector& occ) { #ifdef PRINT_OPERATIONS MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -98,32 +96,30 @@ void DensityMatrix::build(const MatrixType& zmat, work_->symm('r', 'l', 1., gamma, zmat, 0.); kernel4dot_->gemm('n', 't', 1., *work_, zmat, 0.); - stripped_ = false; - orbitals_index_ = new_orbitals_index; + stripped_ = false; + update_index_++; } template -void DensityMatrix::build( - const MatrixType& zmat, const int new_orbitals_index) +void DensityMatrix::build(const MatrixType& zmat) { - build(zmat, occupation_, new_orbitals_index); + build(zmat, occupation_); } // build diagonal matrix template -void DensityMatrix::build( - const std::vector& occ, const int new_orbitals_index) +void DensityMatrix::build(const std::vector& occ) { assert(dm_ != nullptr); assert(!occ.empty()); setOccupations(occ); - build(new_orbitals_index); + build(); } template -void DensityMatrix::build(const int new_orbitals_index) +void DensityMatrix::build() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); #ifdef PRINT_OPERATIONS @@ -148,13 +144,12 @@ void DensityMatrix::build(const int new_orbitals_index) * std::min(1., factor_kernel4dot * occupation_[i])); kernel4dot_->setDiagonal(w); - stripped_ = false; - orbitals_index_ = new_orbitals_index; + stripped_ = false; + update_index_++; } template -void DensityMatrix::setUniform( - const double nel, const int new_orbitals_index) +void DensityMatrix::setUniform(const double nel) { assert(!occupation_.empty()); @@ -171,22 +166,13 @@ void DensityMatrix::setUniform( uniform_occ_ = true; - build(occupation_, new_orbitals_index); -} - -template -void DensityMatrix::buildFromBlock(const MatrixType& block00) -{ - dm_->clear(); - dm_->assign(block00, 0, 0); - dm_->print(std::cout, 0, 0, 25, 25); + build(occupation_); } template void DensityMatrix::rotate( const MatrixType& rotation_matrix, const bool flag_eigen) { - if (!flag_eigen) { MatrixType invU(rotation_matrix); @@ -204,6 +190,8 @@ void DensityMatrix::rotate( invU.getrs('n', tmp, ipiv); *dm_ = tmp; + + update_index_++; } } @@ -370,8 +358,7 @@ double DensityMatrix::computeEntropy() const } template -void DensityMatrix::setto2InvS( - const MatrixType& invS, const int orbitals_index) +void DensityMatrix::setto2InvS(const MatrixType& invS) { *dm_ = invS; dm_->scal(orbital_occupation_); @@ -382,8 +369,8 @@ void DensityMatrix::setto2InvS( occupation_[st] = 1.; occ_uptodate_ = true; } - uniform_occ_ = false; - orbitals_index_ = orbitals_index; + uniform_occ_ = false; + update_index_++; } template @@ -400,8 +387,7 @@ void DensityMatrix::stripS(const MatrixType& ls) } template -void DensityMatrix::dressUpS( - const MatrixType& ls, const int new_orbitals_index) +void DensityMatrix::dressUpS(const MatrixType& ls) { assert(stripped_); @@ -410,10 +396,11 @@ void DensityMatrix::dressUpS( *dm_ = *work_; ls.trtrs('l', 't', 'n', *dm_); - orbitals_index_ = new_orbitals_index; - occ_uptodate_ = false; - uniform_occ_ = false; - stripped_ = false; + update_index_++; + + occ_uptodate_ = false; + uniform_occ_ = false; + stripped_ = false; } // dm_ -> u*dm_*u^T @@ -424,6 +411,8 @@ void DensityMatrix::transform(const MatrixType& u) { work_->gemm('n', 't', 1., *dm_, u, 0.); dm_->gemm('n', 'n', 1., u, *work_, 0.); + + update_index_++; } template @@ -434,13 +423,21 @@ double DensityMatrix::getExpectation(const MatrixType& A) } template -void DensityMatrix::mix( - const double mix, const MatrixType& matA, const int new_orbitals_index) +void DensityMatrix::mix(const double mix, const MatrixType& matA) { dm_->scal(1. - mix); - dm_->axpy(mix, matA); - orbitals_index_ = new_orbitals_index; + + update_index_++; +} + +template +void DensityMatrix::linearExtrapolate(const MatrixType& previous_dm) +{ + dm_->scal(2.); + dm_->axpy(-1., previous_dm); + + update_index_++; } template diff --git a/src/DensityMatrix.h b/src/DensityMatrix.h index 1d09e79e..8ce74542 100644 --- a/src/DensityMatrix.h +++ b/src/DensityMatrix.h @@ -31,7 +31,10 @@ class DensityMatrix MatrixType* kernel4dot_; MatrixType* work_; - int orbitals_index_; + /*! + * Keep track of changes, incremented every time dm_ is updated + */ + int update_index_; bool occ_uptodate_; bool uniform_occ_; @@ -46,16 +49,16 @@ class DensityMatrix DensityMatrix& operator=(const DensityMatrix&); DensityMatrix(const DensityMatrix&); - void build(const int new_orbitals_index); + void build(); public: DensityMatrix(const int ndim); ~DensityMatrix(); - void setUniform(const double nel, const int new_orbitals_index); + void setUniform(const double nel); - int getOrbitalsIndex() const { return orbitals_index_; } + int getIndex() const { return update_index_; } bool occupationsUptodate() const { return occ_uptodate_; } bool fromUniformOccupations() const { return uniform_occ_; } @@ -84,10 +87,10 @@ class DensityMatrix const MatrixType& kernel4dot() const { return *kernel4dot_; } - void setMatrix(const MatrixType& mat, const int orbitals_index) + void setMatrix(const MatrixType& mat) { - *dm_ = mat; - orbitals_index_ = orbitals_index; + *dm_ = mat; + update_index_++; setDummyOcc(); @@ -112,7 +115,7 @@ class DensityMatrix uniform_occ_ = false; stripped_ = false; - orbitals_index_ = 0; + update_index_ = 0; } void getOccupations(std::vector& occ) const @@ -126,22 +129,19 @@ class DensityMatrix void setOccupations(const std::vector& occ); - void setto2InvS(const MatrixType& invS, const int orbitals_index); + void setto2InvS(const MatrixType& invS); void stripS(const MatrixType& ls); - void dressUpS(const MatrixType& ls, const int new_orbitals_index); + void dressUpS(const MatrixType& ls); // dm_ -> u*dm_*u^T void transform(const MatrixType& u); - void buildFromBlock(const MatrixType& block00); - double computeEntropy() const; void computeOccupations(const MatrixType& ls); - void build(const std::vector& occ, const int new_orbitals_index); - void build(const MatrixType& z, const int new_orbitals_index); - void build(const MatrixType& z, const std::vector& occ, - const int new_orbitals_index); + void build(const std::vector& occ); + void build(const MatrixType& z); + void build(const MatrixType& z, const std::vector& occ); void rotate(const MatrixType& rotation_matrix, const bool flag_eigen); void printOccupations(std::ostream& os) const; @@ -149,8 +149,12 @@ class DensityMatrix void diagonalize( const char eigv, std::vector& occ, MatrixType& vect); double getExpectation(const MatrixType& A); - void mix( - const double mix, const MatrixType& matA, const int new_orbitals_index); + void mix(const double mix, const MatrixType& matA); + + /*! + * dm <- dm + (dm-previous_dm) = 2.*dm - previous_dm + */ + void linearExtrapolate(const MatrixType& previous_dm); int write(HDFrestart& h5f_file, std::string& name); int read(HDFrestart& h5f_file, std::string& name); diff --git a/src/DensityMatrixSparse.cc b/src/DensityMatrixSparse.cc index 4217b8ac..e554501a 100644 --- a/src/DensityMatrixSparse.cc +++ b/src/DensityMatrixSparse.cc @@ -26,7 +26,7 @@ DensityMatrixSparse::DensityMatrixSparse( MGmol_MPI& mmpi = *(MGmol_MPI::instance()); orbital_occupation_ = mmpi.nspin() > 1 ? 1. : 2.; - orbitals_index_ = -1; + update_index_ = -1; if (dim_ > 0) { @@ -44,27 +44,24 @@ DensityMatrixSparse::~DensityMatrixSparse() } } -void DensityMatrixSparse::setUniform(const double nel, const int orbitals_index) +void DensityMatrixSparse::setUniform(const double nel) { const double occ = (double)((double)nel / (double)dim_); assert(occ < 1.01); - orbitals_index_ = orbitals_index; + update_index_++; (*dm_).reset(); const double uval = (double)occ * orbital_occupation_; for (std::vector::const_iterator st = locvars_.begin(); st != locvars_.end(); ++st) (*dm_).insertMatrixElement(*st, *st, uval, INSERT, true); - - return; } -void DensityMatrixSparse::setto2InvS( - const VariableSizeMatrix& invS, const int orbitals_index) +void DensityMatrixSparse::setto2InvS(const VariableSizeMatrix& invS) { *dm_ = invS; dm_->scale(orbital_occupation_); - orbitals_index_ = orbitals_index; + update_index_++; } // build density matrix, given computed locally centered data void DensityMatrixSparse::assembleMatrixFromCenteredData( @@ -92,7 +89,7 @@ void DensityMatrixSparse::assembleMatrixFromCenteredData( dtor_DM.updateLocalRows((*dm_)); gather_DM_tm_.stop(); - orbitals_index_ = orbitals_index; + update_index_ = orbitals_index; } // compute trace of dot product dm_ . vsmat double DensityMatrixSparse::getTraceDotProductWithMat( diff --git a/src/DensityMatrixSparse.h b/src/DensityMatrixSparse.h index b574bd0b..170057fa 100644 --- a/src/DensityMatrixSparse.h +++ b/src/DensityMatrixSparse.h @@ -32,7 +32,7 @@ class DensityMatrixSparse VariableSizeMatrix* dm_; - int orbitals_index_; + int update_index_; double orbital_occupation_; @@ -43,16 +43,14 @@ class DensityMatrixSparse ~DensityMatrixSparse(); - void setUniform(const double nel, const int new_orbitals_index); + void setUniform(const double nel); - void setto2InvS( - const VariableSizeMatrix& invS, const int orbitals_index); - int getOrbitalsIndex() const { return orbitals_index_; } - void setMatrix( - const VariableSizeMatrix& mat, const int orbitals_index) + void setto2InvS(const VariableSizeMatrix& invS); + int getIndex() const { return update_index_; } + void setMatrix(const VariableSizeMatrix& mat) { - *dm_ = mat; - orbitals_index_ = orbitals_index; + *dm_ = mat; + update_index_++; } void assembleMatrixFromCenteredData(const std::vector& data, const std::vector& locRowIds, const int* globalColIds, diff --git a/src/EigenDMStrategy.cc b/src/EigenDMStrategy.cc index 8fd8443a..358ed5f9 100644 --- a/src/EigenDMStrategy.cc +++ b/src/EigenDMStrategy.cc @@ -37,7 +37,7 @@ int EigenDMStrategy::update(OrbitalsType& orbitals) = dynamic_cast< ProjectedMatrices>*>( proj_matrices_); - pmat->updateDMwithEigenstatesAndRotate(orbitals.getIterativeIndex(), zz); + pmat->updateDMwithEigenstatesAndRotate(zz); // if( onpe0 && ct.verbose>2 ) // (*MPIdata::sout)<<"get_dm_diag: rotate orbitals "<::solve( ProjMatrixType* projmatrices = dynamic_cast(orbitals.getProjMatrices()); - int iterative_index = 0; - // save computed vh for a fair energy "comparison" with vh computed // in close neigborhood const pb::GridFunc vh_init(electrostat_->getVh()); @@ -134,13 +132,11 @@ int HamiltonianMVPSolver::solve( // // evaluate energy at origin // - iterative_index++; - projmatrices->assignH(*hmatrix_); projmatrices->setHB2H(); // update DM and compute entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double ts0 = evalEntropyMVP(projmatrices, true, os_); // Update density rho_->update(orbitals); @@ -168,10 +164,8 @@ int HamiltonianMVPSolver::solve( // MatrixType htarget(projmatrices->getH()); - iterative_index++; - // update DM and compute entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double ts1 = evalEntropyMVP(projmatrices, true, os_); // Update density rho_->update(orbitals); @@ -204,10 +198,8 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(h11); projmatrices->setHB2H(); - iterative_index++; - // update DM and entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); double tsi = evalEntropyMVP(projmatrices, true, os_); // Update density @@ -262,10 +254,8 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(h11); projmatrices->setHB2H(); - iterative_index++; - // update DM and entropy - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); tsi = evalEntropyMVP(projmatrices, true, os_); // Update density @@ -325,9 +315,7 @@ int HamiltonianMVPSolver::solve( projmatrices->assignH(*hmatrix_); projmatrices->setHB2H(); - iterative_index++; - - projmatrices->updateDM(iterative_index); + projmatrices->updateDM(); // Generate new density rho_->update(orbitals); diff --git a/src/MGmol.cc b/src/MGmol.cc index 018a07a2..b3db1b77 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -364,9 +364,7 @@ int MGmol::initial() // initialize Rho if (ct.verbose > 0) printWithTimeStamp("Initialize Rho...", os_); - if (ct.restart_info <= 1) - proj_matrices_->setDMuniform( - ct.getNelSpin(), current_orbitals_->getIterativeIndex()); + if (ct.restart_info <= 1) proj_matrices_->setDMuniform(ct.getNelSpin()); rho_->setup(ct.getOrthoType(), current_orbitals_->getOverlappingGids()); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 1c093e53..4f9165a5 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -104,8 +104,8 @@ double MVPSolver::evaluateDerivative( const double dbeta = 0.001; *work_ = dmInit; work_->axpy(dbeta, delta_dm); - // proj_mat_work_->setDM(*work_,orbitals.getIterativeIndex()); - proj_mat_work_->setDM(*work_, -1); + + proj_mat_work_->setDM(*work_); proj_mat_work_->computeOccupationsFromDM(); const double tsd0e = evalEntropyMVP(proj_mat_work_, false, os_); @@ -145,7 +145,7 @@ void MVPSolver::buildTarget_MVP( // proj_mat_work_->setHB2H(); - proj_mat_work_->updateDM(orbitals_index); + proj_mat_work_->updateDM(); target = proj_mat_work_->dm(); if (ct.verbose > 2) @@ -301,7 +301,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) // if (onpe0 && ct.verbose > 2) std::cout << "MVP --- Target energy..." << std::endl; - proj_mat_work_->setDM(target, orbitals.getIterativeIndex()); + proj_mat_work_->setDM(target); proj_mat_work_->computeOccupationsFromDM(); if (ct.verbose > 2) proj_mat_work_->printOccupations(os_); const double nel = proj_mat_work_->getNel(); @@ -353,7 +353,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) { *work_ = target; } - proj_mat_work_->setDM(*work_, orbitals.getIterativeIndex()); + proj_mat_work_->setDM(*work_); if (inner_it < n_inner_steps_ - 1) { @@ -373,7 +373,7 @@ int MVPSolver::solve(OrbitalsType& orbitals) ProjectedMatrices* projmatrices = dynamic_cast*>( orbitals.getProjMatrices()); - projmatrices->setDM(*work_, orbitals.getIterativeIndex()); + projmatrices->setDM(*work_); projmatrices->setEigenvalues(proj_mat_work_->getEigenvalues()); projmatrices->assignH(proj_mat_work_->getH()); projmatrices->setHB2H(); diff --git a/src/NonOrthoDMStrategy.cc b/src/NonOrthoDMStrategy.cc index 3835b039..28a8f1d5 100644 --- a/src/NonOrthoDMStrategy.cc +++ b/src/NonOrthoDMStrategy.cc @@ -31,7 +31,7 @@ void NonOrthoDMStrategy::initialize(OrbitalsType& orbitals) (*MPIdata::sout) << "NonOrthoDMStrategy::initialize()..." << std::endl; } - proj_matrices_->updateDM(orbitals.getIterativeIndex()); + proj_matrices_->updateDM(); } template @@ -59,11 +59,11 @@ int NonOrthoDMStrategy::update(OrbitalsType& orbitals) if (mix_ < 1.) proj_matrices_->saveDM(); // compute new density matrix - proj_matrices_->updateDM(orbitals.getIterativeIndex()); + proj_matrices_->updateDM(); if (mix_ < 1.) { - proj_matrices_->updateDMwithRelax(mix_, orbitals.getIterativeIndex()); + proj_matrices_->updateDMwithRelax(mix_); } if (ct.verbose > 2) diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 74ff01ed..87c9fd48 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -271,7 +271,7 @@ void ProjectedMatrices::setDMto2InvS() if (mmpi.instancePE0() && ct.verbose > 1) std::cout << "ProjectedMatrices::setDMto2InvS()..." << std::endl; - dm_->setto2InvS(gm_->getInverse(), gm_->getAssociatedOrbitalsIndex()); + dm_->setto2InvS(gm_->getInverse()); } template @@ -296,30 +296,28 @@ void ProjectedMatrices::solveGenEigenProblem( } template -void ProjectedMatrices::buildDM( - const MatrixType& z, const int orbitals_index) +void ProjectedMatrices::buildDM(const MatrixType& z) { - dm_->build(z, orbitals_index); + dm_->build(z); } template -void ProjectedMatrices::buildDM(const MatrixType& z, - const std::vector& occ, const int orbitals_index) +void ProjectedMatrices::buildDM( + const MatrixType& z, const std::vector& occ) { - dm_->build(z, occ, orbitals_index); + dm_->build(z, occ); } template void ProjectedMatrices::buildDM( - const std::vector& occ, const int orbitals_index) + const std::vector& occ) { - dm_->build(occ, orbitals_index); + dm_->build(occ); } // Use Chebyshev approximation to compute chemical potential and density matrix template -void ProjectedMatrices::updateDMwithChebApproximation( - const int iterative_index) +void ProjectedMatrices::updateDMwithChebApproximation() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -354,15 +352,14 @@ void ProjectedMatrices::updateDMwithChebApproximation( } // compute chemical potential and density matrix with Chebyshev // approximation. - double final_mu = computeChemicalPotentialAndDMwithChebyshev( - order, emin, emax, iterative_index); + double final_mu + = computeChemicalPotentialAndDMwithChebyshev(order, emin, emax); if (mmpi.instancePE0() && ct.verbose > 1) std::cout << "Final mu_ = " << final_mu << " [Ha]" << std::endl; } template -void ProjectedMatrices::updateDMwithEigenstates( - const int iterative_index) +void ProjectedMatrices::updateDMwithEigenstates() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -381,14 +378,14 @@ void ProjectedMatrices::updateDMwithEigenstates( // Build the density matrix X // X = Z * gamma * Z^T - buildDM(zz, iterative_index); + buildDM(zz); } //"replicated" implementation of SP2. // Theta is replicated on each MPI task, and SP2 solve run independently // by each MPI task template -void ProjectedMatrices::updateDMwithSP2(const int iterative_index) +void ProjectedMatrices::updateDMwithSP2() { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); Control& ct = *(Control::instance()); @@ -433,21 +430,21 @@ void ProjectedMatrices::updateDMwithSP2(const int iterative_index) MatrixType dm("dm", dim_, dim_); sp2.getDM(dm, gm_->getInverse()); - dm_->setMatrix(dm, iterative_index); + dm_->setMatrix(dm); } template -void ProjectedMatrices::updateDM(const int iterative_index) +void ProjectedMatrices::updateDM() { Control& ct = *(Control::instance()); MGmol_MPI& mmpi = *(MGmol_MPI::instance()); if (ct.DMEigensolver() == DMEigensolverType::Eigensolver) - updateDMwithEigenstates(iterative_index); + updateDMwithEigenstates(); else if (ct.DMEigensolver() == DMEigensolverType::Chebyshev) - updateDMwithChebApproximation(iterative_index); + updateDMwithChebApproximation(); else if (ct.DMEigensolver() == DMEigensolverType::SP2) - updateDMwithSP2(iterative_index); + updateDMwithSP2(); else { std::cerr << "Eigensolver not available in " @@ -470,7 +467,7 @@ void ProjectedMatrices::updateDM(const int iterative_index) template void ProjectedMatrices::updateDMwithEigenstatesAndRotate( - const int iterative_index, MatrixType& zz) + MatrixType& zz) { // solves generalized eigenvalue problem // and return solution in zz @@ -479,7 +476,7 @@ void ProjectedMatrices::updateDMwithEigenstatesAndRotate( rotateAll(zz, true); - dm_->build(zz, iterative_index); + dm_->build(zz); } template @@ -614,7 +611,7 @@ void ProjectedMatrices::dressupDM() if (mmpi.instancePE0()) std::cout << "ProjectedMatrices::dressupDM()" << std::endl; #endif - dm_->dressUpS(gm_->getCholeskyL(), gm_->getAssociatedOrbitalsIndex()); + dm_->dressUpS(gm_->getCholeskyL()); } template @@ -999,8 +996,7 @@ double ProjectedMatrices::computeTraceInvSmultMatMultTheta( template double ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( - const int order, const double emin, const double emax, - const int iterative_index) + const int order, const double emin, const double emax) { assert(emax > emin); assert(nel_ >= 0.); @@ -1175,7 +1171,7 @@ ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( dm.gemm('N', 'N', 1., tmp, gm_->getInverse(), 0.); double orbital_occupation = mmpi.nspin() > 1 ? 1. : 2.; dm.scal(orbital_occupation); - dm_->setMatrix(dm, iterative_index); + dm_->setMatrix(dm); return mu_; } diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index f2881763..142dbf70 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -80,8 +80,8 @@ class ProjectedMatrices : public ProjectedMatricesInterface { computeChemicalPotentialAndOccupations(width_, dim_); } - double computeChemicalPotentialAndDMwithChebyshev(const int order, - const double emin, const double emax, const int iterative_index); + double computeChemicalPotentialAndDMwithChebyshev( + const int order, const double emin, const double emax); protected: // indexes corresponding to valid function in each subdomain @@ -197,12 +197,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface int getDMMatrixIndex() const override { assert(dm_); - return dm_->getOrbitalsIndex(); - } - void setDMuniform(const double nel, const int orbitals_index) override - { - dm_->setUniform(nel, orbitals_index); + return dm_->getIndex(); } + void setDMuniform(const double nel) override { dm_->setUniform(nel); } int dim() const { return dim_; } void computeInvS() override; @@ -280,10 +277,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface } void setDMto2InvS() override; - void buildDM(const MatrixType& z, const int orbitals_index); - void buildDM(const MatrixType& z, const std::vector&, - const int orbitals_index); - void buildDM(const std::vector&, const int orbitals_index); + void buildDM(const MatrixType& z); + void buildDM(const MatrixType& z, const std::vector&); + void buildDM(const std::vector&); double getEigSum() override; double getExpectation(const MatrixType& A); @@ -323,12 +319,11 @@ class ProjectedMatrices : public ProjectedMatricesInterface int readDM(HDFrestart& h5f_file) override; int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; - void updateDM(const int iterative_index) override; - void updateDMwithEigenstates(const int iterative_index); - void updateDMwithSP2(const int iterative_index); - void updateDMwithEigenstatesAndRotate( - const int iterative_index, MatrixType& zz); - void updateDMwithChebApproximation(const int iterative_index) override; + void updateDM() override; + void updateDMwithEigenstates(); + void updateDMwithSP2(); + void updateDMwithEigenstatesAndRotate(MatrixType& zz); + void updateDMwithChebApproximation() override; void computeChemicalPotentialAndOccupations( const double width, const int max_numst) { @@ -354,16 +349,16 @@ class ProjectedMatrices : public ProjectedMatricesInterface void resetDM() override { - dm_->setMatrix(*mat_X_old_, 0); + dm_->setMatrix(*mat_X_old_); dm_->stripS(*mat_L_old_); } - void updateDMwithRelax(const double mix, const int itindex) override + void updateDMwithRelax(const double mix) override { // cout<<"ProjectedMatrices::updateDMwithRelax()..."<mix(mix, *mat_X_old_, itindex); + dm_->mix(mix, *mat_X_old_); } SquareLocalMatrices getReplicatedDM(); @@ -414,10 +409,7 @@ class ProjectedMatrices : public ProjectedMatricesInterface pmat.gemm('n', 'n', 1.0, mat, *theta_, 0.); } MatrixType& getMatHB() { return *matHB_; } - void setDM(const MatrixType& mat, const int orbitals_index) - { - dm_->setMatrix(mat, orbitals_index); - } + void setDM(const MatrixType& mat) { dm_->setMatrix(mat); } void setEigenvalues(const std::vector& eigenvalues) { memcpy(eigenvalues_.data(), eigenvalues.data(), diff --git a/src/ProjectedMatrices2N.cc b/src/ProjectedMatrices2N.cc index b0696f94..479ee4f0 100644 --- a/src/ProjectedMatrices2N.cc +++ b/src/ProjectedMatrices2N.cc @@ -39,7 +39,7 @@ void ProjectedMatrices2N::assignBlocksH( template void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( - const double occ_width, const int iterative_index, const bool flag_reduce_T) + const double occ_width, const bool flag_reduce_T) { MGmol_MPI& mmpi = *(MGmol_MPI::instance()); @@ -64,7 +64,7 @@ void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( (*MPIdata::sout) << "MVP target with mu = " << ProjectedMatricesInterface::mu_ << " [Ry]" << std::endl; - ProjectedMatrices::buildDM(*work2N_, iterative_index); + ProjectedMatrices::buildDM(*work2N_); } template class ProjectedMatrices2N>; diff --git a/src/ProjectedMatrices2N.h b/src/ProjectedMatrices2N.h index 69ca2838..a87bf83f 100644 --- a/src/ProjectedMatrices2N.h +++ b/src/ProjectedMatrices2N.h @@ -26,8 +26,8 @@ class ProjectedMatrices2N : public ProjectedMatrices void assignBlocksH(MatrixType&, MatrixType&, MatrixType&, MatrixType&); - void iterativeUpdateDMwithEigenstates(const double occ_width, - const int iterative_index, const bool flag_reduce_T = true); + void iterativeUpdateDMwithEigenstates( + const double occ_width, const bool flag_reduce_T = true); void diagonalizeDM(std::vector& occ, MatrixType& vect) { // we are assuming Gram matrix=identity diff --git a/src/ProjectedMatricesInterface.h b/src/ProjectedMatricesInterface.h index c52595e1..53508a2e 100644 --- a/src/ProjectedMatricesInterface.h +++ b/src/ProjectedMatricesInterface.h @@ -199,10 +199,10 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction virtual double computeEntropy() = 0; virtual double getEigSum() = 0; - virtual void updateTheta() = 0; - virtual void computeInvB() = 0; - virtual void printGramMM(std::ofstream& tfile) = 0; - virtual void setDMuniform(const double nel, const int orbitals_index) = 0; + virtual void updateTheta() = 0; + virtual void computeInvB() = 0; + virtual void printGramMM(std::ofstream& tfile) = 0; + virtual void setDMuniform(const double nel) = 0; virtual double dotProductWithInvS( const SquareLocalMatrices& ss) @@ -258,10 +258,9 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction } virtual void saveDM() { exitWithErrorMessage("saveDM"); } virtual void resetDM() { exitWithErrorMessage("resetDM"); } - virtual void updateDMwithRelax(const double mix, const int itindex) + virtual void updateDMwithRelax(const double mix) { (void)mix; - (void)itindex; exitWithErrorMessage("updateDMwithRelax"); } @@ -297,18 +296,11 @@ class ProjectedMatricesInterface : public ChebyshevApproximationFunction return 0; } - virtual void updateDMwithChebApproximation(const int iterative_index) + virtual void updateDMwithChebApproximation() { - (void)iterative_index; - exitWithErrorMessage("updateDMwithChebApproximation"); } - virtual void updateDM(const int iterative_index) - { - (void)iterative_index; - - exitWithErrorMessage("updateDM"); - } + virtual void updateDM() { exitWithErrorMessage("updateDM"); } virtual void setDMto2InvS() { exitWithErrorMessage("setDMto2InvS"); } virtual void initializeMatB( diff --git a/src/ProjectedMatricesSparse.h b/src/ProjectedMatricesSparse.h index fd3f6918..fdbb4f88 100644 --- a/src/ProjectedMatricesSparse.h +++ b/src/ProjectedMatricesSparse.h @@ -170,15 +170,12 @@ class ProjectedMatricesSparse : public ProjectedMatricesInterface /* scale H */ // (*matHB_).scale(vel_); } - void setDMuniform(const double nel, const int orbitals_index) override - { - dm_->setUniform(nel, orbitals_index); - } + void setDMuniform(const double nel) override { dm_->setUniform(nel); } int getDMMatrixIndex() const override { assert(dm_ != nullptr); - return dm_->getOrbitalsIndex(); + return dm_->getIndex(); } int getGramMatrixIndex() const { @@ -253,7 +250,7 @@ class ProjectedMatricesSparse : public ProjectedMatricesInterface void setDMto2InvS() override { assert(invS_ != nullptr); - dm_->setto2InvS(invS_->getInvS(), invS_->getGramMatrixOrbitalsIndex()); + dm_->setto2InvS(invS_->getInvS()); } void computeInvS() override diff --git a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc index 1cf1cf88..a5a086fe 100644 --- a/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc +++ b/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc @@ -172,7 +172,7 @@ int main(int argc, char** argv) // // reset initial DM to test iterative solve for it - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); ct.dm_inner_steps = 50; eks = mgmol->evaluateDMandEnergyAndForces( &orbitals, positions, anumbers, forces); diff --git a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc index b1fb0094..058b0a8e 100644 --- a/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc +++ b/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc @@ -162,7 +162,7 @@ int main(int argc, char** argv) orbitals.setIterativeIndex(1); // set initial DM with uniform occupations - projmatrices->setDMuniform(ct.getNelSpin(), 0); + projmatrices->setDMuniform(ct.getNelSpin()); projmatrices->printDM(std::cout); // swap H and O to make sure order of atoms in list does not matter diff --git a/tests/testDensityMatrix.cc b/tests/testDensityMatrix.cc index 8a3649c1..6a65c995 100644 --- a/tests/testDensityMatrix.cc +++ b/tests/testDensityMatrix.cc @@ -95,10 +95,10 @@ TEST_CASE( // setup density matrix DensityMatrix dm(n); - dm.setMatrix(matK, 0); + dm.setMatrix(matK); dm.stripS(ls); - dm.dressUpS(ls, 1); + dm.dressUpS(ls); const MatrixType& newM = dm.getMatrix(); if (myrank == 0) std::cout << "new M" << std::endl; From 47cba98972fad2cc08e704e1736bb13ebd9fddcf Mon Sep 17 00:00:00 2001 From: "Siu Wun \"Tony\" Cheung" Date: Mon, 14 Apr 2025 07:34:23 -0700 Subject: [PATCH 46/58] Fixing DM tolerance (#341) * Fix Density Matrix tolerance read --------- Co-authored-by: Jean-Luc Fattebert --- src/Control.cc | 5 ++--- src/read_config.cc | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 57e12336..208a8975 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -224,7 +224,8 @@ void Control::print(std::ostream& os) << conv_tol << std::endl; os << std::fixed; os << " Density matrix mixing = " << dm_mix << std::endl; - os << " Density matrix tol = " << dm_tol << std::endl; + os << std::setprecision(4) << std::scientific + << " Density matrix tol = " << dm_tol << std::endl; if (DMEigensolver() == DMEigensolverType::Eigensolver) { os << " Density matrix computation algorithm = " @@ -1743,8 +1744,6 @@ void Control::setOptions(const boost::program_options::variables_map& vm) else dm_algo_ = 2; - dm_tol = vm["DensityMatrix.tol"].as(); - str = vm["DensityMatrix.solver"].as(); if (str.compare("Mixing") == 0) DM_solver_ = 0; if (str.compare("MVP") == 0) DM_solver_ = 1; diff --git a/src/read_config.cc b/src/read_config.cc index ed0e66e1..54bf98f8 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -321,10 +321,7 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(100), "Maximum number of iterations for power method " "to compute interval for Chebyshev " - "approximation of density matrix. ")("DensityMatrix.tol", - po::value()->default_value(1.e-7), - "tolerance, used in iterative DM computation convergence " - "criteria"); + "approximation of density matrix. "); po::options_description cmdline_options; cmdline_options.add(generic); From 6c75cbf36ac299197ef6306ada40eb877b54001e Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Mon, 30 Jun 2025 13:50:03 -0700 Subject: [PATCH 47/58] Clean-up implementation --- src/md.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/md.cc b/src/md.cc index e8d06133..b425d4b8 100644 --- a/src/md.cc +++ b/src/md.cc @@ -407,8 +407,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) } bool ROM_MVP = (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF); + DensityMatrix>* dm = nullptr; + dist_matrix::DistMatrix* previous_dm = nullptr; #ifdef MGMOL_HAS_LIBROM // ROM - initialize orbitals and density matrix + // assume ct.Mehrstellen() nor ct.short_sighted nor use_replicated_matrix + // so that Line 267 of MGmol.cc is activated if (ROM_MVP) { if (onpe0) os_ << "Setup ROM MVP solver..." << std::endl; @@ -422,6 +426,12 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) = getProjectedMatrices(); projmatrices->setDMuniform(ct.getNelSpin(), 0); projmatrices->printDM(os_); + + + std::shared_ptr>> projmatrices_downcast = + std::dynamic_pointer_cast>>(projmatrices); + dm = &(projmatrices_downcast->getDM()); + previous_dm = new dist_matrix::DistMatrix(dm->getMatrix()); } #endif @@ -489,6 +499,13 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) md_updateDMandEnergy_tm.start(); updateDMandEnergy(**orbitals, *ROM_ions, eks); md_updateDMandEnergy_tm.stop(); + + // save last DM in tmp + dist_matrix::DistMatrix tmp(dm->getMatrix()); + // extrapolate DM using previous_dm + dm->linearExtrapolate(*previous_dm); + // reset previous_dm + *previous_dm = tmp; } else { From ffc087cd7396ed0b9247a864cfe3df44794f5ac7 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Fri, 11 Jul 2025 18:04:04 -0400 Subject: [PATCH 48/58] Rm oversubscribe from CMakeLists.txt (#346) * already set in build script --- tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d90b9ac0..61f4c3e5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -547,7 +547,7 @@ add_test(NAME ReplicatedSP2 ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testMD_D72 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/test.py - ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/mgmol_quench.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MD_D72/mgmol_md.cfg @@ -601,7 +601,7 @@ add_test(NAME ChebyshevMVP if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py - ${MPIEXEC} --oversubscribe ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 5 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/quench.cfg ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/md.cfg From 089787685b10c5261d43212f90c2accd0b243f86 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 14 Jul 2025 13:38:48 -0400 Subject: [PATCH 49/58] Adapt to new environment on condo @ ORNL (#348) --- scripts/build_condo-mod.sh | 8 +++----- scripts/build_condo.sh | 21 +++++---------------- scripts/modules.condo | 9 ++++----- scripts/modules.condo-mod | 10 ++++------ 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/scripts/build_condo-mod.sh b/scripts/build_condo-mod.sh index d9abe034..4dfe5fc6 100755 --- a/scripts/build_condo-mod.sh +++ b/scripts/build_condo-mod.sh @@ -3,9 +3,6 @@ #Before compiling, load the following modules: source scripts/modules.condo-mod -# We need to define the cmake blas vendor option here to find the right one. -BLAS_VENDOR=OpenBLAS - MGMOL_ROOT=`pwd` INSTALL_DIR=${MGMOL_ROOT}/install @@ -15,16 +12,17 @@ BUILD_DIR=${MGMOL_ROOT}/build mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} +SCALAPACK_DIR=/home/q8j/Software/ScaLapack/scalapack-2.2.2 + # call cmake cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ - -DBLA_VENDOR=${BLAS_VENDOR} \ -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.3" \ + -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.5" \ -DMPIEXEC_EXECUTABLE=${OPENMPI_DIR}/bin/mpiexec \ .. diff --git a/scripts/build_condo.sh b/scripts/build_condo.sh index 3a349cbe..68aa16dc 100755 --- a/scripts/build_condo.sh +++ b/scripts/build_condo.sh @@ -1,41 +1,30 @@ #/bin/bash ## An example script to build on ONRL condo systems (CADES). -## This script assumes intel/ mkl libraries are being used. #Before compiling, load the following modules: source scripts/modules.condo -# set some environment variables using loaded module path -export SCALAPACK_ROOT=${MKLROOT} - -# We need to define the cmake blas vendor option here to find the right one. -BLAS_VENDOR=Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -BLACS_LIB=${MKLROOT}/lib/intel64 - MGMOL_ROOT=`pwd` -INSTALL_DIR=${MGMOL_ROOT}/mgmol_install +INSTALL_DIR=${MGMOL_ROOT}/install mkdir -p ${INSTALL_DIR} BUILD_DIR=${MGMOL_ROOT}/build mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} +SCALAPACK_DIR=/home/q8j/Software/ScaLapack/scalapack-2.2.2 + # call cmake cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=mpiCC \ -DCMAKE_Fortran_COMPILER=mpif77 \ - -DHDF5_LIBRARIES=${HDF5_DIR}/lib/libhdf5.so \ - -DHDF5_HL_LIBRARIES=${HDF5_DIR}/lib/libhdf5_hl.so \ - -DHDF5_INCLUDE_DIRS=${HDF5_DIR}/include \ - -DBLA_VENDOR=${BLAS_VENDOR} \ + -DMGMOL_USE_HDF5P=OFF \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ + -DSCALAPACK_LIBRARY="${SCALAPACK_DIR}/lib/libscalapack.a;/lib64/libgfortran.so.5" \ -DMPIEXEC_EXECUTABLE=${OPENMPI_DIR}/bin/mpiexec \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_openmpi_lp64.so \ .. # call make install diff --git a/scripts/modules.condo b/scripts/modules.condo index 6f27e62b..de36562f 100644 --- a/scripts/modules.condo +++ b/scripts/modules.condo @@ -1,6 +1,5 @@ -module load PE-intel/3.0 -module load boost/1.67.0-pe3 -module load mkl -module load hdf5_parallel/1.10.3 -module load cmake/3.18.4 +module load hdf5 +module load boost +module load cmake module load python +module load openblas diff --git a/scripts/modules.condo-mod b/scripts/modules.condo-mod index 18377905..de36562f 100644 --- a/scripts/modules.condo-mod +++ b/scripts/modules.condo-mod @@ -1,7 +1,5 @@ -module load PE-gnu/3.0 -module load hdf5-parallel/1.8.20 +module load hdf5 module load boost -module load cmake/3.20.3 -module load python/3.6.6 -module load openBLAS/0.2.19 -module load scalapack/2.0.2 +module load cmake +module load python +module load openblas From 3cd53dcf413ab2d4bb8a933f071bf6a35a6e50ba Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Mon, 14 Jul 2025 13:50:23 -0700 Subject: [PATCH 50/58] Attempt to fix error --- src/md.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/md.cc b/src/md.cc index b425d4b8..7babc23d 100644 --- a/src/md.cc +++ b/src/md.cc @@ -427,7 +427,6 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) projmatrices->setDMuniform(ct.getNelSpin(), 0); projmatrices->printDM(os_); - std::shared_ptr>> projmatrices_downcast = std::dynamic_pointer_cast>>(projmatrices); dm = &(projmatrices_downcast->getDM()); From 832ff173535dcb71b82c9660f91341ad3e8e6833 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 15 Jul 2025 12:31:52 -0700 Subject: [PATCH 51/58] Attempt to fix bugs --- src/md.cc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/md.cc b/src/md.cc index 7babc23d..ea29b89c 100644 --- a/src/md.cc +++ b/src/md.cc @@ -406,10 +406,10 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) h5f_file_.reset(); } +#ifdef MGMOL_HAS_LIBROM bool ROM_MVP = (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF); DensityMatrix>* dm = nullptr; dist_matrix::DistMatrix* previous_dm = nullptr; -#ifdef MGMOL_HAS_LIBROM // ROM - initialize orbitals and density matrix // assume ct.Mehrstellen() nor ct.short_sighted nor use_replicated_matrix // so that Line 267 of MGmol.cc is activated @@ -496,15 +496,11 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) if (ROM_MVP) { md_updateDMandEnergy_tm.start(); + dist_matrix::DistMatrix tmp(dm->getMatrix()); + dm->linearExtrapolate(*previous_dm); + *previous_dm = tmp; updateDMandEnergy(**orbitals, *ROM_ions, eks); md_updateDMandEnergy_tm.stop(); - - // save last DM in tmp - dist_matrix::DistMatrix tmp(dm->getMatrix()); - // extrapolate DM using previous_dm - dm->linearExtrapolate(*previous_dm); - // reset previous_dm - *previous_dm = tmp; } else { From 7d89625fd44c9035284f7c8877c26184d8e3f0c4 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 15 Jul 2025 10:24:48 -0400 Subject: [PATCH 52/58] Remove unused function --- src/MGmol.h | 1 - src/readInput.cc | 42 ------------------------------------------ 2 files changed, 43 deletions(-) diff --git a/src/MGmol.h b/src/MGmol.h index c85e7b59..9ce4b81f 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -225,7 +225,6 @@ class MGmol : public MGmolInterface void printEigAndOcc(); - int readCoordinates(std::ifstream* tfile, const bool cell_relative); int readCoordinates(const std::string& filename, const bool cell_relative); double computeConstraintResidual(OrbitalsType& orbitals, const OrbitalsType& hphi, OrbitalsType& res, const bool print_residual, diff --git a/src/readInput.cc b/src/readInput.cc index 79fbecf1..8c9f9e03 100644 --- a/src/readInput.cc +++ b/src/readInput.cc @@ -162,48 +162,6 @@ int MGmol::readLRsFromInput(std::ifstream* tfile) return ct.numst; } -template -int MGmol::readCoordinates( - std::ifstream* tfile, const bool cell_relative) -{ - Control& ct = *(Control::instance()); - if (ct.verbose > 0) printWithTimeStamp("Read atomic coordinates...", os_); - Mesh* mymesh = Mesh::instance(); - const pb::Grid& mygrid = mymesh->grid(); - - const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; - - // setup ions - const std::vector& sp(ct.getSpecies()); - ions_.reset(new Ions(lattice, sp)); - - if (ct.restart_info > 0 - && ct.override_restart == 0) // read restart ionic positions - { - if (ct.restart_info > 0) - { - if (onpe0 && ct.verbose > 0) - { - os_ << "Initialize ionic positions from restart file " - << ct.restart_file << std::endl; - } - ions_->initFromRestartFile(*h5f_file_); - } - } - else - { - // Coordinates and species type for each ion. - int info = ions_->readAtoms(tfile, cell_relative); - - return info; - } - - const int num_ions = ions_->getNumIons(); - if (onpe0) os_ << num_ions << " ions in simulation" << std::endl; - - return 0; -} - template int MGmol::readCoordinates( const std::string& filename, const bool cell_relative) From c0dd3b32b157fe47e12286745afef2c63de63715 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 15 Jul 2025 11:41:11 -0400 Subject: [PATCH 53/58] Clean up use of coords file when restarting * use coordinates file to override atomic coordinates when using restart wavefunctions --- src/Control.cc | 74 +++++++++++++++++++------------------- src/Control.h | 1 - src/main.cc | 10 +++--- src/md.cc | 2 +- src/readInput.cc | 21 +++++------ src/read_config.cc | 17 +++++---- tests/LBFGS/test.py | 2 +- tests/MD_D72/test.py | 2 +- tests/MD_MVP/test.py | 2 +- tests/RhoVhRestart/test.py | 4 +-- 10 files changed, 64 insertions(+), 71 deletions(-) diff --git a/src/Control.cc b/src/Control.cc index 208a8975..2980ce33 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -112,7 +112,6 @@ Control::Control() out_restart_file_type = -1; spread_radius = -1.; iprint_residual = -1; - override_restart = -1; dot_product_type = -1; spread_penalty_damping_ = -1; spread_penalty_alpha_ = -1; @@ -386,7 +385,7 @@ void Control::sync(void) short_buffer[47] = out_restart_file_naming_strategy; short_buffer[48] = enforceVmass0; short_buffer[49] = dm_inner_steps; - short_buffer[50] = override_restart; + short_buffer[50] = -1; short_buffer[51] = fgmres_kim; short_buffer[52] = fgmres_maxits; short_buffer[53] = ilu_type; @@ -599,41 +598,41 @@ void Control::sync(void) out_restart_file_naming_strategy = short_buffer[47]; enforceVmass0 = short_buffer[48]; dm_inner_steps = short_buffer[49]; - override_restart = short_buffer[50]; - fgmres_kim = short_buffer[51]; - fgmres_maxits = short_buffer[52]; - ilu_type = short_buffer[53]; - ilu_lof = short_buffer[54]; - ilu_maxfil = short_buffer[55]; - coloring_algo_ = short_buffer[56]; - diel_flag_ = short_buffer[57]; - poisson_pc_nu1 = short_buffer[58]; - poisson_pc_nu2 = short_buffer[59]; - poisson_pc_nlev = short_buffer[60]; - system_charge_ = short_buffer[61]; - md_print_freq = short_buffer[62]; - use_kernel_functions = short_buffer[63]; - ngpts_[0] = short_buffer[64]; - ngpts_[1] = short_buffer[65]; - ngpts_[2] = short_buffer[66]; - computeCondGram_ = short_buffer[67]; - lrs_extrapolation = short_buffer[68]; - parallel_transport = (bool)short_buffer[69]; - with_spin_ = (bool)short_buffer[70]; - conv_criterion_ = short_buffer[71]; - load_balancing_max_iterations = short_buffer[72]; - load_balancing_modulo = short_buffer[73]; - write_clusters = short_buffer[74]; - DM_solver_ = short_buffer[75]; - dm_algo_ = short_buffer[80]; - dm_approx_order = short_buffer[81]; - dm_approx_ndigits = short_buffer[82]; - dm_approx_power_maxits = short_buffer[83]; - spread_penalty_type_ = short_buffer[84]; - dm_use_old_ = short_buffer[85]; - max_electronic_steps_tight_ = short_buffer[86]; - hartree_reset_ = short_buffer[88]; - MD_last_step_ = short_buffer[89]; + //... = short_buffer[50]; + fgmres_kim = short_buffer[51]; + fgmres_maxits = short_buffer[52]; + ilu_type = short_buffer[53]; + ilu_lof = short_buffer[54]; + ilu_maxfil = short_buffer[55]; + coloring_algo_ = short_buffer[56]; + diel_flag_ = short_buffer[57]; + poisson_pc_nu1 = short_buffer[58]; + poisson_pc_nu2 = short_buffer[59]; + poisson_pc_nlev = short_buffer[60]; + system_charge_ = short_buffer[61]; + md_print_freq = short_buffer[62]; + use_kernel_functions = short_buffer[63]; + ngpts_[0] = short_buffer[64]; + ngpts_[1] = short_buffer[65]; + ngpts_[2] = short_buffer[66]; + computeCondGram_ = short_buffer[67]; + lrs_extrapolation = short_buffer[68]; + parallel_transport = (bool)short_buffer[69]; + with_spin_ = (bool)short_buffer[70]; + conv_criterion_ = short_buffer[71]; + load_balancing_max_iterations = short_buffer[72]; + load_balancing_modulo = short_buffer[73]; + write_clusters = short_buffer[74]; + DM_solver_ = short_buffer[75]; + dm_algo_ = short_buffer[80]; + dm_approx_order = short_buffer[81]; + dm_approx_ndigits = short_buffer[82]; + dm_approx_power_maxits = short_buffer[83]; + spread_penalty_type_ = short_buffer[84]; + dm_use_old_ = short_buffer[85]; + max_electronic_steps_tight_ = short_buffer[86]; + hartree_reset_ = short_buffer[88]; + MD_last_step_ = short_buffer[89]; poisson_lap_type_ = static_cast(short_buffer[90]); numst = int_buffer[0]; @@ -1787,7 +1786,6 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // options not available in configure file lr_updates_type = 0; precond_factor_computed = false; - override_restart = 0; mix_pot = 1.; project_out_psd = 0; multipole_order = 1; diff --git a/src/Control.h b/src/Control.h index 9481cc24..c9a128b3 100644 --- a/src/Control.h +++ b/src/Control.h @@ -555,7 +555,6 @@ class Control short out_restart_file_naming_strategy; short restart_file_type; short out_restart_file_type; - short override_restart; short verbose; diff --git a/src/main.cc b/src/main.cc index 3c6830a0..621b72de 100644 --- a/src/main.cc +++ b/src/main.cc @@ -58,7 +58,7 @@ int main(int argc, char** argv) /* * read runtime parameters */ - std::string input_filename(""); + std::string coords_filename(""); std::string lrs_filename; std::string constraints_filename(""); @@ -70,7 +70,7 @@ int main(int argc, char** argv) // read from PE0 only if (MPIdata::onpe0) { - read_config(argc, argv, vm, input_filename, lrs_filename, + read_config(argc, argv, vm, coords_filename, lrs_filename, constraints_filename, total_spin, with_spin); } @@ -89,7 +89,7 @@ int main(int argc, char** argv) int ret = ct.checkOptions(); if (ret < 0) return ret; - mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(coords_filename); mmpi.bcastGlobal(lrs_filename); // Enter main scope @@ -97,10 +97,10 @@ int main(int argc, char** argv) MGmolInterface* mgmol; if (ct.isLocMode()) mgmol = new MGmol(global_comm, *MPIdata::sout, - input_filename, lrs_filename, constraints_filename); + coords_filename, lrs_filename, constraints_filename); else mgmol = new MGmol(global_comm, *MPIdata::sout, - input_filename, lrs_filename, constraints_filename); + coords_filename, lrs_filename, constraints_filename); mgmol->setup(); diff --git a/src/md.cc b/src/md.cc index edccdb17..b34ca8c8 100644 --- a/src/md.cc +++ b/src/md.cc @@ -333,7 +333,7 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) constraints_->printConstraints(os_); - if (ct.restart_info > 0 && !ct.override_restart) + if (ct.restart_info > 0) { if (onpe0) os_ << "Use restart file to initialize MD..." << std::endl; stepper->init(*h5f_file_); diff --git a/src/readInput.cc b/src/readInput.cc index 8c9f9e03..bd6a7dd1 100644 --- a/src/readInput.cc +++ b/src/readInput.cc @@ -164,7 +164,7 @@ int MGmol::readLRsFromInput(std::ifstream* tfile) template int MGmol::readCoordinates( - const std::string& filename, const bool cell_relative) + const std::string& coords_filename, const bool cell_relative) { Control& ct = *(Control::instance()); if (ct.verbose > 0) printWithTimeStamp("Read atomic coordinates...", os_); @@ -177,29 +177,26 @@ int MGmol::readCoordinates( const std::vector& sp(ct.getSpecies()); ions_.reset(new Ions(lattice, sp)); - if (ct.restart_info > 0 - && ct.override_restart == 0) // read restart ionic positions + if (ct.restart_info > 0 && coords_filename.empty()) { - if (ct.restart_info > 0) + // read restart atomic positions + if (onpe0 && ct.verbose > 0) { - if (onpe0 && ct.verbose > 0) - { - os_ << "Initialize ionic positions from restart file " - << ct.restart_file << std::endl; - } - ions_->initFromRestartFile(*h5f_file_); + os_ << "Initialize atomic positions from restart file " + << ct.restart_file << std::endl; } + ions_->initFromRestartFile(*h5f_file_); } else { // Coordinates and species type for each ion. - int info = ions_->readAtoms(filename, cell_relative); + int info = ions_->readAtoms(coords_filename, cell_relative); return info; } const int num_ions = ions_->getNumIons(); - if (onpe0) os_ << num_ions << " ions in simulation" << std::endl; + if (onpe0) os_ << num_ions << " atoms in simulation" << std::endl; return 0; } diff --git a/src/read_config.cc b/src/read_config.cc index 54bf98f8..3ca9ec69 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -19,7 +19,7 @@ namespace po = boost::program_options; int read_config(int argc, char** argv, po::variables_map& vm, - std::string& input_file, std::string& lrs_filename, + std::string& coords_filename, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin) { // use configure file if it can be found @@ -35,9 +35,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, "help,h", "produce help message")("check", "check input")( "config,c", po::value(&config_file)->default_value("mgmol.cfg"), - "name of a file of a configuration.")("atomicCoordinates,i", - po::value>(), - "coordinates filename")("LRsFilename,l", + "name of configuration file")("atomicCoordinates,i", + po::value(&coords_filename), + "atomic coordinates filename")("LRsFilename,l", po::value(&lrs_filename), "LRs filename"); // Declare a group of options (with default when appropriate) that @@ -149,9 +149,8 @@ int read_config(int argc, char** argv, po::variables_map& vm, "Tolerance on forces for Geometry optimization")( "GeomOpt.max_steps", po::value()->default_value(1), "max. number of Geometry optimization steps")("GeomOpt.dt", - po::value(), "Delta t for trial pseudo-time steps")( - "atomicCoordinates", po::value>(), - "coordinates filename")("Thermostat.type", + po::value(), + "Delta t for trial pseudo-time steps")("Thermostat.type", po::value()->default_value("Langevin"), "Thermostat type")("Thermostat.temperature", po::value()->default_value(-1.), "Thermostat temperature")( @@ -379,9 +378,9 @@ int read_config(int argc, char** argv, po::variables_map& vm, } if (vm.count("atomicCoordinates")) { - input_file + coords_filename = vm["atomicCoordinates"].as>()[0]; - std::cout << "Input files is: " << input_file << "\n"; + std::cout << "Coordinates files is: " << coords_filename << "\n"; } else { diff --git a/tests/LBFGS/test.py b/tests/LBFGS/test.py index 324fd129..7150b3ce 100755 --- a/tests/LBFGS/test.py +++ b/tests/LBFGS/test.py @@ -56,7 +56,7 @@ os.remove('wave.out') os.symlink(restart_file, 'wave.out') -command = "{} {} -c {} -i {} -l {}".format(mpicmd,exe,inp2,coords,lrs) +command = "{} {} -c {} -l {}".format(mpicmd,exe,inp2,lrs) print(command) output = subprocess.check_output(command,shell=True) diff --git a/tests/MD_D72/test.py b/tests/MD_D72/test.py index 3f98b2bd..22773b6b 100755 --- a/tests/MD_D72/test.py +++ b/tests/MD_D72/test.py @@ -54,7 +54,7 @@ os.remove('wave.out') os.symlink(restart_file, 'wave.out') -command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) +command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) #remove created files diff --git a/tests/MD_MVP/test.py b/tests/MD_MVP/test.py index 47b5462d..c767031e 100755 --- a/tests/MD_MVP/test.py +++ b/tests/MD_MVP/test.py @@ -51,7 +51,7 @@ os.rename(restart_file, 'snapshotMVP') #run MGmol - command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords) + command = "{} {} -c {}".format(mpicmd,exe,inp2) output2 = subprocess.check_output(command,shell=True) #remove used restart files diff --git a/tests/RhoVhRestart/test.py b/tests/RhoVhRestart/test.py index a34b962f..0d14383e 100755 --- a/tests/RhoVhRestart/test.py +++ b/tests/RhoVhRestart/test.py @@ -53,7 +53,7 @@ sys.exit(1) #run MD -command = "{} {} -c {} -i {}".format(mpicmd,mgmol_exe,input2,coords) +command = "{} {} -c {}".format(mpicmd,mgmol_exe,input2) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') @@ -68,7 +68,7 @@ sys.exit(1) #run test -command = "{} {} -c {} -i {}".format(mpicmd,test_exe,input3,coords) +command = "{} {} -c {}".format(mpicmd,test_exe,input3) print("Run command: {}".format(command)) output = subprocess.check_output(command,shell=True) lines=output.split(b'\n') From a8be389ff3df7f15070d1d29344b7e20c7145ee0 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 15 Jul 2025 13:16:26 -0400 Subject: [PATCH 54/58] Add test restart with override atomic coordinates --- tests/CMakeLists.txt | 9 ++++ tests/OverrideCoords/mgmol.cfg | 27 +++++++++++ tests/OverrideCoords/restart.cfg | 24 +++++++++ tests/OverrideCoords/shifted.xyz | 8 +++ tests/OverrideCoords/sih4.xyz | 8 +++ tests/OverrideCoords/test.py | 83 ++++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+) create mode 100644 tests/OverrideCoords/mgmol.cfg create mode 100644 tests/OverrideCoords/restart.cfg create mode 100644 tests/OverrideCoords/shifted.xyz create mode 100644 tests/OverrideCoords/sih4.xyz create mode 100755 tests/OverrideCoords/test.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 61f4c3e5..01805cd7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -456,6 +456,15 @@ add_test(NAME testSiH4 ${CMAKE_CURRENT_SOURCE_DIR}/SiH4/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/SiH4/sih4.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testOverrideCoords + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/restart.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/sih4.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/OverrideCoords/shifted.xyz + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testCl2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/Cl2_ONCVPSP_LDA/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} diff --git a/tests/OverrideCoords/mgmol.cfg b/tests/OverrideCoords/mgmol.cfg new file mode 100644 index 00000000..fad77b2d --- /dev/null +++ b/tests/OverrideCoords/mgmol.cfg @@ -0,0 +1,27 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=45 +atol=1.e-9 +num_lin_iterations=2 +[Orbitals] +initial_type=Gaussian +initial_width=2. +[Restart] +output_level=3 diff --git a/tests/OverrideCoords/restart.cfg b/tests/OverrideCoords/restart.cfg new file mode 100644 index 00000000..0715e6b1 --- /dev/null +++ b/tests/OverrideCoords/restart.cfg @@ -0,0 +1,24 @@ +verbosity=2 +xcFunctional=LDA +[Mesh] +nx=40 +ny=40 +nz=40 +[Domain] +ox=-6.75 +oy=-6.75 +oz=-6.75 +lx=13.5 +ly=13.5 +lz=13.5 +[Potentials] +pseudopotential=pseudo.Si +pseudopotential=pseudo.H +[Run] +type=QUENCH +[Quench] +max_steps=25 +atol=1.e-8 +[Restart] +input_level=3 +input_filename=wave.out diff --git a/tests/OverrideCoords/shifted.xyz b/tests/OverrideCoords/shifted.xyz new file mode 100644 index 00000000..089658f8 --- /dev/null +++ b/tests/OverrideCoords/shifted.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.935 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/OverrideCoords/sih4.xyz b/tests/OverrideCoords/sih4.xyz new file mode 100644 index 00000000..b3f921e3 --- /dev/null +++ b/tests/OverrideCoords/sih4.xyz @@ -0,0 +1,8 @@ +5 +SiH4 molecule (coordinates in Angstrom) +Si 0.0 0.0 0.0 +H 0.885 0.885 0.885 +H -0.885 -0.885 0.885 +H -0.885 0.885 -0.885 +H 0.885 -0.885 -0.885 + diff --git a/tests/OverrideCoords/test.py b/tests/OverrideCoords/test.py new file mode 100755 index 00000000..2de3fd4f --- /dev/null +++ b/tests/OverrideCoords/test.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +import sys +import os +import subprocess +import string +import shutil + +print("Test Override coordinates...") + +nargs=len(sys.argv) + +mpicmd = sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3] +for i in range(4,nargs-6): + mpicmd = mpicmd + " "+sys.argv[i] +print("MPI run command: {}".format(mpicmd)) + +exe = sys.argv[nargs-6] +inp1 = sys.argv[nargs-5] +inp2 = sys.argv[nargs-4] +coords1 = sys.argv[nargs-3] +coords2 = sys.argv[nargs-2] +print("coordinates file: %s"%coords1) + +#create links to potentials files +dst1 = 'pseudo.Si' +dst2 = 'pseudo.H' +src1 = sys.argv[nargs-1] + '/' + dst1 +src2 = sys.argv[nargs-1] + '/' + dst2 + +if not os.path.exists(dst1): + print("Create link to %s"%dst1) + os.symlink(src1, dst1) +if not os.path.exists(dst2): + print("Create link to %s"%dst2) + os.symlink(src2, dst2) + +#run quench +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp1,coords1) +print("Run command: {}".format(command)) +output1 = subprocess.check_output(command,shell=True) +lines=output1.split(b'\n') + +#analyse output of quench +for line in lines: + if line.count(b'%%'): + print(line) + +#run quench with shifted coordinates +command = "ls -ld snapshot* | awk '{ print $9 }' | tail -n1" +print(command) +restart_file = subprocess.check_output(command,shell=True) +restart_file=str(restart_file[:-1],'utf-8') +print(restart_file) + +try: + os.symlink(restart_file, 'wave.out') +except FileExistsError: + os.remove('wave.out') + os.symlink(restart_file, 'wave.out') + +command = "{} {} -c {} -i {}".format(mpicmd,exe,inp2,coords2) +print(command) +output2 = subprocess.check_output(command,shell=True) + +#remove created files +shutil.rmtree(restart_file) +os.remove('wave.out') + +#analyse mgmol standard output +lines=output2.split(b'\n') + +flag = 0 +for line in lines: + if line.count(b'%%'): + print(line) + if line.count(b'achieved'): + flag=1 + +if flag==0: + print("second run did not converge...") + sys.exit(1) + +sys.exit(0) From de6eb6baa56577fb73217d175ac1d7c6df6c3f5e Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Mon, 14 Jul 2025 11:26:05 -0400 Subject: [PATCH 55/58] Add some missing include files * required for gcc12 --- tests/testSetGhostValues.cc | 2 ++ tests/testTradeGhostValues.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/testSetGhostValues.cc b/tests/testSetGhostValues.cc index c3680087..90aafde0 100644 --- a/tests/testSetGhostValues.cc +++ b/tests/testSetGhostValues.cc @@ -12,6 +12,8 @@ #include "catch.hpp" +#include + TEST_CASE("Set ghost values", "[set ghosts") { const double origin[3] = { 0., 0., 0. }; diff --git a/tests/testTradeGhostValues.cc b/tests/testTradeGhostValues.cc index 069b9374..8294aaec 100644 --- a/tests/testTradeGhostValues.cc +++ b/tests/testTradeGhostValues.cc @@ -13,6 +13,8 @@ #include "catch.hpp" +#include + // function of periodicity nx, ny, nz double cos3(const int i, const int j, const int k, const int nx, const int ny, const int nz) From a3c1074d83ef3c852062bb6a9147bda516393e8f Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Mon, 21 Jul 2025 15:57:56 -0700 Subject: [PATCH 56/58] Using one CPU only --- examples/PinnedH2O/job.ref | 2 +- examples/PinnedH2O/job.rom_3DOF | 2 +- examples/PinnedH2O/mgmol_ref_test2.cfg | 2 +- examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 4 ++-- tests/PinnedH2O_3DOF/job.offline | 19 ++++++++++++------- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref index 32024e6f..f9826e8e 100644 --- a/examples/PinnedH2O/job.ref +++ b/examples/PinnedH2O/job.ref @@ -8,7 +8,7 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 8 +set ncpus = 1 set case = 2 set maindir = /p/lustre2/cheung26/mgmol diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF index 0e38688b..b7dc5d6c 100644 --- a/examples/PinnedH2O/job.rom_3DOF +++ b/examples/PinnedH2O/job.rom_3DOF @@ -8,7 +8,7 @@ date setenv OMP_NUM_THREADS 1 #setenv KMP_DETERMINISTIC_REDUCTION 1 -set ncpus = 8 +set ncpus = 1 set case = 2 set maindir = /p/lustre2/cheung26/mgmol diff --git a/examples/PinnedH2O/mgmol_ref_test2.cfg b/examples/PinnedH2O/mgmol_ref_test2.cfg index 67aa443c..d6120c30 100644 --- a/examples/PinnedH2O/mgmol_ref_test2.cfg +++ b/examples/PinnedH2O/mgmol_ref_test2.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=500 +num_steps=50 dt=40. thermostat=OFF [Quench] diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg index 24273c40..f7084d20 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=500 +num_steps=50 dt=40. thermostat=OFF [Quench] @@ -38,7 +38,7 @@ output_level=4 [ROM] stage=online_pinned_H2O_3dof [ROM.offline] -basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_1/PinnedH2O_3DOF_orbitals_basis_2_2 [ROM.basis] compare_md=false number_of_orbital_basis=34 diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline index 5b4224f0..13393c88 100644 --- a/tests/PinnedH2O_3DOF/job.offline +++ b/tests/PinnedH2O_3DOF/job.offline @@ -16,15 +16,8 @@ setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH set exe = mgmol-opt -cp $maindir/install_quartz/bin/$exe . - set cfg_offline = mgmol_offline.cfg -ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . -ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . - -source $maindir/scripts/modules.quartz - set bondlength_min = 0.95 set bondlength_max = 1.05 set bondlength_num_increments = 10 @@ -32,6 +25,16 @@ set bondangle_min = -5.0 set bondangle_max = 5.0 set bondangle_num_increments = 10 +cp $maindir/install_quartz/bin/$exe . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +mkdir data +cd data + foreach i (`seq 0 $bondlength_num_increments`) set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` set bondlength_one = `printf "%.2f" $bondlength_one` @@ -56,4 +59,6 @@ foreach i (`seq 0 $bondlength_num_increments`) end end +cd .. + date From 5c9d31b80384e83930a6244b1ad34bfdffc3587a Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 22 Jul 2025 12:52:14 -0700 Subject: [PATCH 57/58] Switch to 500 time steps with longer time --- examples/PinnedH2O/job.ref | 4 ++-- examples/PinnedH2O/job.rom_3DOF | 4 ++-- examples/PinnedH2O/mgmol_ref_test2.cfg | 2 +- examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref index f9826e8e..c716b724 100644 --- a/examples/PinnedH2O/job.ref +++ b/examples/PinnedH2O/job.ref @@ -1,7 +1,7 @@ #!/bin/tcsh #SBATCH -N 1 -#SBATCH -t 1:00:00 -#SBATCH -p pdebug +#SBATCH -t 3:00:00 +#SBATCH -p pbatch date diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF index b7dc5d6c..fdbb1a55 100644 --- a/examples/PinnedH2O/job.rom_3DOF +++ b/examples/PinnedH2O/job.rom_3DOF @@ -1,7 +1,7 @@ #!/bin/tcsh #SBATCH -N 1 -#SBATCH -t 1:00:00 -#SBATCH -p pdebug +#SBATCH -t 2:00:00 +#SBATCH -p pbatch date diff --git a/examples/PinnedH2O/mgmol_ref_test2.cfg b/examples/PinnedH2O/mgmol_ref_test2.cfg index d6120c30..67aa443c 100644 --- a/examples/PinnedH2O/mgmol_ref_test2.cfg +++ b/examples/PinnedH2O/mgmol_ref_test2.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=50 +num_steps=500 dt=40. thermostat=OFF [Quench] diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg index f7084d20..bb2912d9 100644 --- a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -18,7 +18,7 @@ pseudopotential=pseudo.H_ONCV_PBE_SG15 [Run] type=MD [MD] -num_steps=50 +num_steps=500 dt=40. thermostat=OFF [Quench] From 1b06e02b7d48e5a2b7530c96b7762a210f29a386 Mon Sep 17 00:00:00 2001 From: Jean-Luc Fattebert Date: Tue, 22 Jul 2025 16:34:00 -0400 Subject: [PATCH 58/58] Print MVP timers --- src/MGmol.cc | 4 ++++ src/MVPSolver.cc | 8 ++------ src/MVPSolver.h | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/MGmol.cc b/src/MGmol.cc index b3db1b77..d895aabc 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -45,6 +45,7 @@ #include "MGmol.h" #include "MLWFTransform.h" #include "MPIdata.h" +#include "MVPSolver.h" #include "MasksSet.h" #include "Mesh.h" #include "OrbitalsPreconditioning.h" @@ -900,6 +901,9 @@ void MGmol::printTimers() AndersonMix::update_tm().print(os_); proj_matrices_->printTimers(os_); ShortSightedInverse::printTimers(os_); + if (std::is_same::value) + MVPSolver>::printTimers(os_); VariableSizeMatrixInterface::printTimers(os_); DataDistribution::printTimers(os_); PackedCommunicationBuffer::printTimers(os_); diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index 4f9165a5..699a01f3 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -398,12 +398,8 @@ int MVPSolver::solve(OrbitalsType& orbitals) template void MVPSolver::printTimers(std::ostream& os) { - if (onpe0) - { - os << std::setprecision(2) << std::fixed << std::endl; - solve_tm_.print(os); - target_tm_.print(os); - } + solve_tm_.print(os); + target_tm_.print(os); } template class MVPSolver class ProjectedMatrices; @@ -67,7 +66,7 @@ class MVPSolver ~MVPSolver(); int solve(OrbitalsType& orbitals); - void printTimers(std::ostream& os); + static void printTimers(std::ostream& os); }; #endif