Merge branch 'dfpt_band_parallel' into 'develop'

Add distributed band parallelization to Wstat

See merge request west-devel/West!35

.gitlab-ci.yml

@@ -54,7 +54,7 @@ stages:
     - make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
     - make -j4 all
     - cd test-suite
-    - make NP=$CI_NP NI=$CI_NI NT=$CI_NT
+    - make NP=$CI_NP NI=$CI_NI NB=$CI_NB NT=$CI_NT
   artifacts:
     when: on_failure
     paths:
@@ -83,6 +83,7 @@ gcc840_t:
   variables:
     CI_NP: 8
     CI_NI: 1
+    CI_NB: 1
     CI_NT: 1
   extends:
     - .template_bot_start
@@ -92,7 +93,8 @@ gcc840_t:
 gcc930_t:
   variables:
     CI_NP: 8
-    CI_NI: 1
+    CI_NI: 2
+    CI_NB: 2
     CI_NT: 1
   extends:
     - .template_bot_start
@@ -103,9 +105,10 @@ gcc930_t:
 
 gcc840_t2:
   variables:
-    CI_NP: 4
+    CI_NP: 8
     CI_NI: 2
-    CI_NT: 1
+    CI_NB: 1
+    CI_NT: 2
   only:
     - schedules
   extends:
@@ -115,8 +118,9 @@ gcc840_t2:
 
 gcc930_t2:
   variables:
-    CI_NP: 2
-    CI_NI: 2
+    CI_NP: 8
+    CI_NI: 1
+    CI_NB: 2
     CI_NT: 2
   only:
     - schedules

DFPT_kernel/Makefile

@@ -9,6 +9,7 @@ MODFLAGS= $(MOD_FLAG)../../iotk/src $(MOD_FLAG)../../Modules $(MOD_FLAG)../../LA
           $(MOD_FLAG)../Tools \
           $(MOD_FLAG)../FFT_kernel \
           $(MOD_FLAG)../Coulomb_kernel \
+          $(MOD_FLAG)../Para_kernel \
           $(MOD_FLAG).
 IFLAGS=
 

DFPT_kernel/apply_sternheimerop_to_m_wfcs.f90

 !
-! Copyright (C) 2015-2017 M. Govoni 
+! Copyright (C) 2015-2017 M. Govoni
 ! This file is distributed under the terms of the
 ! GNU General Public License. See the file `License'
 ! in the root directory of the present distribution,
@@ -7,7 +7,7 @@
 !
 ! This file is part of WEST.
 !
-! Contributors to this file: 
+! Contributors to this file:
 ! Marco Govoni
 !
 !-----------------------------------------------------------------------
@@ -40,7 +40,7 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
   ! input: the vector
   ! output: the operator applied to the vector
   !
-  ! Workspace 
+  ! Workspace
   !
   INTEGER :: ibnd, ig
   COMPLEX(DP) :: za
@@ -50,11 +50,15 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
   !   compute the product of the hamiltonian with the h vector
   !
   hpsi=(0.0_DP,0.0_DP)
-  !  
+  !
   IF(l_kinetic_only) THEN
      CALL k_psi( npwx, npw, m, psi, hpsi )
   ELSE
-     CALL h_psi( npwx, npw, m, psi, hpsi )
+     !
+     ! use h_psi_, i.e. h_psi without band parallelization, as wstat
+     ! handles band parallelization separately in dfpt_module
+     !
+     CALL h_psi_( npwx, npw, m, psi, hpsi )
   ENDIF
   !
   !   then we compute the operator H-epsilon S
@@ -74,4 +78,4 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
   !
   CALL stop_clock ('stern')
   !
-END SUBROUTINE 
+END SUBROUTINE

DFPT_kernel/make.depend

@@ -7,13 +7,9 @@ apply_sternheimerop_to_m_wfcs.o : ../../Modules/uspp.o
 apply_sternheimerop_to_m_wfcs.o : ../../Modules/wavefunctions.o
 apply_sternheimerop_to_m_wfcs.o : ../../PW/src/pwcom.o
 apply_sternheimerop_to_m_wfcs.o : ../Modules/westcom.o
-dfpt_module.o : ../../FFTXlib/fft_interfaces.o
 dfpt_module.o : ../../Modules/cell_base.o
-dfpt_module.o : ../../Modules/constants.o
 dfpt_module.o : ../../Modules/control_flags.o
 dfpt_module.o : ../../Modules/fft_base.o
-dfpt_module.o : ../../Modules/gvecw.o
-dfpt_module.o : ../../Modules/io_files.o
 dfpt_module.o : ../../Modules/io_global.o
 dfpt_module.o : ../../Modules/kind.o
 dfpt_module.o : ../../Modules/mp.o
@@ -28,6 +24,8 @@ dfpt_module.o : ../../PW/src/pwcom.o
 dfpt_module.o : ../FFT_kernel/fft_at_gamma.o
 dfpt_module.o : ../FFT_kernel/fft_at_k.o
 dfpt_module.o : ../Modules/westcom.o
+dfpt_module.o : ../Para_kernel/class_idistribute.o
+dfpt_module.o : ../Para_kernel/distribution_center.o
 dfpt_module.o : ../Tools/bar.o
 dfpt_module.o : ../Tools/io_push.o
 dfpt_module.o : ../Tools/types_bz_grid.o

Wfreq/make.depend

@@ -204,11 +204,8 @@ wfreq.o : ../../Modules/mp.o
 wfreq.o : ../../Modules/mp_global.o
 wfreq.o : ../Modules/westcom.o
 wfreq.o : ../Tools/west_environment.o
-wfreq_readin.o : ../../Modules/io_files.o
-wfreq_readin.o : ../../Modules/io_global.o
-wfreq_readin.o : ../../Modules/ions_base.o
-wfreq_readin.o : ../../Modules/mp.o
-wfreq_readin.o : ../../Modules/noncol.o
+wfreq_readin.o : ../../Modules/funct.o
+wfreq_readin.o : ../../Modules/mp_bands.o
 wfreq_readin.o : ../../Modules/recvec.o
 wfreq_readin.o : ../../Modules/uspp.o
 wfreq_readin.o : ../../PW/src/pwcom.o

Wfreq/wfreq_readin.f90

 !
-! Copyright (C) 2015-2017 M. Govoni 
+! Copyright (C) 2015-2017 M. Govoni
 ! This file is distributed under the terms of the
 ! GNU General Public License. See the file `License'
 ! in the root directory of the present distribution,
@@ -7,47 +7,38 @@
 !
 ! This file is part of WEST.
 !
-! Contributors to this file: 
+! Contributors to this file:
 ! Marco Govoni
 !
 !-----------------------------------------------------------------------
 SUBROUTINE wfreq_readin()
   !-----------------------------------------------------------------------
   !
-  USE ions_base,        ONLY : nat
   USE uspp,             ONLY : okvan
-  USE io_files,         ONLY : tmp_dir, prefix
-  USE io_global,        ONLY : stdout
-  USE noncollin_module, ONLY : noncolin
-  USE mp,               ONLY : mp_bcast
   USE gvecs,            ONLY : doublegrid
   USE spin_orb,         ONLY : domag
+  USE mp_bands,         ONLY : nbgrp
+  USE funct,            ONLY : dft_is_hybrid
   !
   IMPLICIT NONE
   !
-  ! Workspace
-  !
-  INTEGER :: iunit =5, ios
-  !
   CALL start_clock('wfreq_readin')
   !
-  !CALL fetch_namelist(3,(/1,2,3/))
-  !
   ! READ INPUT_WEST
   !
   CALL fetch_input_yml(1,(/1/),.TRUE.,.FALSE.)
   !
-  !  read the input file produced by the pwscf program
-  !  allocate memory and recalculate what is needed
+  ! read the input file produced by the pwscf program
+  ! allocate memory and recalculate what is needed
   !
-  !CALL read_file( )
   CALL read_pwout( )
   !
   ! PW checks
   !
   IF (domag) CALL errore('wfreq_readin','domag version not available',1)
   IF (okvan) CALL errore('wfreq_readin','ultrasoft pseudopotential not implemented',1)
-  IF (doublegrid) CALL errore('wfreq_readin', 'double grid not implemented',1)
+  IF (doublegrid) CALL errore('wfreq_readin','double grid not implemented',1)
+  IF (nbgrp > 1 .AND. dft_is_hybrid()) CALL errore('wfreq_readin','band groups not implemented for hybrids',1)
   !
   ! READ other sections of the input file
   !

Wstat/make.depend

@@ -43,6 +43,7 @@ wstat_memory_report.o : ../../Modules/control_flags.o
 wstat_memory_report.o : ../../Modules/fft_base.o
 wstat_memory_report.o : ../../Modules/io_global.o
 wstat_memory_report.o : ../../Modules/kind.o
+wstat_memory_report.o : ../../Modules/mp_bands.o
 wstat_memory_report.o : ../../Modules/mp_world.o
 wstat_memory_report.o : ../../Modules/noncol.o
 wstat_memory_report.o : ../../Modules/recvec.o
@@ -51,15 +52,12 @@ wstat_memory_report.o : ../../PW/src/pwcom.o
 wstat_memory_report.o : ../Libraries/Json/json_module.o
 wstat_memory_report.o : ../Modules/westcom.o
 wstat_memory_report.o : ../Para_kernel/distribution_center.o
-wstat_readin.o : ../../Modules/io_files.o
-wstat_readin.o : ../../Modules/io_global.o
-wstat_readin.o : ../../Modules/ions_base.o
-wstat_readin.o : ../../Modules/mp.o
-wstat_readin.o : ../../Modules/mp_world.o
-wstat_readin.o : ../../Modules/noncol.o
+wstat_readin.o : ../../Modules/funct.o
+wstat_readin.o : ../../Modules/mp_bands.o
 wstat_readin.o : ../../Modules/recvec.o
 wstat_readin.o : ../../Modules/uspp.o
 wstat_readin.o : ../../PW/src/pwcom.o
+wstat_readin.o : ../Modules/westcom.o
 wstat_restart.o : ../../Modules/io_files.o
 wstat_restart.o : ../../Modules/io_global.o
 wstat_restart.o : ../../Modules/kind.o

Wstat/wstat_memory_report.f90

@@ -22,6 +22,7 @@ SUBROUTINE wstat_memory_report()
   USE gvecs,               ONLY : ngms
   USE uspp,                ONLY : nkb
   USE control_flags,       ONLY : gamma_only
+  USE mp_bands,            ONLY : nbgrp
   USE mp_world,            ONLY : mpime,root
   USE westcom,             ONLY : nbnd_occ,n_pdep_basis,npwqx,logfile
   USE distribution_center, ONLY : pert
@@ -141,15 +142,15 @@ SUBROUTINE wstat_memory_report()
   WRITE(stdout,'(5x,"[MEM] Allocated arrays      ",5x,"est. size (Mb)", 5x,"dimensions")')
   WRITE(stdout,'(5x,"[MEM] ----------------------------------------------------------")')
   !
-  mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*nbnd_occ(1)
+  mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*((nbnd_occ(1)-1)/nbgrp+1)
   WRITE( stdout, '(5x,"[MEM] dvpsi                   ",f10.2," Mb", 5x,"(",i7,",",i5,")")') &
-     mem_partial, npwx*npol, nbnd_occ(1)
+     mem_partial, npwx*npol, ((nbnd_occ(1)-1)/nbgrp+1)
   IF( mpime == root ) CALL json%add( 'memory.dvpsi', mem_partial )
   mem_tot = mem_tot + mem_partial
   !
-  mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*nbnd_occ(1)
+  mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*((nbnd_occ(1)-1)/nbgrp+1)
   WRITE( stdout, '(5x,"[MEM] dpsi                    ",f10.2," Mb", 5x,"(",i7,",",i5,")")') &
-     mem_partial, npwx*npol, nbnd_occ(1)
+     mem_partial, npwx*npol, ((nbnd_occ(1)-1)/nbgrp+1)
   IF( mpime == root ) CALL json%add( 'memory.dpsi', mem_partial )
   mem_tot = mem_tot + mem_partial
   !

Wstat/wstat_readin.f90

 !
-! Copyright (C) 2015-2017 M. Govoni 
+! Copyright (C) 2015-2017 M. Govoni
 ! This file is distributed under the terms of the
 ! GNU General Public License. See the file `License'
 ! in the root directory of the present distribution,
@@ -7,48 +7,42 @@
 !
 ! This file is part of WEST.
 !
-! Contributors to this file: 
+! Contributors to this file:
 ! Marco Govoni
 !
 !-----------------------------------------------------------------------
 SUBROUTINE wstat_readin()
   !-----------------------------------------------------------------------
   !
-  USE ions_base,        ONLY : nat
   USE uspp,             ONLY : okvan
-  USE io_files,         ONLY : tmp_dir, prefix
-  USE io_global,        ONLY : stdout
-  USE noncollin_module, ONLY : noncolin
-  USE mp,               ONLY : mp_bcast
-  USE mp_world,         ONLY : nproc,mpime,root
   USE gvecs,            ONLY : doublegrid
   USE spin_orb,         ONLY : domag
+  USE mp_bands,         ONLY : nbgrp
+  USE funct,            ONLY : dft_is_hybrid
+  USE westcom,          ONLY : l_kinetic_only
   !
   IMPLICIT NONE
   !
-  ! Workspace
-  !
-  INTEGER :: iunit =5, ios
-  !
   CALL start_clock('wstat_readin')
   !
-  !CALL fetch_namelist(2,(/1,2/))
-  !
   ! READ INPUT_WEST
   !
   CALL fetch_input_yml(1,(/1/),.TRUE.,.FALSE.)
   !
-  !  read the input file produced by the pwscf program
-  !  allocate memory and recalculate what is needed
+  ! read the input file produced by the pwscf program
+  ! allocate memory and recalculate what is needed
   !
-  !CALL read_file( )
   CALL read_pwout( )
   !
   ! PW checks
   !
   IF (domag) CALL errore('wstat_readin','domag version not available',1)
   IF (okvan) CALL errore('wstat_readin','ultrasoft pseudopotential not implemented',1)
-  IF (doublegrid) CALL errore('wstat_readin', 'double grid not implemented',1)
+  IF (doublegrid) CALL errore('wstat_readin','double grid not implemented',1)
+  IF (nbgrp > 1) THEN
+     IF (dft_is_hybrid()) CALL errore('wstat_readin','band groups not implemented for hybrids',1)
+     IF (l_kinetic_only) CALL errore('wstat_readin','band groups not implemented for kinetic only',1)
+  ENDIF
   !
   ! READ other sections of the input file
   !

test-suite/maketest.inc

@@ -4,6 +4,7 @@
 
 export NP=2 # Number of MPI processes
 export NI=1 # Number of images
+export NB=1 # Number of band groups
 export NT=1 # Number of OpenMP threads
 
 #
@@ -25,5 +26,6 @@ export WGET=wget -N -q
 ###### DO NOT TOUCH BELOW ######
 
 export NIMAGE=${NI}
+export NBAND=${NB}
 export OMP_NUM_THREADS=${NT}
 

test-suite/test001/Makefile

@@ -15,7 +15,7 @@ pw:
 	${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
 
 wstat: pw
-	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
+	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
 
 wfreq: wstat
 	${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err

test-suite/test002/Makefile

@@ -15,7 +15,7 @@ pw:
 	${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
 
 wstat: pw
-	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
+	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
 
 wfreq: wstat
 	${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err

test-suite/test003/Makefile

@@ -15,7 +15,7 @@ pw:
 	${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
 
 wstat: pw
-	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
+	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
 
 wfreq: wstat
 	${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err

test-suite/test004/Makefile

@@ -15,7 +15,7 @@ pw:
 	${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
 
 wstat: pw
-	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
+	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
 
 wfreq: wstat
 	${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err

test-suite/test005/Makefile

@@ -15,7 +15,7 @@ pw:
 	${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
 
 wstat: pw
-	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
+	${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
 
 wfreq: wstat
 	${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err