Commit 8c9bdcef authored by Marco Govoni's avatar Marco Govoni

Merge branch 'dfpt_band_parallel' into 'develop'

Add distributed band parallelization to Wstat

See merge request west-devel/West!35
parents 6e13d0ca 9d17440e
......@@ -54,7 +54,7 @@ stages:
- make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
- make -j4 all
- cd test-suite
- make NP=$CI_NP NI=$CI_NI NT=$CI_NT
- make NP=$CI_NP NI=$CI_NI NB=$CI_NB NT=$CI_NT
artifacts:
when: on_failure
paths:
......@@ -83,6 +83,7 @@ gcc840_t:
variables:
CI_NP: 8
CI_NI: 1
CI_NB: 1
CI_NT: 1
extends:
- .template_bot_start
......@@ -92,7 +93,8 @@ gcc840_t:
gcc930_t:
variables:
CI_NP: 8
CI_NI: 1
CI_NI: 2
CI_NB: 2
CI_NT: 1
extends:
- .template_bot_start
......@@ -103,9 +105,10 @@ gcc930_t:
gcc840_t2:
variables:
CI_NP: 4
CI_NP: 8
CI_NI: 2
CI_NT: 1
CI_NB: 1
CI_NT: 2
only:
- schedules
extends:
......@@ -115,8 +118,9 @@ gcc840_t2:
gcc930_t2:
variables:
CI_NP: 2
CI_NI: 2
CI_NP: 8
CI_NI: 1
CI_NB: 2
CI_NT: 2
only:
- schedules
......
......@@ -9,6 +9,7 @@ MODFLAGS= $(MOD_FLAG)../../iotk/src $(MOD_FLAG)../../Modules $(MOD_FLAG)../../LA
$(MOD_FLAG)../Tools \
$(MOD_FLAG)../FFT_kernel \
$(MOD_FLAG)../Coulomb_kernel \
$(MOD_FLAG)../Para_kernel \
$(MOD_FLAG).
IFLAGS=
......
!
! Copyright (C) 2015-2017 M. Govoni
! Copyright (C) 2015-2017 M. Govoni
! This file is distributed under the terms of the
! GNU General Public License. See the file `License'
! in the root directory of the present distribution,
......@@ -7,7 +7,7 @@
!
! This file is part of WEST.
!
! Contributors to this file:
! Contributors to this file:
! Marco Govoni
!
!-----------------------------------------------------------------------
......@@ -40,7 +40,7 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
! input: the vector
! output: the operator applied to the vector
!
! Workspace
! Workspace
!
INTEGER :: ibnd, ig
COMPLEX(DP) :: za
......@@ -50,11 +50,15 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
! compute the product of the hamiltonian with the h vector
!
hpsi=(0.0_DP,0.0_DP)
!
!
IF(l_kinetic_only) THEN
CALL k_psi( npwx, npw, m, psi, hpsi )
ELSE
CALL h_psi( npwx, npw, m, psi, hpsi )
!
! use h_psi_, i.e. h_psi without band parallelization, as wstat
! handles band parallelization separately in dfpt_module
!
CALL h_psi_( npwx, npw, m, psi, hpsi )
ENDIF
!
! then we compute the operator H-epsilon S
......@@ -74,4 +78,4 @@ SUBROUTINE apply_sternheimerop_to_m_wfcs(nbndval, psi, hpsi, e, alpha, m)
!
CALL stop_clock ('stern')
!
END SUBROUTINE
END SUBROUTINE
This diff is collapsed.
......@@ -7,13 +7,9 @@ apply_sternheimerop_to_m_wfcs.o : ../../Modules/uspp.o
apply_sternheimerop_to_m_wfcs.o : ../../Modules/wavefunctions.o
apply_sternheimerop_to_m_wfcs.o : ../../PW/src/pwcom.o
apply_sternheimerop_to_m_wfcs.o : ../Modules/westcom.o
dfpt_module.o : ../../FFTXlib/fft_interfaces.o
dfpt_module.o : ../../Modules/cell_base.o
dfpt_module.o : ../../Modules/constants.o
dfpt_module.o : ../../Modules/control_flags.o
dfpt_module.o : ../../Modules/fft_base.o
dfpt_module.o : ../../Modules/gvecw.o
dfpt_module.o : ../../Modules/io_files.o
dfpt_module.o : ../../Modules/io_global.o
dfpt_module.o : ../../Modules/kind.o
dfpt_module.o : ../../Modules/mp.o
......@@ -28,6 +24,8 @@ dfpt_module.o : ../../PW/src/pwcom.o
dfpt_module.o : ../FFT_kernel/fft_at_gamma.o
dfpt_module.o : ../FFT_kernel/fft_at_k.o
dfpt_module.o : ../Modules/westcom.o
dfpt_module.o : ../Para_kernel/class_idistribute.o
dfpt_module.o : ../Para_kernel/distribution_center.o
dfpt_module.o : ../Tools/bar.o
dfpt_module.o : ../Tools/io_push.o
dfpt_module.o : ../Tools/types_bz_grid.o
......
......@@ -204,11 +204,8 @@ wfreq.o : ../../Modules/mp.o
wfreq.o : ../../Modules/mp_global.o
wfreq.o : ../Modules/westcom.o
wfreq.o : ../Tools/west_environment.o
wfreq_readin.o : ../../Modules/io_files.o
wfreq_readin.o : ../../Modules/io_global.o
wfreq_readin.o : ../../Modules/ions_base.o
wfreq_readin.o : ../../Modules/mp.o
wfreq_readin.o : ../../Modules/noncol.o
wfreq_readin.o : ../../Modules/funct.o
wfreq_readin.o : ../../Modules/mp_bands.o
wfreq_readin.o : ../../Modules/recvec.o
wfreq_readin.o : ../../Modules/uspp.o
wfreq_readin.o : ../../PW/src/pwcom.o
......
!
! Copyright (C) 2015-2017 M. Govoni
! Copyright (C) 2015-2017 M. Govoni
! This file is distributed under the terms of the
! GNU General Public License. See the file `License'
! in the root directory of the present distribution,
......@@ -7,47 +7,38 @@
!
! This file is part of WEST.
!
! Contributors to this file:
! Contributors to this file:
! Marco Govoni
!
!-----------------------------------------------------------------------
SUBROUTINE wfreq_readin()
!-----------------------------------------------------------------------
!
USE ions_base, ONLY : nat
USE uspp, ONLY : okvan
USE io_files, ONLY : tmp_dir, prefix
USE io_global, ONLY : stdout
USE noncollin_module, ONLY : noncolin
USE mp, ONLY : mp_bcast
USE gvecs, ONLY : doublegrid
USE spin_orb, ONLY : domag
USE mp_bands, ONLY : nbgrp
USE funct, ONLY : dft_is_hybrid
!
IMPLICIT NONE
!
! Workspace
!
INTEGER :: iunit =5, ios
!
CALL start_clock('wfreq_readin')
!
!CALL fetch_namelist(3,(/1,2,3/))
!
! READ INPUT_WEST
!
CALL fetch_input_yml(1,(/1/),.TRUE.,.FALSE.)
!
! read the input file produced by the pwscf program
! allocate memory and recalculate what is needed
! read the input file produced by the pwscf program
! allocate memory and recalculate what is needed
!
!CALL read_file( )
CALL read_pwout( )
!
! PW checks
!
IF (domag) CALL errore('wfreq_readin','domag version not available',1)
IF (okvan) CALL errore('wfreq_readin','ultrasoft pseudopotential not implemented',1)
IF (doublegrid) CALL errore('wfreq_readin', 'double grid not implemented',1)
IF (doublegrid) CALL errore('wfreq_readin','double grid not implemented',1)
IF (nbgrp > 1 .AND. dft_is_hybrid()) CALL errore('wfreq_readin','band groups not implemented for hybrids',1)
!
! READ other sections of the input file
!
......
......@@ -43,6 +43,7 @@ wstat_memory_report.o : ../../Modules/control_flags.o
wstat_memory_report.o : ../../Modules/fft_base.o
wstat_memory_report.o : ../../Modules/io_global.o
wstat_memory_report.o : ../../Modules/kind.o
wstat_memory_report.o : ../../Modules/mp_bands.o
wstat_memory_report.o : ../../Modules/mp_world.o
wstat_memory_report.o : ../../Modules/noncol.o
wstat_memory_report.o : ../../Modules/recvec.o
......@@ -51,15 +52,12 @@ wstat_memory_report.o : ../../PW/src/pwcom.o
wstat_memory_report.o : ../Libraries/Json/json_module.o
wstat_memory_report.o : ../Modules/westcom.o
wstat_memory_report.o : ../Para_kernel/distribution_center.o
wstat_readin.o : ../../Modules/io_files.o
wstat_readin.o : ../../Modules/io_global.o
wstat_readin.o : ../../Modules/ions_base.o
wstat_readin.o : ../../Modules/mp.o
wstat_readin.o : ../../Modules/mp_world.o
wstat_readin.o : ../../Modules/noncol.o
wstat_readin.o : ../../Modules/funct.o
wstat_readin.o : ../../Modules/mp_bands.o
wstat_readin.o : ../../Modules/recvec.o
wstat_readin.o : ../../Modules/uspp.o
wstat_readin.o : ../../PW/src/pwcom.o
wstat_readin.o : ../Modules/westcom.o
wstat_restart.o : ../../Modules/io_files.o
wstat_restart.o : ../../Modules/io_global.o
wstat_restart.o : ../../Modules/kind.o
......
......@@ -22,6 +22,7 @@ SUBROUTINE wstat_memory_report()
USE gvecs, ONLY : ngms
USE uspp, ONLY : nkb
USE control_flags, ONLY : gamma_only
USE mp_bands, ONLY : nbgrp
USE mp_world, ONLY : mpime,root
USE westcom, ONLY : nbnd_occ,n_pdep_basis,npwqx,logfile
USE distribution_center, ONLY : pert
......@@ -141,15 +142,15 @@ SUBROUTINE wstat_memory_report()
WRITE(stdout,'(5x,"[MEM] Allocated arrays ",5x,"est. size (Mb)", 5x,"dimensions")')
WRITE(stdout,'(5x,"[MEM] ----------------------------------------------------------")')
!
mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*nbnd_occ(1)
mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*((nbnd_occ(1)-1)/nbgrp+1)
WRITE( stdout, '(5x,"[MEM] dvpsi ",f10.2," Mb", 5x,"(",i7,",",i5,")")') &
mem_partial, npwx*npol, nbnd_occ(1)
mem_partial, npwx*npol, ((nbnd_occ(1)-1)/nbgrp+1)
IF( mpime == root ) CALL json%add( 'memory.dvpsi', mem_partial )
mem_tot = mem_tot + mem_partial
!
mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*nbnd_occ(1)
mem_partial = (1.0_DP/Mb)*complex_size*npwx*npol*((nbnd_occ(1)-1)/nbgrp+1)
WRITE( stdout, '(5x,"[MEM] dpsi ",f10.2," Mb", 5x,"(",i7,",",i5,")")') &
mem_partial, npwx*npol, nbnd_occ(1)
mem_partial, npwx*npol, ((nbnd_occ(1)-1)/nbgrp+1)
IF( mpime == root ) CALL json%add( 'memory.dpsi', mem_partial )
mem_tot = mem_tot + mem_partial
!
......
!
! Copyright (C) 2015-2017 M. Govoni
! Copyright (C) 2015-2017 M. Govoni
! This file is distributed under the terms of the
! GNU General Public License. See the file `License'
! in the root directory of the present distribution,
......@@ -7,48 +7,42 @@
!
! This file is part of WEST.
!
! Contributors to this file:
! Contributors to this file:
! Marco Govoni
!
!-----------------------------------------------------------------------
SUBROUTINE wstat_readin()
!-----------------------------------------------------------------------
!
USE ions_base, ONLY : nat
USE uspp, ONLY : okvan
USE io_files, ONLY : tmp_dir, prefix
USE io_global, ONLY : stdout
USE noncollin_module, ONLY : noncolin
USE mp, ONLY : mp_bcast
USE mp_world, ONLY : nproc,mpime,root
USE gvecs, ONLY : doublegrid
USE spin_orb, ONLY : domag
USE mp_bands, ONLY : nbgrp
USE funct, ONLY : dft_is_hybrid
USE westcom, ONLY : l_kinetic_only
!
IMPLICIT NONE
!
! Workspace
!
INTEGER :: iunit =5, ios
!
CALL start_clock('wstat_readin')
!
!CALL fetch_namelist(2,(/1,2/))
!
! READ INPUT_WEST
!
CALL fetch_input_yml(1,(/1/),.TRUE.,.FALSE.)
!
! read the input file produced by the pwscf program
! allocate memory and recalculate what is needed
! read the input file produced by the pwscf program
! allocate memory and recalculate what is needed
!
!CALL read_file( )
CALL read_pwout( )
!
! PW checks
!
IF (domag) CALL errore('wstat_readin','domag version not available',1)
IF (okvan) CALL errore('wstat_readin','ultrasoft pseudopotential not implemented',1)
IF (doublegrid) CALL errore('wstat_readin', 'double grid not implemented',1)
IF (doublegrid) CALL errore('wstat_readin','double grid not implemented',1)
IF (nbgrp > 1) THEN
IF (dft_is_hybrid()) CALL errore('wstat_readin','band groups not implemented for hybrids',1)
IF (l_kinetic_only) CALL errore('wstat_readin','band groups not implemented for kinetic only',1)
ENDIF
!
! READ other sections of the input file
!
......
......@@ -4,6 +4,7 @@
export NP=2 # Number of MPI processes
export NI=1 # Number of images
export NB=1 # Number of band groups
export NT=1 # Number of OpenMP threads
#
......@@ -25,5 +26,6 @@ export WGET=wget -N -q
###### DO NOT TOUCH BELOW ######
export NIMAGE=${NI}
export NBAND=${NB}
export OMP_NUM_THREADS=${NT}
......@@ -15,7 +15,7 @@ pw:
${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
wstat: pw
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
wfreq: wstat
${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err
......
......@@ -15,7 +15,7 @@ pw:
${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
wstat: pw
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
wfreq: wstat
${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err
......
......@@ -15,7 +15,7 @@ pw:
${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
wstat: pw
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
wfreq: wstat
${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err
......
......@@ -15,7 +15,7 @@ pw:
${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
wstat: pw
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
wfreq: wstat
${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err
......
......@@ -15,7 +15,7 @@ pw:
${PARA_PREFIX_QE} ${BINDIR}/pw.x -i pw.in > pw.out 2> pw.err
wstat: pw
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -i wstat.in > wstat.out 2> wstat.err
${PARA_PREFIX} ${BINDIR}/wstat.x -nimage ${NIMAGE} -nband ${NBAND} -i wstat.in > wstat.out 2> wstat.err
wfreq: wstat
${PARA_PREFIX} ${BINDIR}/wfreq.x -nimage ${NIMAGE} -i wfreq.in > wfreq.out 2> wfreq.err
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment