Commit 22643095 authored by Victor Yu's avatar Victor Yu
Browse files

Revamp GitLab CI

* Use updated Docker images.
* Check numerical results and fail the CI if results don't match.
* For now pw.x is always run with 2 MPI processes. Using more
  processes leads to relatively small noises in the wavefunctions,
  which make a noticeable difference in Wstat, and even a larger
  difference in Wfreq. Why this is happening needs further
  investigation.
* Fix doc build with sphinx 3.5.0+.
  (See https://github.com/sphinx-doc/sphinx/issues/8885)
parent 1e98e7f1
variables:
GIT_STRATEGY: none
GIT_STRATEGY: none
QE_VERSION: qe-6.1.0
stages:
- build
- test
- doc
- build
- test
- doc
.template_bot_start:
tags: [rcc,docker,node-01]
image: miccomcenter/bot:$CI_JOB_NAME
before_script:
before_script:
- if [ -z $GCC_VERSION ]; then echo "not set"; else echo $GCC_VERSION; fi
- if [ -z $LAPACK_VERSION ]; then echo "not set"; else echo $LAPACK_VERSION; fi
- if [ -z $XERCES_VERSION ]; then echo "not set"; else echo $XERCES_VERSION; fi
......@@ -17,299 +18,92 @@ stages:
- if [ -z $MPICH_VERSION ]; then echo "not set"; else echo $MPICH_VERSION; fi
- if [ -z $SCALAPACK_VERSION ]; then echo "not set"; else echo $SCALAPACK_VERSION; fi
- if [ -z $FFTW_VERSION ]; then echo "not set"; else echo $FFTW_VERSION; fi
- if [ -z PYTHON_VERSION ]; then echo "not set"; else echo $PYTHON_VERSION; fi
- if [ -z PYTHON_PIP_VERSION ]; then echo "not set"; else echo $PYTHON_PIP_VERSION; fi
- if [ -z $PYTHON_VERSION ]; then echo "not set"; else echo $PYTHON_VERSION; fi
- if [ -z $PYTHON_PIP_VERSION ]; then echo "not set"; else echo $PYTHON_PIP_VERSION; fi
.template_build:
stage: build
script:
- git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
- git clone -b $QE_VERSION --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
- cd QEDIR
- git describe
- ./configure
- make -j pw
- git describe --tags --always
- ./configure
- make -j8 pw
- ls bin
- git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
- cd West
- git describe
- git describe --tags --always
- make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
- make all
- ls ../bin
- ls ../bin
.template_test:
stage: test
script:
- git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
- git clone -b $QE_VERSION --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
- cd QEDIR
- git describe
- ./configure
- make -j pw
- git describe --tags --always
- ./configure
- make -j8 pw
- ls bin
- git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
- cd West
- cd West
- git describe --tags --always
- make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
- make all > /dev/null
- useradd -m qe
- cp -r test-suite /home/qe
- su - qe
- cd test-suite
- make NP=14 NI=1 NT=1
gcc650_0001:
extends:
- .template_bot_start
- .template_build
gcc650_0002:
extends:
- .template_bot_start
- .template_build
gcc740_0001:
extends:
- .template_bot_start
- .template_build
gcc740_0002:
extends:
- .template_bot_start
- .template_build
gcc830_0001:
extends:
- .template_bot_start
- .template_build
gcc830_0002:
extends:
- .template_bot_start
- .template_build
gcc830_0001_t:
extends:
- .template_bot_start
- .template_test
image: miccomcenter/bot:gcc830_0001
gcc830_0002_t:
extends:
- .template_bot_start
- .template_test
image: miccomcenter/bot:gcc830_0002
#west_build_gcc:
# tags: [rcc,docker,node-01]
# stage: build
# image: ubuntu:latest
# before_script:
# - apt-get update > /dev/null
# - apt-get install -qq git > /dev/null
# - apt-get install -qq build-essential gfortran wget python-pip python-dev > /dev/null
# - apt-get install -qq libopenmpi-dev openmpi-bin > /dev/null
# - apt-get install -qq libblas-dev liblapack-dev fftw3 fftw3-dev pkg-config > /dev/null
# script:
# - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
# - cd QEDIR
# - git describe
# - ./configure
# - make -j pw
# - ls bin
# - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
# - cd West
# - git describe
# - make
# - ls ../bin
#west_build_intel:
# tags: [rcc,docker,node-01]
# stage: build
# image: centos:centos7
# before_script:
# - yum -y update > /dev/null
# - yum -y install yum-utils > /dev/null
# - yum -y install centos-release-scl-rh > /dev/null
# - yum -y install devtoolset-3 > /dev/null
# - yum -y install gcc gcc-c++ autoconf automake
# - yum -y install vim > /dev/null
# - yum -y install wget pkgconfig git make > /dev/null
# - export TZ="US/Central"
# - rm -f /etc/localtime
# - cp /usr/share/zoneinfo/$TZ /etc/localtime
# - export PATH=/software/intel/impi/5.1.3.210/intel64/bin:/software/intel/parallel_studio_xe_2016_update3/bin:$PATH
# - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:/software/intel/impi/5.1.3.210/intel64/lib:$LD_LIBRARY_PATH
# - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2016_update3/ipp/lib/intel64:/software/intel/parallel_studio_xe_2016_update3/lib/intel64:$LD_LIBRARY_PATH
# - export LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:$LIBRARY_PATH
# - export CPATH=/software/intel/parallel_studio_xe_2016_update3/ipp/include:/software/intel/parallel_studio_xe_2017/mkl/include
# - export I_MPI_CC=icc
# - export I_MPI_CXX=icpc
# - export I_MPI_FC=ifort
# - export I_MPI_F77=ifort
# - export I_MPI_F90=ifort
# - export I_MPI_F95=ifort
# - export I_MPI_PROCESS_MANAGER=hydra
# - export I_MPI_ROOT=/software/intel/impi/5.1.3.210
# script:
# - echo "$PATH"
# - echo "$LD_LIBRARY_PATH"
# - echo "$I_MPI_F90"
# - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
# - cd QEDIR
# - git describe
# - export F77=mpiifort
# - export CC=mpiicc
# - export MPIF90=mpiifort
# - export FC=mpiifort
# - which ifort
# - which icc
# - which mpiifort
# - export CFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
# - export FFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
# - export DFLAGS="-D__DFTI -D__MPI -D__SCALAPACK"
# - export BLAS_LIBS_SWITCH="external"
# - export BLAS_LIBS=" -lmkl_intel_lp64 -lmkl_sequential -lmkl_core"
# - export LAPACK_LIBS_SWITCH="external"
# - export LAPACK_LIBS=""
# - export SCALAPACK_LIBS=" -lmkl_scalapack_lp64 -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -Wl,--end-group"
# - ./configure --enable-openmp --with-scalapack
# - make -j pw
# - ls bin
# - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
# - cd West
# - git describe
# - make
# - ls ../bin
# - ldd ../bin/pw.x
#west_test_intel:
# tags: [rcc,docker,node-01]
# stage: test
# image: centos:centos7
# before_script:
# - yum -y update > /dev/null
# - yum -y install yum-utils > /dev/null
# - yum -y install centos-release-scl-rh > /dev/null
# - yum -y install devtoolset-3 > /dev/null
# - yum -y install gcc gcc-c++ autoconf automake
# - yum -y install vim > /dev/null
# - yum -y install wget pkgconfig git make > /dev/null
# - export TZ="US/Central"
# - rm -f /etc/localtime
# - cp /usr/share/zoneinfo/$TZ /etc/localtime
# - export PATH=/software/intel/impi/5.1.3.210/intel64/bin:/software/intel/parallel_studio_xe_2016_update3/bin:$PATH
# - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:/software/intel/impi/5.1.3.210/intel64/lib:$LD_LIBRARY_PATH
# - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2016_update3/ipp/lib/intel64:/software/intel/parallel_studio_xe_2016_update3/lib/intel64:$LD_LIBRARY_PATH
# - export LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:$LIBRARY_PATH
# - export CPATH=/software/intel/parallel_studio_xe_2016_update3/ipp/include:/software/intel/parallel_studio_xe_2017/mkl/include
# - export I_MPI_CC=icc
# - export I_MPI_CXX=icpc
# - export I_MPI_FC=ifort
# - export I_MPI_F77=ifort
# - export I_MPI_F90=ifort
# - export I_MPI_F95=ifort
# - export I_MPI_PROCESS_MANAGER=hydra
# - export I_MPI_ROOT=/software/intel/impi/5.1.3.210
# script:
# - echo "$PATH"
# - echo "$LD_LIBRARY_PATH"
# - echo "$I_MPI_F90"
# - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
# - cd QEDIR
# - git describe
# - export F77=mpiifort
# - export CC=mpiicc
# - export MPIF90=mpiifort
# - export FC=mpiifort
# - which ifort
# - which icc
# - which mpiifort
# - export CFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
# - export FFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
# - export DFLAGS="-D__DFTI -D__MPI -D__SCALAPACK"
# - export BLAS_LIBS_SWITCH="external"
# - export BLAS_LIBS=" -lmkl_intel_lp64 -lmkl_sequential -lmkl_core"
# - export LAPACK_LIBS_SWITCH="external"
# - export LAPACK_LIBS=""
# - export SCALAPACK_LIBS=" -lmkl_scalapack_lp64 -Wl,--start-group -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -Wl,--end-group"
# - ./configure --enable-openmp --with-scalapack
# - make -j pw
# - ls bin
# - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
# - cd West
# - git describe
# - make
# - ls ../bin
# - useradd -m qe
# - cp -r test-suite /home/qe
# - su - qe
# - cd test-suite
# - make NP=14 NI=1 NT=1
- make all
- cd test-suite
- make NP=8 NI=1 NT=1
artifacts:
when: on_failure
paths:
- QEDIR/West/test-suite/test*/*.out
- QEDIR/West/test-suite/test*/*.err
- QEDIR/West/test-suite/test*/*.tab
- QEDIR/West/test-suite/test*/test*/*.xml
- QEDIR/West/test-suite/test*/test*/*.json
expire_in: 3 days
#west_test_gcc:
# tags: [rcc,docker,node-01]
# stage: test
# image: ubuntu:latest
# before_script:
# - apt-get update > /dev/null
# - apt-get install -qq git > /dev/null
# - apt-get install -qq build-essential gfortran wget python-pip python-dev > /dev/null
# - apt-get install -qq libopenmpi-dev openmpi-bin > /dev/null
# - apt-get install -qq libblas-dev liblapack-dev fftw3 fftw3-dev pkg-config > /dev/null
# script:
# - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
# - cd QEDIR
# - ./configure FFLAGS="-ffpe-summary=none" > /dev/null
# - make -j pw > /dev/null
# - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
# - cd West
# - make > /dev/null
# - useradd -m qe
# - cp -r test-suite /home/qe
# - su - qe
# - cd test-suite
# - make NP=14 NI=1 NT=1
#gcc830_0001:
# extends: .template_bot
# script:
# - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
# - cd QEDIR
# - git describe
# - ./configure
# - make -j pw
# - ls bin
# - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
# - cd West
# - make > /dev/null
# - useradd -m qe
# - cp -r test-suite /home/qe
# - su - qe
# - cd test-suite
# - make NP=14 NI=1 NT=1
gcc840:
extends:
- .template_bot_start
- .template_build
gcc930:
extends:
- .template_bot_start
- .template_build
#gcc830_0002:
# extends: .template_bot
gcc840_t:
extends:
- .template_bot_start
- .template_test
image: miccomcenter/bot:gcc840
gcc930_t:
extends:
- .template_bot_start
- .template_test
image: miccomcenter/bot:gcc930
west_doc_build:
tags: [rcc,docker,node-01]
stage: doc
image: continuumio/miniconda3:latest
before_script:
- apt-get update > /dev/null
- export TZ="US/Central"
- apt-get install -qq make > /dev/null
- apt-get install -qq pandoc > /dev/null
- rm -f /etc/localtime
- cp /usr/share/zoneinfo/$TZ /etc/localtime
- pip install -q --upgrade pip
- pip install -q nbsphinx
- pip install -q sphinx_rtd_theme
- pip install -q ipython
script:
- git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
- cd West
- git describe
- cd Doc
- make html
- ls _build/html
tags: [rcc,docker,node-01]
stage: doc
image: continuumio/miniconda3:latest
before_script:
- apt-get update > /dev/null
- export TZ="US/Central"
- apt-get install -qq make > /dev/null
- apt-get install -qq pandoc > /dev/null
- rm -f /etc/localtime
- cp /usr/share/zoneinfo/$TZ /etc/localtime
- pip install -q --upgrade pip
- pip install -q nbsphinx
- pip install -q sphinx_rtd_theme
- pip install -q ipython
script:
- git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
- cd West
- git describe --tags --always
- cd Doc
- make html
- ls _build/html
......@@ -157,11 +157,7 @@ html_logo = "images/logo.jpg"
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_context = {
'css_files': [
'_static/theme_overrides.css', # override wide tables in RTD theme
],
}
html_ccs_files = ['theme_overrides.css'] # override wide tables in RTD theme
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
......
# Makefile
#
testdirs:= test001 test002 test003 test004 test005
default: title all
all:
for d in $(testdirs); do cd $$d; $(MAKE); [[ $$? != 0 ]] && exit -1; echo 'done'; cd ..; done
for d in $(testdirs); do cd $$d; $(MAKE); cd ..; done
python3 check.py
clean:
for d in $(testdirs); do cd $$d; $(MAKE) clean; cd ..; done
......
import json
import os
import shutil
import sys
activeTests = [1,2,3,4,5]
tolerance = 0.0001
def check_pw(prefix):
# Check "JOB DONE"
fileName = prefix+'/pw.out'
if not os.path.isfile(fileName):
print(prefix+': pw.out not found')
return 1
ok = False
test = 0.0
with open(fileName,'r') as f:
for line in f:
if 'JOB DONE' in line:
ok = True
if '! total energy' in line:
test = float(line.split()[4])
if not ok:
print(prefix+': pwscf failed')
return 1
# Remove stderr (may exceed artifact size limit)
fileName = prefix+'/pw.err'
if os.path.isfile(fileName):
os.remove(fileName)
# Load reference
fileName = prefix+'/ref/pw.ref'
with open(fileName,'r') as f:
for line in f:
if '! total energy' in line:
ref = float(line.split()[4])
# Total energy error
absDiff = abs(ref-test)
if absDiff > tolerance:
print(prefix+': pwscf bad (diff '+"{:.2e}".format(absDiff)+')')
return 1
else:
print(prefix+': pwscf good (diff '+"{:.2e}".format(absDiff)+')')
return 0
def check_wstat(prefix):
# Check "JOB DONE"
fileName = prefix+'/wstat.out'
if not os.path.isfile(fileName):
print(prefix+': wstat.out not found')
return 1
ok = False
with open(fileName,'r') as f:
for line in f:
if 'JOB DONE' in line:
ok = True
break
if not ok:
print(prefix+': wstat failed')
return 1
# Remove stderr (may exceed artifact size limit)
fileName = prefix+'/wstat.err'
if os.path.isfile(fileName):
os.remove(fileName)
# Load reference
fileName = prefix+'/ref/summary.json'
with open(fileName,'r') as f:
jsonData = json.load(f)
# Load test
fileName = prefix+'/test.wstat.save/summary.json'
if not os.path.isfile(fileName):
print(prefix+': wstat output not found')
return 1
ok = True
maxDiff = 0.0
with open(fileName,'r') as f:
jsonData2 = json.load(f)
for ii in range(len(jsonData['dielectric_matrix']['pdep'])):
ref = jsonData['dielectric_matrix']['pdep'][ii]['eigenval']
test = jsonData2['dielectric_matrix']['pdep'][ii]['eigenval']
for jj in range(len(ref)):
absDiff = abs(ref[jj]-test[jj])
maxDiff = max(maxDiff,absDiff)
if absDiff > tolerance:
ok = False
if ok:
print(prefix+': wstat good (max diff '+"{:.2e}".format(maxDiff)+')')
return 0
else:
print(prefix+': wstat bad (max diff '+"{:.2e}".format(maxDiff)+')')
return 1
def check_wfreq(prefix):
# Check "JOB DONE"
fileName = prefix+'/wfreq.out'
if not os.path.isfile(fileName):
print(prefix+': wfreq.out not found')
return 1
ok = False
with open(fileName,'r') as f:
for line in f:
if 'JOB DONE' in line:
ok = True
break
if not ok:
print(prefix+': wfreq failed')
return 1
# Remove stderr (may exceed artifact size limit)
fileName = prefix+'/wfreq.err'
if os.path.isfile(fileName):
os.remove(fileName)
# Load reference
fileName = prefix+'/ref/wfreq.json'
with open(fileName,'r') as f:
jsonData = json.load(f)
# Load test
fileName = prefix+'/test.wfreq.save/wfreq.json'
if not os.path.isfile(fileName):
print(prefix+': wfreq output not found')
return 1
ok = True
maxDiff = 0.0
with open(fileName,'r') as f:
jsonData2 = json.load(f)
for ii in range(jsonData['system']['electron']['nkstot']):
ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmax']
test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmax']
for jj in range(len(ref)):
absDiff = abs(ref[jj]-test[jj])
maxDiff = max(maxDiff,absDiff)
if absDiff > tolerance:
ok = False
ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['re']
test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['re']
for jj in range(len(ref)):
absDiff = abs(ref[jj]-test[jj])
maxDiff = max(maxDiff,absDiff)
if absDiff > tolerance:
ok = False
ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['im']
test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['im']
for jj in range(len(ref)):
absDiff = abs(ref[jj]-test[jj])
maxDiff = max(maxDiff,absDiff)
if absDiff > tolerance:
ok = False
if ok:
print(prefix+': wfreq good (max diff '+"{:.2e}".format(maxDiff)+')')
return 0
else:
print(prefix+': wfreq bad (max diff '+"{:.2e}".format(maxDiff)+')')
return 1
if __name__ == "__main__":
ok = True
for ii in activeTests:
print()
prefix = 'test'+str(ii).zfill(3)
err = check_pw(prefix)
if err != 0:
ok = False
err = check_wstat(prefix)
if err != 0:
ok = False
err = check_wfreq(prefix)