Revamp GitLab CI

* Use updated Docker images.
* Check numerical results and fail the CI if results don't match.
* For now pw.x is always run with 2 MPI processes. Using more
  processes leads to relatively small noises in the wavefunctions,
  which make a noticeable difference in Wstat, and even a larger
  difference in Wfreq. Why this is happening needs further
  investigation.
* Fix doc build with sphinx 3.5.0+.
  (See https://github.com/sphinx-doc/sphinx/issues/8885)

.gitlab-ci.yml

 variables:
- GIT_STRATEGY: none 
+  GIT_STRATEGY: none
+  QE_VERSION: qe-6.1.0
 
 stages:
- - build
- - test
- - doc
- 
+  - build
+  - test
+  - doc
+
 .template_bot_start:
   tags: [rcc,docker,node-01]
   image: miccomcenter/bot:$CI_JOB_NAME
-  before_script:    
+  before_script:
     - if [ -z $GCC_VERSION ]; then echo "not set"; else echo $GCC_VERSION; fi
     - if [ -z $LAPACK_VERSION ]; then echo "not set"; else echo $LAPACK_VERSION; fi
     - if [ -z $XERCES_VERSION ]; then echo "not set"; else echo $XERCES_VERSION; fi
@@ -17,299 +18,92 @@ stages:
     - if [ -z $MPICH_VERSION ]; then echo "not set"; else echo $MPICH_VERSION; fi
     - if [ -z $SCALAPACK_VERSION ]; then echo "not set"; else echo $SCALAPACK_VERSION; fi
     - if [ -z $FFTW_VERSION ]; then echo "not set"; else echo $FFTW_VERSION; fi
-    - if [ -z PYTHON_VERSION ]; then echo "not set"; else echo $PYTHON_VERSION; fi
-    - if [ -z PYTHON_PIP_VERSION ]; then echo "not set"; else echo $PYTHON_PIP_VERSION; fi
-    
+    - if [ -z $PYTHON_VERSION ]; then echo "not set"; else echo $PYTHON_VERSION; fi
+    - if [ -z $PYTHON_PIP_VERSION ]; then echo "not set"; else echo $PYTHON_PIP_VERSION; fi
+
 .template_build:
   stage: build
   script:
-    - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
+    - git clone -b $QE_VERSION --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
     - cd QEDIR
-    - git describe
-    - ./configure 
-    - make -j pw
+    - git describe --tags --always
+    - ./configure
+    - make -j8 pw
     - ls bin
     - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
     - cd West
-    - git describe
+    - git describe --tags --always
     - make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
     - make all
-    - ls ../bin 
-    
+    - ls ../bin
+
 .template_test:
   stage: test
   script:
-    - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
+    - git clone -b $QE_VERSION --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
     - cd QEDIR
-    - git describe
-    - ./configure 
-    - make -j pw
+    - git describe --tags --always
+    - ./configure
+    - make -j8 pw
     - ls bin
     - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-    - cd West 
+    - cd West
+    - git describe --tags --always
     - make conf PYT=python3 PYT_LDFLAGS="`python3-config --ldflags --embed`"
-    - make all > /dev/null
-    - useradd -m qe
-    - cp -r test-suite /home/qe
-    - su - qe
-    - cd test-suite 
-    - make NP=14 NI=1 NT=1 
-
-gcc650_0001:
- extends: 
-   - .template_bot_start
-   - .template_build
-  
-gcc650_0002:
- extends: 
-   - .template_bot_start
-   - .template_build
-
-gcc740_0001:
- extends: 
-   - .template_bot_start
-   - .template_build
- 
-gcc740_0002:
- extends: 
-   - .template_bot_start
-   - .template_build
-
-gcc830_0001:
- extends: 
-   - .template_bot_start
-   - .template_build
-
-gcc830_0002:
- extends: 
-   - .template_bot_start
-   - .template_build
- 
-gcc830_0001_t:
- extends: 
-   - .template_bot_start
-   - .template_test
- image: miccomcenter/bot:gcc830_0001
-
-gcc830_0002_t:
- extends: 
-   - .template_bot_start
-   - .template_test
- image: miccomcenter/bot:gcc830_0002
-
-#west_build_gcc:
-# tags: [rcc,docker,node-01]
-# stage: build
-# image: ubuntu:latest
-# before_script:
-#   - apt-get update > /dev/null
-#   - apt-get install -qq git > /dev/null 
-#   - apt-get install -qq build-essential gfortran wget python-pip python-dev > /dev/null
-#   - apt-get install -qq libopenmpi-dev openmpi-bin > /dev/null
-#   - apt-get install -qq libblas-dev liblapack-dev fftw3 fftw3-dev pkg-config > /dev/null
-# script:
-#   - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
-#   - cd QEDIR
-#   - git describe
-#   - ./configure
-#   - make -j pw
-#   - ls bin
-#   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-#   - cd West
-#   - git describe
-#   - make
-#   - ls ../bin 
-
-#west_build_intel:
-# tags: [rcc,docker,node-01]
-# stage: build
-# image: centos:centos7
-# before_script:
-#   - yum -y update > /dev/null
-#   - yum -y install yum-utils > /dev/null
-#   - yum -y install centos-release-scl-rh > /dev/null
-#   - yum -y install devtoolset-3 > /dev/null
-#   - yum -y install gcc gcc-c++ autoconf automake
-#   - yum -y install vim > /dev/null 
-#   - yum -y install wget pkgconfig git make > /dev/null
-#   - export TZ="US/Central"
-#   - rm -f /etc/localtime
-#   - cp  /usr/share/zoneinfo/$TZ  /etc/localtime
-#   - export PATH=/software/intel/impi/5.1.3.210/intel64/bin:/software/intel/parallel_studio_xe_2016_update3/bin:$PATH
-#   - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:/software/intel/impi/5.1.3.210/intel64/lib:$LD_LIBRARY_PATH
-#   - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2016_update3/ipp/lib/intel64:/software/intel/parallel_studio_xe_2016_update3/lib/intel64:$LD_LIBRARY_PATH
-#   - export LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:$LIBRARY_PATH
-#   - export CPATH=/software/intel/parallel_studio_xe_2016_update3/ipp/include:/software/intel/parallel_studio_xe_2017/mkl/include
-#   - export I_MPI_CC=icc
-#   - export I_MPI_CXX=icpc
-#   - export I_MPI_FC=ifort
-#   - export I_MPI_F77=ifort
-#   - export I_MPI_F90=ifort
-#   - export I_MPI_F95=ifort
-#   - export I_MPI_PROCESS_MANAGER=hydra
-#   - export I_MPI_ROOT=/software/intel/impi/5.1.3.210
-# script:
-#   - echo "$PATH"
-#   - echo "$LD_LIBRARY_PATH"
-#   - echo "$I_MPI_F90"
-#   - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
-#   - cd QEDIR
-#   - git describe
-#   - export F77=mpiifort
-#   - export CC=mpiicc
-#   - export MPIF90=mpiifort
-#   - export FC=mpiifort
-#   - which ifort
-#   - which icc
-#   - which mpiifort 
-#   - export CFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
-#   - export FFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
-#   - export DFLAGS="-D__DFTI -D__MPI -D__SCALAPACK"
-#   - export BLAS_LIBS_SWITCH="external"
-#   - export BLAS_LIBS=" -lmkl_intel_lp64  -lmkl_sequential -lmkl_core"
-#   - export LAPACK_LIBS_SWITCH="external"
-#   - export LAPACK_LIBS=""
-#   - export SCALAPACK_LIBS=" -lmkl_scalapack_lp64 -Wl,--start-group  -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -Wl,--end-group"
-#   - ./configure --enable-openmp --with-scalapack
-#   - make -j pw
-#   - ls bin
-#   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-#   - cd West
-#   - git describe
-#   - make
-#   - ls ../bin 
-#   - ldd ../bin/pw.x
-
-#west_test_intel:
-# tags: [rcc,docker,node-01]
-# stage: test
-# image: centos:centos7
-# before_script:
-#   - yum -y update > /dev/null
-#   - yum -y install yum-utils > /dev/null
-#   - yum -y install centos-release-scl-rh > /dev/null
-#   - yum -y install devtoolset-3 > /dev/null
-#   - yum -y install gcc gcc-c++ autoconf automake
-#   - yum -y install vim > /dev/null 
-#   - yum -y install wget pkgconfig git make > /dev/null
-#   - export TZ="US/Central"
-#   - rm -f /etc/localtime
-#   - cp  /usr/share/zoneinfo/$TZ  /etc/localtime
-#   - export PATH=/software/intel/impi/5.1.3.210/intel64/bin:/software/intel/parallel_studio_xe_2016_update3/bin:$PATH
-#   - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:/software/intel/impi/5.1.3.210/intel64/lib:$LD_LIBRARY_PATH
-#   - export LD_LIBRARY_PATH=/software/intel/parallel_studio_xe_2016_update3/ipp/lib/intel64:/software/intel/parallel_studio_xe_2016_update3/lib/intel64:$LD_LIBRARY_PATH
-#   - export LIBRARY_PATH=/software/intel/parallel_studio_xe_2017/mkl/lib/intel64:$LIBRARY_PATH
-#   - export CPATH=/software/intel/parallel_studio_xe_2016_update3/ipp/include:/software/intel/parallel_studio_xe_2017/mkl/include
-#   - export I_MPI_CC=icc
-#   - export I_MPI_CXX=icpc
-#   - export I_MPI_FC=ifort
-#   - export I_MPI_F77=ifort
-#   - export I_MPI_F90=ifort
-#   - export I_MPI_F95=ifort
-#   - export I_MPI_PROCESS_MANAGER=hydra
-#   - export I_MPI_ROOT=/software/intel/impi/5.1.3.210
-# script:
-#   - echo "$PATH"
-#   - echo "$LD_LIBRARY_PATH"
-#   - echo "$I_MPI_F90"
-#   - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
-#   - cd QEDIR
-#   - git describe
-#   - export F77=mpiifort
-#   - export CC=mpiicc
-#   - export MPIF90=mpiifort
-#   - export FC=mpiifort
-#   - which ifort
-#   - which icc
-#   - which mpiifort 
-#   - export CFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
-#   - export FFLAGS="-O3 -xHost -fno-alias -ansi-alias -g -mkl"
-#   - export DFLAGS="-D__DFTI -D__MPI -D__SCALAPACK"
-#   - export BLAS_LIBS_SWITCH="external"
-#   - export BLAS_LIBS=" -lmkl_intel_lp64  -lmkl_sequential -lmkl_core"
-#   - export LAPACK_LIBS_SWITCH="external"
-#   - export LAPACK_LIBS=""
-#   - export SCALAPACK_LIBS=" -lmkl_scalapack_lp64 -Wl,--start-group  -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -Wl,--end-group"
-#   - ./configure --enable-openmp --with-scalapack
-#   - make -j pw
-#   - ls bin
-#   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-#   - cd West
-#   - git describe
-#   - make
-#   - ls ../bin 
-#   - useradd -m qe
-#   - cp -r test-suite /home/qe
-#   - su - qe
-#   - cd test-suite 
-#   - make NP=14 NI=1 NT=1 
+    - make all
+    - cd test-suite
+    - make NP=8 NI=1 NT=1
+  artifacts:
+    when: on_failure
+    paths:
+      - QEDIR/West/test-suite/test*/*.out
+      - QEDIR/West/test-suite/test*/*.err
+      - QEDIR/West/test-suite/test*/*.tab
+      - QEDIR/West/test-suite/test*/test*/*.xml
+      - QEDIR/West/test-suite/test*/test*/*.json
+    expire_in: 3 days
 
-#west_test_gcc:
-# tags: [rcc,docker,node-01]
-# stage: test
-# image: ubuntu:latest
-# before_script:
-#   - apt-get update > /dev/null
-#   - apt-get install -qq git > /dev/null 
-#   - apt-get install -qq build-essential gfortran wget python-pip python-dev > /dev/null
-#   - apt-get install -qq libopenmpi-dev openmpi-bin > /dev/null
-#   - apt-get install -qq libblas-dev liblapack-dev fftw3 fftw3-dev pkg-config > /dev/null
-# script:
-#   - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
-#   - cd QEDIR
-#   - ./configure FFLAGS="-ffpe-summary=none" > /dev/null
-#   - make -j pw > /dev/null
-#   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-#   - cd West 
-#   - make > /dev/null
-#   - useradd -m qe
-#   - cp -r test-suite /home/qe
-#   - su - qe
-#   - cd test-suite 
-#   - make NP=14 NI=1 NT=1 
-   
-#gcc830_0001:
-# extends: .template_bot
-# script:
-#   - git clone -b 'qe-6.1.0' --single-branch --depth 1 https://gitlab.com/QEF/q-e.git QEDIR
-#   - cd QEDIR
-#   - git describe
-#   - ./configure
-#   - make -j pw
-#   - ls bin
-#   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-#   - cd West 
-#   - make > /dev/null
-#   - useradd -m qe
-#   - cp -r test-suite /home/qe
-#   - su - qe
-#   - cd test-suite 
-#   - make NP=14 NI=1 NT=1 
+gcc840:
+  extends:
+    - .template_bot_start
+    - .template_build
 
+gcc930:
+  extends:
+    - .template_bot_start
+    - .template_build
 
-#gcc830_0002:
-# extends: .template_bot
+gcc840_t:
+  extends:
+    - .template_bot_start
+    - .template_test
+  image: miccomcenter/bot:gcc840
 
+gcc930_t:
+  extends:
+    - .template_bot_start
+    - .template_test
+  image: miccomcenter/bot:gcc930
 
 west_doc_build:
- tags: [rcc,docker,node-01]
- stage: doc
- image: continuumio/miniconda3:latest
- before_script:
-   - apt-get update > /dev/null
-   - export TZ="US/Central"
-   - apt-get install -qq make > /dev/null
-   - apt-get install -qq pandoc > /dev/null
-   - rm -f /etc/localtime
-   - cp  /usr/share/zoneinfo/$TZ  /etc/localtime
-   - pip install -q --upgrade pip
-   - pip install -q nbsphinx
-   - pip install -q sphinx_rtd_theme
-   - pip install -q ipython
- script:
-   - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
-   - cd West
-   - git describe
-   - cd Doc
-   - make html
-   - ls _build/html
+  tags: [rcc,docker,node-01]
+  stage: doc
+  image: continuumio/miniconda3:latest
+  before_script:
+    - apt-get update > /dev/null
+    - export TZ="US/Central"
+    - apt-get install -qq make > /dev/null
+    - apt-get install -qq pandoc > /dev/null
+    - rm -f /etc/localtime
+    - cp /usr/share/zoneinfo/$TZ /etc/localtime
+    - pip install -q --upgrade pip
+    - pip install -q nbsphinx
+    - pip install -q sphinx_rtd_theme
+    - pip install -q ipython
+  script:
+    - git clone -b $CI_COMMIT_REF_NAME $CI_REPOSITORY_URL West
+    - cd West
+    - git describe --tags --always
+    - cd Doc
+    - make html
+    - ls _build/html

Doc/conf.py

@@ -157,11 +157,7 @@ html_logo = "images/logo.jpg"
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
-html_context = {
-    'css_files': [
-        '_static/theme_overrides.css',  # override wide tables in RTD theme
-        ],
-     }
+html_ccs_files = ['theme_overrides.css']  # override wide tables in RTD theme
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

test-suite/Makefile

 # Makefile
-#
+
 testdirs:= test001 test002 test003 test004 test005
 
 default: title all
+
 all:
-	for d in $(testdirs); do cd $$d; $(MAKE); [[ $$? != 0 ]] && exit -1; echo 'done'; cd ..; done
+	for d in $(testdirs); do cd $$d; $(MAKE); cd ..; done
+	python3 check.py
+
 clean:
 	for d in $(testdirs); do cd $$d; $(MAKE) clean; cd ..; done
 

test-suite/check.py

+import json
+import os
+import shutil
+import sys
+
+activeTests = [1,2,3,4,5]
+tolerance = 0.0001
+
+def check_pw(prefix):
+    # Check "JOB DONE"
+    fileName = prefix+'/pw.out'
+
+    if not os.path.isfile(fileName):
+        print(prefix+': pw.out not found')
+        return 1
+
+    ok = False
+    test = 0.0
+    with open(fileName,'r') as f:
+        for line in f:
+            if 'JOB DONE' in line:
+                ok = True
+
+            if '!    total energy' in line:
+                test = float(line.split()[4])
+
+    if not ok:
+        print(prefix+': pwscf failed')
+        return 1
+
+    # Remove stderr (may exceed artifact size limit)
+    fileName = prefix+'/pw.err'
+
+    if os.path.isfile(fileName):
+        os.remove(fileName)
+
+    # Load reference
+    fileName = prefix+'/ref/pw.ref'
+
+    with open(fileName,'r') as f:
+        for line in f:
+            if '!    total energy' in line:
+                ref = float(line.split()[4])
+
+    # Total energy error
+    absDiff = abs(ref-test)
+
+    if absDiff > tolerance:
+        print(prefix+': pwscf bad (diff '+"{:.2e}".format(absDiff)+')')
+        return 1
+    else:
+        print(prefix+': pwscf good (diff '+"{:.2e}".format(absDiff)+')')
+        return 0
+
+def check_wstat(prefix):
+    # Check "JOB DONE"
+    fileName = prefix+'/wstat.out'
+
+    if not os.path.isfile(fileName):
+        print(prefix+': wstat.out not found')
+        return 1
+
+    ok = False
+    with open(fileName,'r') as f:
+        for line in f:
+            if 'JOB DONE' in line:
+                ok = True
+                break
+
+    if not ok:
+        print(prefix+': wstat failed')
+        return 1
+
+    # Remove stderr (may exceed artifact size limit)
+    fileName = prefix+'/wstat.err'
+
+    if os.path.isfile(fileName):
+        os.remove(fileName)
+
+    # Load reference
+    fileName = prefix+'/ref/summary.json'
+
+    with open(fileName,'r') as f:
+        jsonData = json.load(f)
+
+    # Load test
+    fileName = prefix+'/test.wstat.save/summary.json'
+
+    if not os.path.isfile(fileName):
+        print(prefix+': wstat output not found')
+        return 1
+
+    ok = True
+    maxDiff = 0.0
+
+    with open(fileName,'r') as f:
+        jsonData2 = json.load(f)
+
+    for ii in range(len(jsonData['dielectric_matrix']['pdep'])):
+        ref = jsonData['dielectric_matrix']['pdep'][ii]['eigenval']
+        test = jsonData2['dielectric_matrix']['pdep'][ii]['eigenval']
+
+        for jj in range(len(ref)):
+            absDiff = abs(ref[jj]-test[jj])
+            maxDiff = max(maxDiff,absDiff)
+
+            if absDiff > tolerance:
+                ok = False
+
+    if ok:
+        print(prefix+': wstat good (max diff '+"{:.2e}".format(maxDiff)+')')
+        return 0
+    else:
+        print(prefix+': wstat bad (max diff '+"{:.2e}".format(maxDiff)+')')
+        return 1
+
+def check_wfreq(prefix):
+    # Check "JOB DONE"
+    fileName = prefix+'/wfreq.out'
+
+    if not os.path.isfile(fileName):
+        print(prefix+': wfreq.out not found')
+        return 1
+
+    ok = False
+    with open(fileName,'r') as f:
+        for line in f:
+            if 'JOB DONE' in line:
+                ok = True
+                break
+
+    if not ok:
+        print(prefix+': wfreq failed')
+        return 1
+
+    # Remove stderr (may exceed artifact size limit)
+    fileName = prefix+'/wfreq.err'
+
+    if os.path.isfile(fileName):
+        os.remove(fileName)
+
+    # Load reference
+    fileName = prefix+'/ref/wfreq.json'
+
+    with open(fileName,'r') as f:
+        jsonData = json.load(f)
+
+    # Load test
+    fileName = prefix+'/test.wfreq.save/wfreq.json'
+
+    if not os.path.isfile(fileName):
+        print(prefix+': wfreq output not found')
+        return 1
+
+    ok = True
+    maxDiff = 0.0
+
+    with open(fileName,'r') as f:
+        jsonData2 = json.load(f)
+
+    for ii in range(jsonData['system']['electron']['nkstot']):
+        ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmax']
+        test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmax']
+
+        for jj in range(len(ref)):
+            absDiff = abs(ref[jj]-test[jj])
+            maxDiff = max(maxDiff,absDiff)
+
+            if absDiff > tolerance:
+                ok = False
+
+        ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['re']
+        test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['re']
+
+        for jj in range(len(ref)):
+            absDiff = abs(ref[jj]-test[jj])
+            maxDiff = max(maxDiff,absDiff)
+
+            if absDiff > tolerance:
+                ok = False
+
+        ref = jsonData['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['im']
+        test = jsonData2['output']['Q']['K'+str(ii+1).zfill(6)]['sigmac_eqpSec']['im']
+
+        for jj in range(len(ref)):
+            absDiff = abs(ref[jj]-test[jj])
+            maxDiff = max(maxDiff,absDiff)
+
+            if absDiff > tolerance:
+                ok = False
+
+    if ok:
+        print(prefix+': wfreq good (max diff '+"{:.2e}".format(maxDiff)+')')
+        return 0
+    else:
+        print(prefix+': wfreq bad (max diff '+"{:.2e}".format(maxDiff)+')')
+        return 1
+
+if __name__ == "__main__":
+    ok = True
+
+    for ii in activeTests:
+        print()
+
+        prefix = 'test'+str(ii).zfill(3)
+
+        err = check_pw(prefix)
+        if err != 0:
+            ok = False
+
+        err = check_wstat(prefix)
+        if err != 0:
+            ok = False
+
+        err = check_wfreq(prefix)