Get BigDFT from
http://inac.cea.fr/L_Sim/BigDFT/
BigDFT Package developer documentation
http://inac.cea.fr/L_Sim/BigDFT/html-sources-1.5/
Building IBM/P (Surveyor & Intrepid)
Add to ~/.softenv
@ibm-compilers-aug2012
+mpiwrappers
Create and Run bigdft-bgp.sh
#! /bin/bash
##########################################
# BigDFT 1.7+ESSL+ScaLAPACK+OMP+XL-IBM
##########################################
# File bigdft-bgp.sh
LANG=C
export PATH=/soft/apps/fen/autoconf-2.65/bin:$PATH
export PATH=/soft/apps/fen/automake-1.9.6/bin:$PATH
export XL_TOP="/soft/apps/ibmcmp-aug2012"
export LIBS="-L."
export XL_TOP="/soft/apps/ibmcmp-aug2012"
export LIBS="${LIBS} -L${XL_TOP}/xlmass/bg/4.4/bglib -lmass -lmassv"
export LIBS="${LIBS} -L${XL_TOP}/bg/9.0/bglib -libmc++"
export LIBS="${LIBS} -L${XL_TOP}/xlf/bg/11.1/bglib -lxlf90_r -lxlopt -lxl"
export LIBS="${LIBS} -L${XL_TOP}/xlsmp/bg/1.7/bglib -lxlsmp "
export CPPFLAGS="-I."
export CPPFLAGS="${CPPFLAGS} -I/bgsys/drivers/ppcfloor/arch/include "
export CPPFLAGS="${CPPFLAGS} -I/bgsys/drivers/ppcfloor/comm/include "
#WITH OPENMP
export FCFLAGS=" -qsmp=omp -g -O3 -qxlf90=autodealloc -qessl -qtune=450 -qnoipa -qnohot -qstrict"
#export FFLAGS=" -O2 -qstrict -qxlf77=leadzero -qxlf77=leadzero -qnoipa -qnohot -qarch=450d -qtune=450 -qthreaded -qnosave -qalign=4k"
#PERFORMANCE
#export FCFLAGS=" -qsmp=omp -g -O5 -qxlf90=autodealloc -qessl -qtune=450 -qstrict"
#export FCFLAGS=" -O2 -qstrict -qxlf77=leadzero -qxlf77=leadzero -qnoipa -qnohot -qarch=450d -qtune=450 -qthreaded -qnosave -qalign=4k"
#export SAFEFFLAGS=" -O2 -qstrict -qmaxmem=-1 -qarch=450d -qtune=450 -qnoipa -qnohot"
#--with-netcdf-module-path=$NETCDF/include/ --with-netcdf-ldflags
./configure \
--prefix=$HOME/install/local \
--build=powerpc64-unknown-linux-gnu \
--with-ext-linalg=" -L/soft/apps/current/SCALAPACK/ -lscalapack_lb32 -L/soft/apps/current/BLACS/ -lblacsF77init_MPI-BGP-0 -lblacs_MPI-BGP-0 \
-L/soft/apps/LAPACK -llapack_bgp -L/soft/apps/ESSL-4.4.1-1/lib -lesslbg -Wl,--allow-multiple-definition" \
--without-yaml \
CFLAGS=-O2 \
CC=tmpixlc \
CXX=tmpixlcxx \
FC=tmpixlf90
Building IBM/Q [Scalapack+ESSL+OMP]
make sure you have at the end of ~/.soft
#FOR BIGDFT
+mpiwrapper-xl
@ibm-compilers-nov2012
#!/bin/sh
LANG=C
export PATH=/soft/buildtools/9july2012/bin:$PATH
export XL_TOP="/soft/compilers/ibmcmp-nov2012"
export LIBS="-L."
export LIBS="${LIBS} -L${XL_TOP}/xlf/bg/14.1/bglib64 -lxlf90_r -lxlopt -lxl "
export LIBS="${LIBS} -L${XL_TOP}/xlmass/bg/7.3/bglib64 -lmass "
export LIBS="${LIBS} -L${XL_TOP}/xlsmp/bg/3.1/bglib64 -lxlsmp -lgfortran -lpthread -ldl "
export CPPFLAGS="-I."
#performance
export FCFLAGS=" -g -O3 -qsmp=omp -qarch=qp -qtune=qp -qcache=auto -qunroll=auto "
export FCFLAGS+="-qfree=f90 -qxlf90=autodealloc -qessl -qstrict -qmaxmem=-1 -qsimd=noauto -qnoipa -qnohot"
export LD_LIBRARY_PATH=/soft/compilers/ibmcmp-nov2012/lib64/bg/bglib64:/bgsys/drivers/ppcfloor/comm/sys/lib:$LD_LIBRARY_PATH
./configure \
--prefix=$HOME/install/local \
--build=powerpc64-unknown-linux-gnu \
--with-ext-linalg-path="-L/soft/libraries/essl/current/lib64 -L/soft/libraries/alcf/current/xl/SCALAPACK/lib" \
--with-ext-linalg="-L/soft/libraries/alcf/current/xl/LAPACK/lib/ \
-L/soft/libraries/essl/current/lib64 \
-L/soft/libraries/alcf/current/xl/SCALAPACK/lib \
-llapack -lscala pack -lesslsmpbg -Wl,--allow-multiple-definition " \
--with-blas-path="-L/soft/libraries/essl/current/lib64" \
--without-yaml \
CFLAGS=" -g -O3 " \
CC=mpixlc \
CXX=mpixlcxx \
FC=mpixlf90
Submit a job as
qsub -t 20 -n 1024 --mode c4 --env OMP_NUM_THREADS=16 -A my_project-O ./bigdft
Building IBM/P (Julich)
#!/bin/bash
export PATH=/u/mjr/BlueGene/soft/gnu/bin:$PATH
module ()
{
eval `'/usr/bin/tclsh' '/opt/modules/init/modulecmd.tcl' 'bash' $*`
}
module load gsl;
module load lapack/3.3.0;
module load blacs;
module load scalapack;
module load xlf/11.1.0.12;
export CC=mpixlc_r
export FC=mpixlf90_r
export LIBS_BLAS=" -L/opt/ibmmath/essl/4.4/lib -lesslbg";
export CFLAGS="-qlanglvl=extc99 -O2 -qspill=2000 -qarch=450d -qtune=auto -qcache=auto -qstrict -qsup
press=1520-031:1520-003"
export FCFLAGS="-O2 -lxlopt -qarch=450d -qtune=auto -qstrict -qsuppress=1520-031:1520-003 -qmaxmem=-
1 -qcache=auto"
../configure --prefix=$HOME/Software/bigdft \
--with-blas-path=/opt/ibmmath/essl/4.4/lib \
--with-ext-linalg-path=" -L/soft/apps/LAPACK -L/soft/apps/ESSL-4.4.1-1/lib -Wl,--allow-multiple-definition"\
--with-ext-linalg=" -L/u/system/BlueGene/libs/lapack/lapack-3.3.0/lib -L/opt/ibmmath/essl/4.4/lib -llapack -lesslbg -Wl,--allow-multiple-definition"
Tukey
#Tukey cluster
#MVAPICH+MKL+OMP+GCC
export MKLROOT=/soft/x86_64/compilers/intel-cc-11.0.074/mkl/lib/em64t
export IMKLROOT=/soft/x86_64/compilers/intel-cc-11.0.074/mkl/include
./configure \
--with-ext-linalg-path="-L$MKLROOT" \
--with-yaml=no \
--with-ext-linalg=" $MKLROOT/libmkl_scalapack_lp64.a -Wl,--start-group $MKLROOT/libmkl_intel_lp64.a $MKLROOT/libmkl_gnu_thread.a \
$MKLROOT/libmkl_core.a $MKLROOT/libmkl_blacs_intelmpi_lp64.a -Wl,--end-group -ldl -lpthread -lm" \
CFLAGS=-O2 CC=gcc CXX=g++ FC=mpif90 F77=gfortran FCFLAGS="-O3 -fopenmp -m64 -I$IMKLROOT" CFLAGS="-O3 -g -fopenmp -DMKL_ILP64 -m64 -I$IMKLROOT"