NWCHEM BGP

Jeff Hammond 's Wiki Page regarding his experience of NWChem 5.1.1 in BGP

https://wiki.alcf.anl.gov/index.php/NWChem

Jeff's effort for profiling NWChem with TAU tools

https://wiki.alcf.anl.gov/old/index.php/NWChem_TAU

Performance characterization of global address space applications: a case study with NWChem

JR Hammond, S Krishnamoorthy, S Shende, NA Romero, AD Malony

http://onlinelibrary.wiley.com/doi/10.1002/cpe.1881/abstract

Compiling NWChem 6.0 in BGP (ALCF Systems)

I compiled the June snapshot SVN developers version of the NWChem (avaliable in the 'Download' area of the main site). In this exercise I am not using any external linear algebra package ( such ESSL or BLAS+LAPACK),

Also I changed the optimization level to -O2 and I added the -qstrict flag

Changes in NWCHEM_TOP/src/config/makefile.h

ifeq ($(TARGET),$(findstring $(TARGET),BGL BGP))

CORE_SUBDIRS_EXTRA = lapack blas

ARFLAGS = urs

INSTALL = @echo $@ is built

EXPLICITF = TRUE

CPP=/usr/bin/cpp -P -C -traditional

FCONVERT = $(CPP) $(CPPFLAGS) $< > $*.f

LDOPTIONS = -Wl,--relax

DEFINES = -DLINUX -DXLFLINUX

FDEBUG = -g -O2

COPTIMIZE = -g -O2 -qstrict

FOPTIMIZE = -g -O2 -qnoipa -qfloat=rsqrt:fltint

FOPTIONS = -qstrict -qEXTNAME -qxlf77=leadzero

#for BGL

# ifeq ($(FC),blrts_xlf)

ifeq ($(TARGET),BGL)

CC = $(BGCOMPILERS)/powerpc-bgl-blrts-gnu-gcc

AR = $(BGCOMPILERS)/powerpc-bgl-blrts-gnu-ar

AS = $(BGCOMPILERS)/powerpc-bgl-blrts-gnu-as

RANLIB = $(BGCOMPILERS)/powerpc-bgl-blrts-gnu-ranlib

DEFINES += -DBGL

FOPTIMIZE += -qarch=440 -qtune=440

endif

#for BGP

# ifeq ($(FC),$(findstring $(FC),bgxlf bgxlf_r mpixlf77_r tmpixlf77_r))

ifeq ($(TARGET),BGP)

DEFINES += -DDCMF -DBGP

FOPTIONS += -qthreaded -qnosave -qalign=4k

FOPTIMIZE += -qarch=450d -qtune=450

COPTIMIZE += -qarch=450 -qtune=450

XLF11 = $(shell bgxlf -qversion 2>&1|grep Version|head -1| awk ' / 11./ {print "Y"}')

endif

Bash Shell Environmental variables used before compilation:

export TARGET=BGP

export ARMCI_NETWORK=DCMFMPI

export MSG_COMMS=DCMFMPI

export USE_MPI=yes

export BGP_INSTALLDIR=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc

export BGCOMPILERS=${BGP_INSTALLDIR}/gnu-linux/bin

export BGP_RUNTIMEPATH=${BGP_INSTALLDIR}/runtime

export ARMCIDRV=${BGP_INSTALLDIR}

export BGDRIVER=${BGP_INSTALLDIR}

export MPI_LIB=${BGDRIVER}/comm/lib

export MPI_INCLUDE=${BGDRIVER}/comm/include

export LIBMPI="-L${MPI_LIB} -lfmpich_.cnk -lmpich.cnk -ldcmf.cnk -ldcmfcoll.cnk -lpthread -lrt -L${BGP_RUNTIMEPATH}/SPI -lSPI.cna"

# ================================================

# NWChem Settings

# ================================================

export FC=bgxlf_r

export PATH=${BGP_INSTALLDIR}/gnu-linux/bin:$PATH

export CC=powerpc-bgp-linux-gcc

export AR=powerpc-bgp-linux-ar

export AS=powerpc-bgp-linux-as

export RANLIB=powerpc-bgp-linux-ranlib

export alias make='gmake'

export NWCHEM_TARGET=BGP

export NWCHEM_BASIS_LIBRARY="/home/avazquez/soft/nwchem/nwchem-6.0/src/basis/libraries"

export NWCHEM_TOP="/home/avazquez/soft/nwchem/nwchem-6.0"

export NWCHEM_MODULES="ddscf nwdft driver gradients property hessian vib cphf dplot"

export LARGE_FILES=TRUE

# ================================================

# NO BLAS Settings

# ================================================

export USE_64TO32=y

export CPPFLAGS=" "

export CPPFLAGS="${CPPFLAGS} -I/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/arch/include "

export CPPFLAGS="${CPPFLAGS} -I/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/comm/include "

export CPPFLAGS="${CPPFLAGS} -I/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/arch/include "

export LIBS="-L/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/comm/lib"

export LIBS="${LIBS} -L/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/comm/lib -lfmpich_.cnk -lmpich.cnk -ldcmf.cnk -ldcmfcoll.cnk "

export LIBS="${LIBS} -L/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/runtime/SPI -lSPI.cna "

export LIBS="${LIBS} -L/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/gnu-linux/powerpc-bgp-linux/lib -lpthread -lrt "

export FFLAGS="-I/bgsys/drivers/V1R4M2_200_2010-100508P/ppc/comm/include "

export CFLAGS=$CPPFLAGS

export LIBSTOOLS=$LIBS

export CPPFLAGSTOOLS=$CPPFLAGS

export FFLAGSTOOLS=$FFLAGS

export USE_SUBGROUPS=yes

Compilation command used

make FC=bgxlf_r CC=powerpc-bgp-linux-gcc

Compiling NWChem 6.1 in BGP (ALCF Systems)

I compiled successfully the version 6.1, some modules did not work for me (ie. plane waves). See how to compile Global Arrays in the next section

see changes to NWCHEM_TOP/src/config/makefile.h I suggested for nwchem 6.0.

This is my variables file vars.4.nw61.sh

export TARGET=BGP

export ARMCI_NETWORK=DCMFMPI

export MSG_COMMS=DCMFMPI

export USE_MPI=yes

export BGP_INSTALLDIR=/bgsys/drivers/V1R4M2_200_2010-100508P/ppc

export BGCOMPILERS=${BGP_INSTALLDIR}/gnu-linux/bin

export BGP_RUNTIMEPATH=${BGP_INSTALLDIR}/runtime

export ARMCIDRV=${BGP_INSTALLDIR}

export BGDRIVER=${BGP_INSTALLDIR}

export MPI_LIB=${BGDRIVER}/comm/lib

export MPI_INCLUDE=${BGDRIVER}/comm/include

export LIBMPI="-L${MPI_LIB} -lfmpich_.cnk -lmpich.cnk -ldcmf.cnk -ldcmfcoll.cnk -lpthread -lrt -L${BGP_RUNTIMEPATH}/SPI -lSPI.cna"

# ================================================

# NWChem Settings

# ================================================

export FC=bgxlf_r

export PATH=${BGP_INSTALLDIR}/gnu-linux/bin:$PATH

export CC=powerpc-bgp-linux-gcc

export AR=powerpc-bgp-linux-ar

export AS=powerpc-bgp-linux-as

export RANLIB=powerpc-bgp-linux-ranlib

export alias make='gmake'

export NWCHEM_TARGET=BGP

export NWCHEM_BASIS_LIBRARY="/home/avazquez/soft/nwchem/nwchem-6.0/src/basis/libraries"

export NWCHEM_TOP=/scratch/avazquez/nwchem-6.1

export NWCHEM_TOP=/home/avazquez/soft/nwchem/nwchem-6.1

export NWCHEM_MODULES="ddscf nwdft driver gradients property hessian vib cphf dplot"

export NWCHEM_MODULES=all