Help to compile the Parallel OpenSees 2.4.1 in Linux Cluster

[xuty]

Hello I'm trying to compile the OpenSees 2.4.1 in a Linux cluster. But I got the following errors:


/admin/sw-src/OpenSees/2.4.1/OpenSees/SRC/tcl/commands.o: In function `eigenAnalysis(void*, Tcl_Interp*, int, char const**)':
commands.cpp:(.text+0x53de): undefined reference to `DistributedSuperLU::DistributedSuperLU(int, int)'
commands.cpp:(.text+0x53f6): undefined reference to `DistributedSparseGenColLinSOE::DistributedSparseGenColLinSOE(SparseGenColLinSolver&)'
/admin/sw-src/OpenSees/2.4.1/OpenSees/SRC/tcl/commands.o: In function `specifySOE(void*, Tcl_Interp*, int, char const**)':
commands.cpp:(.text+0xcd2f): undefined reference to `DistributedSparseGenColLinSOE::DistributedSparseGenColLinSOE(SparseGenColLinSolver&)'
commands.cpp:(.text+0xcdbd): undefined reference to `DistributedSuperLU::DistributedSuperLU(int, int)'
/admin/sw-src/OpenSees/2.4.1/lib/libOpenSees.a(FEM_ObjectBrokerAllClasses.o): In function `FEM_ObjectBrokerAllClasses::getNewLinearSOE(int)':
FEM_ObjectBrokerAllClasses.cpp:(.text+0x6f1): undefined reference to `DistributedSparseGenColLinSOE::DistributedSparseGenColLinSOE()'
collect2: error: ld returned 1 exit status
make[1]: *** [tcl] Error 1
make: *** [all] Error 2

[xuty]

And here is my Makefile.def

############################################################################
#
# Program: OpenSees
#
# Purpose: A Top-level Makefile to create the libraries needed
# to use the OpenSees framework.
#
# version created for Redhat 8.0 LINUX distribution
#
# Written: fmk
# Created: 01/2003
#
#
############################################################################

# %---------------------------------%
# | SECTION 1: PROGRAM |
# %---------------------------------%
#
# Specify the location and name of the OpenSees interpreter program
# that will be created (if this all works

OpenSees_PROGRAM = $(HOME)/bin/OpenSeesMP
#OpenSees_PROGRAM_MPI = $(HOME)/bin/OpenSeesMPI

# %---------------------------------%
# | SECTION 2: MAKEFILE CONSTANTS |
# %---------------------------------%
#
# Specify the constants the are used as control structure variables in the Makefiles.

OPERATING_SYSTEM = LINUX
PROGRAMMING_MODE = PARALLEL
RELIABILITY = NO_RELIABILITY
GRAPHICS = NONE
DEBUG_MODE = NO_DEBUG


#PROGRAMMING_MODE = PARALLEL, SEQUENTIAL
#GRAPHICS = NONE, UsingOpenGL
#RELIABILITY = YES_RELIABILITY, NO_RELIABILITY
#DEBUG_MODE = DEBUG, NO_DEBUG


# %---------------------------------%
# | SECTION 3: PATHS |
# %---------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries or if you have
# any of the libraries already leave the directory location blank AND
# remove the directory from DIRS.

BASE =
# PUT YOUR HOME DIRECTOREY HERE
HOME = /admin/sw-src/OpenSees/2.4.1
PARALLELdir = /apps/OpenSees/2.4.1/PARALLEL
FE = $(HOME)/OpenSees/SRC

BLASdir =
CBLASdir = $(HOME)/OpenSees/OTHER/CBLAS
LAPACKdir =
ARPACKdir = $(HOME)/OpenSees/OTHER/ARPACK
UMFPACKdir = $(HOME)/OpenSees/OTHER/UMFPACK
METISdir =
SRCdir = $(HOME)/OpenSees/SRC
SUPERLUdir = $(HOME)/OpenSees/OTHER/SuperLU_4.1/SRC

SUPERLU_DISTdir = $(HOME)/OpenSees/OTHER/SuperLU_DIST_2.5/SRC
CSPARSEdir = $(HOME)/OpenSees/OTHER/CSPARSE
DIRS = $(CBLASdir) \
$(SUPERLUdir) \
$(ARPACKdir) \
$(UMFPACKdir) \
$(SUPERLU_DISTdir) \
$(CSPARSEdir) \
$(SRCdir)

# %-------------------------------------------------------%
# | SECTION 4: LIBRARIES |
# | |
# | The following section defines the libraries that will |
# | be created and/or linked with when the libraries are |
# | being created or linked with. |
# %-------------------------------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries leave the
# libraries blank. You have to get your own copy of the tcl/tk
# library!!
#
# Note: For libraries that will be created (any in DIRS above)
# make sure the directory exsists where you want the library to go!

FE_LIBRARY = $(HOME)/lib/libOpenSees.a
NDARRAY_LIBRARY = $(HOME)/lib/libndarray.a # BJ_UCD jeremic@ucdavis.edu
MATMOD_LIBRARY = $(HOME)/lib/libmatmod.a # BJ_UCD jeremic@ucdavis.edu
BJMISC_LIBRARY = $(HOME)/lib/libBJmisc.a # BJ_UCD jeremic@ucdavis.edu
LAPACK_LIBRARY = $(PARALLELdir)/lapack/liblapack.a
BLAS_LIBRARY = $(PARALLELdir)/blas/blas.a
SUPERLU_LIBRARY = $(HOME)/lib/libSuperLU.a
CBLAS_LIBRARY = $(HOME)/lib/libCBlas.a
ARPACK_LIBRARY = $(HOME)/lib/libArpack.a
UMFPACK_LIBRARY = $(HOME)/lib/libUmfpack.a
METIS_LIBRARY = $(PARALLELdir)/metis/libmetis.a
DISTRIBUTED_SUPERLU_LIBRARY = $(HOME)/lib/libDistributedSuperLU.a
CSPARSE_LIBRARY = $(HOME)/lib/libCSparse.a
AMD_LIBRARY = $(HOME)/lib/libAMD.a

TCL_LIBRARY = /apps/ActiveTCL/8.5.13/lib/libtk8.5.so \
/apps/ActiveTCL/8.5.13/lib/libtcl8.5.so

ifeq ($(RELIABILITY), YES_RELIABILITY)
RELIABILITY_LIBRARY = $(HOME)/lib/libReliability.a
else
RELIABILITY_LIBRARY =
endif

# WATCH OUT .. These libraries are removed when 'make wipe' is invoked.

WIPE_LIBS = $(FE_LIBRARY) \
$(NDARRAY_LIBRARY) \
$(MATMOD_LIBRARY) \
$(DISTRIBUTED_SUPERLU_LIBRARY) \
$(SUPERLU_LIBRARY) \
$(ARPACK_LIBRARY) \
$(CSPARSE_LIBRARY) \
$(UMFPACK_LIBRARY) \
$(CBLAS_LIBRARY) \
$(RELIABILITY_LIBRARY)

# %---------------------------------------------------------%
# | SECTION 5: COMPILERS |
# | |
# | The following macros specify compilers, linker/loaders, |
# | the archiver, and their options. You need to make sure |
# | these are correct for your system. |
# %---------------------------------------------------------%

# ###################################################
# # Compilers
# ###################################################

CC++ = g++
CC = gcc
FC = /usr/bin/gfortran
FORTRAN = /usr/bin/gfortran
F90 =
LINKER = mpicxx

AR = ar
ARFLAGS = cqls
ARCH = ar
ARCHFLAGS = cqls
RANLIB = ranlib
RANLIBFLAGS =

PROGRAMMING_FLAG =

ifeq ($(GRAPHICS), UsinOpenGl)
GRAPHIC_FLAG = -D_GLX
GRAPHIC_LIBRARY = -L/usr/X11/R6/lib -lGL -lGLU
else
GRAPHIC_FLAG = -D_NOGRAPHICS
GRAPHIC_LIBRARY =
endif

ifeq ($(RELIABILITY), YES_RELIABILITY)
RELIABILITY_FLAG = -D_RELIABILITY
else
RELIABILITY_FLAG =
endif

ifeq ($(DEBUG_MODE), DEBUG)
DEBUG_FLAG = -D_G3DEBUG
else
DEBUG_FLAG =
endif

ifeq ($(DEBUG_MODE), DEBUG)

C++FLAGS = -Wall -g -pg -D_LINUX -D_UNIX $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG) -D_PARALLEL_PROCESSING -D_TCL85
CFLAGS = -Wall -g -pg $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG)
FFLAGS = -Wall
LINKFLAGS = -rdynamic -g -pg

else

C++FLAGS = -Wall -O2 -D_LINUX -D_UNIX -D_TCL85 -Wall $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG) -D_PARALLEL_PROCESSING -ffloat-store

CFLAGS = -Wall $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(PROGRAMMING_FLAG)
FFLAGS = -Wall
LINKFLAGS = -rdynamic

endif

# Misc
MAKE = make
CD = cd
ECHO = echo
RM = rm
RMFLAGS = -f
SHELL = /bin/sh

# %---------------------------------------------------------%
# | SECTION 6: COMPILATION |
# | |
# | The following macros specify the macros used in |
# | to compile the source code into object code. |
# %---------------------------------------------------------%

.SUFFIXES:
.SUFFIXES: .C .c .f .f90 .cpp .o .cpp

#
# %------------------%
# | Default command. |
# %------------------%
#
.DEFAULT:
@$(ECHO) "Unknown target $@, try: make help"
#
# %-----------------------------------------------%
# | Command to build .o files from source files. |
# %-----------------------------------------------%
#

.cpp.o:
@$(ECHO) Making $@ from $<
$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.C.o:
@$(ECHO) Making $@ from $<
$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.c.o:
@$(ECHO) Making $@ from $<
$(CC) $(CFLAGS) -c $<

.f.o:
@$(ECHO) Making $@ from $<
$(FC) $(FFLAGS) -c $< -o $@

.f77.o:
@$(ECHO) Making $@ from $<
$(FC) $(FFLAGS) -c $< -o $@

.f90.o:
@$(ECHO) Making $@ from $<
$(FC90) $(FFLAGS) -c $< -o $@

# %---------------------------------------------------------%
# | SECTION 7: OTHER LIBRARIES |
# | |
# | The following macros specify other libraries that must |
# | be linked with when creating executables. These are |
# | platform specific and typically order does matter!! |
# %---------------------------------------------------------%
MACHINE_LINKLIBS = -L$(BASE)/lib \
-L$(HOME)/lib \




MACHINE_NUMERICAL_LIBS = -lm \
$(ARPACK_LIBRARY) \
$(DISTRIBUTED_SUPERLU_LIBRARY) \
$(SUPERLU_LIBRARY) \
$(UMFPACK_LIBRARY) \
$(CSPARSE_LIBRARY) \
$(LAPACK_LIBRARY) \
$(BLAS_LIBRARY) \
$(CBLAS_LIBRARY) \
$(GRAPHIC_LIBRARY) \
$(METIS_LIBRARY) \
$(AMD_LIBRARY) \
$(RELIABILITY_LIBRARY) \
$(PETSC_LIB) \
/usr/lib/gcc/x86_64-redhat-linux/4.1.1/libgfortran.a -ldl

MPICC = mpicc
MPIINC = -I/apps/mpich2/1.4.1p1/include
MPILIB = -L/apps/mpich2/1.4.1p1/lib -lmpich

# PETSC
HAVEPETSC = NO
PETSCINC =
PETSC_LIB =

ifeq ($(HAVEPETSC), YES)
PETSC = YES
PETSC_FLAG = -D_PETSC
PETSC_INC = -I/apps/petsc/3.3-p2/include \
-I$(HOME)/OpenSees/parlibs/petsc-2.2.1 \
-I$(HOME)/OpenSees/parlibs/petsc-2.2.1/bmake/linux-gnu \
-I$(HOME)/OpenSees/parlibs/petsc-2.2.1/include \
-I$(HOME)/OpenSees/parlibs/petsc-2.2.1/include/mpiuni \
-DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX


PETSCINC = -I/apps/petsc/3.3-p2/include -I$(HOME)/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1 -I$(HOME)/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1/bmake/linux-gnu -I$(HOME)/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1/include \
-I$(HOME)/OpenSees/parlibs/petsc-2.2.1/include/mpiuni -DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX

#PETSCINC = -I$(HOME)/OpenSees/parlibs/petsc-2.2.1/include -D_PETSC -I$(HOME)/OpenSees/parlibs/petsc-2.2.1/bmake/linux-gnu -DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX


# $(FE)/system_of_eqn/linearSOE/petsc/PetscSparseSeqSolver.o \


PETSC_LIB = /apps/petsc/3.3-p2/lib/libpetsc.a \
$(FE)/system_of_eqn/linearSOE/petsc/PetscSOE.o \
$(FE)/system_of_eqn/linearSOE/petsc/PetscSolver.o \
$(FE)/system_of_eqn/linearSOE/petsc/PetscSparseSeqSolver.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgebak.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgebal.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgeev.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgehd2.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgehrd.o \
$(HOME)/OpenSees/OTHER/LAPACK/dhseqr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dlahrd.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorg2r.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorghr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorgqr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dpotrf.o \
$(HOME)/OpenSees/OTHER/LAPACK/dpotrs.o \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetscsnes.a \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetscksp.a \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetscdm.a \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetscmat.a \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetscvec.a \
$(HOME)/OpenSees/parlibs/petsc-2.2.1/lib/libg/linux-gnu/libpetsc.a

# -L/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/
# -lpetscsnes -lpetscksp -lpetscdm -lpetscmat -lpetscvec -lpetsc
endif

HAVEMUMPS = YES
MUMPS_INCLUDE =
MUMPS_LIB =

ifeq ($(HAVEMUMPS), YES)

MUMPS = YES
MUMPS_FLAG = -D_MUMPS
#BLAS
BLASdir = $(PARALLELdir)/blas
BLAS_LIB = $(BLASdir)/blas.a
#XBLAS
XBLAS_LIB = $(PARALLELdir)/xblas/libxblas.a
#SCALAPACK
SCALAPACK_LIB = $(PARALLELdir)/scalapack/libscalapack.a
#BLACS
BLACSdir = $(PARALLELdir)/blacs
BLACS_LIB = -L$(BLACSdir)/LIB

#LAPACK
LAPACKdir = $(PARALLELdir)/lapack
LAPACK_LIB = -L$(LAPACKdir)
#MUMPS
MUMPSdir = $(PARALLELdir)/mumps
MUMPSLIB = $(MUMPSdir)/lib
MUMPS_INC = $(MUMPSdir)/include
#ParMETIS
parmetis_lib = $(PARALLELdir)/parmetis/libparmetis.a
metis_lib = $(PARALLELdir)/parmetis/libmetis.a
metis_inc = -I$(PARALLELdir)/parmetis

SCALAP = $(SCALAPACK_LIB) \
$(BLACSdir)/LIB/blacsF77init.a \
$(BLACSdir)/LIB/blacs.a \
$(BLACSdir)/LIB/blacsCinit.a \
$(LAPACKdir)/liblapack.a \
$(XBLAS_LIB) \
$(BLASdir)/blas.a

#-L/usr/lib/x86_64-linux-gnu/gcc/x86_64-linux-gnu/4.5/libgfortran.a $(BLACS_LIB) $(BLAS_LIB)
#$(LAPACK_LIB) $(BLACS_LIB) $(BLAS_LIB) $(BLACS_LIB)

PLAT = LINUX
MUMPS_LIB = $(FE)/system_of_eqn/linearSOE/mumps/MumpsSOE.o \
$(FE)/system_of_eqn/linearSOE/mumps/MumpsSolver.o \
$(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSOE.o \
$(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSolver.o \
$(SCALAP) \
-L$(MUMPSdir)/lib -lcmumps -ldmumps -lsmumps -lzmumps \
-lmumps_common -lpord \
$(parmetis_lib) $(metis_lib) \
$(SCALAP) \
/usr/lib/gcc/x86_64-redhat-linux/4.1.1/libgfortran.a \
/usr/lib64/libdl.so

MUMPS_INCLUDE = -I$(MUMPS_INC)

endif

ifeq ($(PROGRAMMING_MODE), THREADS)
MACHINE_SPECIFIC_LIBS = -lpthread
else
MACHINE_SPECIFIC_LIBS = /usr/lib/gcc/x86_64-redhat-linux/4.1.1/libgfortran.a -ldl -lieee -lm -lc -Wl,-rpath,/usr/lib -lpng $(MPILIB)
endif


# %---------------------------------------------------------%
# | SECTION 8: INCLUDE FILES |
# | |
# | The following macros specify include files needed for |
# | compilation. |
# %---------------------------------------------------------%


MACHINE_INCLUDES = $(MPIINC) \
-I/apps/berkeleydb/5.3.21/include \
-I/usr/include/mysql \
-I$(HOME)/include \
-I$(UMFPACKdir) \
-I$(SUPERLUdir) \
$(MUMPS_INCLUDE) \
$(metis_inc) \
-I$(SUPERLU_DISTdir) #$(PETSCINC)

[xuty]

# this file contains all the OpenSees/SRC includes
include $(FE)/Makefile.incl

#TCL_INCLUDES = -I/home/fmk/tcl8.3.3/generic

INCLUDES = $(MACHINE_INCLUDES) $(TCL_INCLUDES) $(FE_INCLUDES)

[xuty]

I followed the instruction of installing OpenSees 2.3.2.2 first. The gcc is 4.7.1.

[bartverleye]

Hi,
I have the same problem...
Did you get any response, or did you solve the problem?

Thanks.

[fmk]

obviously the files related to Distributed SuperLU are missing ... the .o files should be in SRC/system_of)eqn/linearSOE/sparseGEN .. you can add the .o files as part of the NUMERICAL_LIBS or MUMPS_LIB in the Makefile.def.

[bartverleye]

Hi,

Thanks for the help, I got the program compiled.
However, now we get the error:

number of steps in analysis: 5000
analysis time step: 0.005
StaticDomainDecompositionAnalysis::recvSelf - failed to get the Solver
Fatal error in MPI_Recv: Message truncated, error stack:
MPI_Recv(184).....................: MPI_Recv(buf=0x14a55b0, count=4, MPI_INT, src=0, tag=0, MPI_COMM_WORLD, status=0x7fff6a3c7360) failed
MPIDI_CH3U_Receive_data_found(131): Message from rank 0 and tag 0 truncated; 24 bytes received but buffer size is 16

Solver not found...We use the slopeAnalysis2Deff_SP.tcl as input.
My Makefile.def below.
Any hints perhaps to make this work?

Thanks!
Bart

############################################################################
#
# Program: OpenSees
#
# Purpose: A Top-level Makefile to create the libraries needed
# to use the OpenSees framework.
#
# version created for Redhat 8.0 LINUX distribution
#
# Written: fmk
# Created: 01/2003
#
#
############################################################################

# %---------------------------------%
# | SECTION 1: PROGRAM |
# %---------------------------------%
#
# Specify the location and name of the OpenSees interpreter program
# that will be created (if this all works

OpenSees_PROGRAM = $(HOME)/opensees/trunk/bin/OpenSees
OpenSees_PROGRAM_MPI = $(HOME)/opensees/trunk/bin/OpenSeesMPI

# %---------------------------------%
# | SECTION 2: MAKEFILE CONSTANTS |
# %---------------------------------%
#
# Specify the constants the are used as control structure variables in the Makefiles.

OPERATING_SYSTEM = LINUX
PROGRAMMING_MODE = PARALLEL
RELIABILITY = NO_RELIABILITY
GRAPHICS = NONE
DEBUG_MODE = NO_DEBUG


#PROGRAMMING_MODE = PARALLEL, SEQUENTIAL
#GRAPHICS = NONE, UsingOpenGL
#RELIABILITY = YES_RELIABILITY, NO_RELIABILITY
#DEBUG_MODE = DEBUG, NO_DEBUG


# %---------------------------------%
# | SECTION 3: PATHS |
# %---------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries or if you have
# any of the libraries already leave the directory location blank AND
# remove the directory from DIRS.

BASE =
# PUT YOUR HOME DIRECTOREY HERE
HOME = /home/bver018/
FE = $(HOME)/OpenSees/SRC

BLASdir = $(HOME)/OpenSees/OTHER/BLAS
CBLASdir = $(HOME)/OpenSees/OTHER/CBLAS
LAPACKdir = $(HOME)/OpenSees/OTHER/LAPACK
ARPACKdir = $(HOME)/OpenSees/OTHER/ARPACK
UMFPACKdir = $(HOME)/OpenSees/OTHER/UMFPACK
METISdir = $(HOME)/OpenSees/OTHER/METIS
SRCdir = $(HOME)/OpenSees/SRC
SUPERLUdir = $(HOME)/OpenSees/OTHER/SuperLU_4.1/SRC

SUPERLU_DISTdir = $(HOME)/OpenSees/OTHER/SuperLU_DIST_2.5/SRC

DIRS = $(BLASdir) \
$(CBLASdir) \
$(SUPERLUdir) \
$(LAPACKdir) \
$(ARPACKdir) \
$(UMFPACKdir) \
$(METISdir) \
$(SUPERLU_DISTdir) \
$(SRCdir)

# %-------------------------------------------------------%
# | SECTION 4: LIBRARIES |
# | |
# | The following section defines the libraries that will |
# | be created and/or linked with when the libraries are |
# | being created or linked with. |
# %-------------------------------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries leave the
# libraries blank. You have to get your own copy of the tcl/tk
# library!!
#
# Note: For libraries that will be created (any in DIRS above)
# make sure the directory exsists where you want the library to go!

FE_LIBRARY = $(HOME)/lib/libOpenSees.a
NDARRAY_LIBRARY = $(HOME)/lib/libndarray.a # BJ_UCD jeremic@ucdavis.edu
MATMOD_LIBRARY = $(HOME)/lib/libmatmod.a # BJ_UCD jeremic@ucdavis.edu
BJMISC_LIBRARY = $(HOME)/lib/libBJmisc.a # BJ_UCD jeremic@ucdavis.edu
LAPACK_LIBRARY = $(HOME)/lib/libLapack.a
BLAS_LIBRARY = $(HOME)/lib/libBlas.a
SUPERLU_LIBRARY = $(HOME)/lib/libSuperLU.a
CBLAS_LIBRARY = $(HOME)/lib/libCBlas.a
ARPACK_LIBRARY = $(HOME)/lib/libArpack.a
UMFPACK_LIBRARY = $(HOME)/lib/libUmfpack.a
METIS_LIBRARY = $(HOME)/lib/libMetis.a
DISTRIBUTED_SUPERLU_LIBRARY = $(HOME)/lib/libDistributedSuperLU.a
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/DistributedSparseGenColLinSOE.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/DistributedSparseGenRowLinSOE.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/DistributedSuperLU.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/SparseGenColLinSOE.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/SparseGenColLinSolver.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/SparseGenRowLinSOE.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/SparseGenRowLinSolver.o \
#/home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/SuperLU.o

TCL_LIBRARY = /home/bver018/.local/easybuild/software/./Tk/8.5.12-gmpolf-1.4.8/lib/libtk8.5.so \
/home/bver018/.local/easybuild/software/./Tcl/8.5.12-gmpolf-1.4.8/lib/libtcl8.5.so

ifeq ($(RELIABILITY), YES_RELIABILITY)
RELIABILITY_LIBRARY = $(HOME)/lib/libReliability.a
else
RELIABILITY_LIBRARY =
endif

# WATCH OUT .. These libraries are removed when 'make wipe' is invoked.

WIPE_LIBS = $(FE_LIBRARY) \
$(NDARRAY_LIBRARY) \
$(MATMOD_LIBRARY) \
$(DISTRIBUTED_SUPERLU_LIBRARY) \
$(SUPERLU_LIBRARY) \
$(ARPACK_LIBRARY) \
$(UMFPACK_LIBRARY) \
$(METIS_LIBRARY) \
$(LAPACK_LIBRARY) \
$(BLAS_LIBRARY) \
$(CBLAS_LIBRARY) \
$(RELIABILITY_LIBRARY)

# %--------i-------------------------------------------------%
# | SECTION 5: COMPILERS |
# | |
# | The following macros specify compilers, linker/loaders, |
# | the archiver, and their options. You need to make sure |
# | these are correct for your system. |
# %---------------------------------------------------------%

# ###################################################
# # Compilers
# ###################################################

CC++ = mpicxx
CC = mpicc
FC = mpif90
FORTRAN = mpif90
F90 = mpif90
LINKER = mpicxx

AR = ar
ARFLAGS = cqls
ARCH = ar
ARCHFLAGS = cqls
RANLIB = ranlib
RANLIBFLAGS =

PROGRAMMING_FLAG =

ifeq ($(GRAPHICS), UsinOpenGl)
GRAPHIC_FLAG = -D_GLX
GRAPHIC_LIBRARY = -L/usr/X11/R6/lib -lGL -lGLU
else
GRAPHIC_FLAG = -D_NOGRAPHICS
GRAPHIC_LIBRARY =
endif

ifeq ($(RELIABILITY), YES_RELIABILITY)
RELIABILITY_FLAG = -D_RELIABILITY
else
RELIABILITY_FLAG =
endif

ifeq ($(DEBUG_MODE), DEBUG)
DEBUG_FLAG = -D_G3DEBUG
else
DEBUG_FLAG =
endif

ifeq ($(DEBUG_MODE), DEBUG)

C++FLAGS = -Wall -g -pg -D_LINUX -D_UNIX $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG) -D_PARALLEL_PROCESSING -D_TCL84
CFLAGS = -Wall -g -pg $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG)
FFLAGS = -Wall
LINKFLAGS = -rdynamic -g -pg

else

C++FLAGS = -Wall -O2 -D_LINUX -D_UNIX -D_TCL84 -Wall $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(PROGRAMMING_FLAG) -D_PARALLEL_PROCESSING -ffloat-store

CFLAGS = -Wall $(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(PROGRAMMING_FLAG)
FFLAGS = -Wall
LINKFLAGS = -rdynamic

endif

# Misc
MAKE = make
CD = cd
ECHO = echo
RM = rm
RMFLAGS = -f
SHELL = /bin/sh

# %---------------------------------------------------------%
# | SECTION 6: COMPILATION |
# | |
# | The following macros specify the macros used in |
# | to compile the source code into object code. |
# %---------------------------------------------------------%

.SUFFIXES:
.SUFFIXES: .C .c .f .f90 .cpp .o .cpp

#
# %------------------%
# | Default command. |
# %------------------%
#
.DEFAULT:
@$(ECHO) "Unknown target $@, try: make help"
#
# %-----------------------------------------------%
# | Command to build .o files from source files. |
# %-----------------------------------------------%
#

.cpp.o:
@$(ECHO) Making $@ from $<
$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.C.o:
@$(ECHO) Making $@ from $<
$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.c.o:
@$(ECHO) Making $@ from $<
$(CC) $(CFLAGS) -c $<

.f.o:
@$(ECHO) Making $@ from $<
$(FC) $(FFLAGS) -c $< -o $@

.f77.o:
@$(ECHO) Making $@ from $<
$(FC) $(FFLAGS) -c $< -o $@

.f90.o:
@$(ECHO) Making $@ from $<
$(FC90) $(FFLAGS) -c $< -o $@

# %---------------------------------------------------------%
# | SECTION 7: OTHER LIBRARIES |
# | |
# | The following macros specify other libraries that must |
# | be linked with when creating executables. These are |
# | platform specific and typically order does matter!! |
# %---------------------------------------------------------%
MACHINE_LINKLIBS = -L$(BASE)/lib \
-L$(HOME)/lib \

HAVEPETSC = NO


MACHINE_NUMERICAL_LIBS = -lm \
$(ARPACK_LIBRARY) \
$(DISTRIBUTED_SUPERLU_LIBRARY) \
$(SUPERLU_LIBRARY) \
$(UMFPACK_LIBRARY) \
$(LAPACK_LIBRARY) \
$(BLAS_LIBRARY) \
$(CBLAS_LIBRARY) \
$(GRAPHIC_LIBRARY)\
$(METIS_LIBRARY)\
$(RELIABILITY_LIBRARY)\
$(PETSC_LIB)\
-lgfortran -ldl

MPICC = mpicc
MPIINC = -I/usr/local/mpi/include
MPILIB = -L/usr/local/mpi/lib -lmpich


ifeq ($(HAVEPETSC), YES)


PETSC_INC = -I/scratch/fmckenna/petsc-2.2.1 \
-I/scratch/fmckenna/petsc-2.2.1/bmake/linux-gnu \
-I/scratch/fmckenna/petsc-2.2.1/include \
-I/scratch/fmckenna/petsc-2.2.1/include/mpiuni \
-DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX


PETSCINC = -I/scratch/fmckenna/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1 -I/scratch/fmckenna/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1/bmake/linux-gnu -I/scratch/fmckenna/OpenSees/SRC/system_of_eqn/linear/petsc/petsc-2.2.1/include \
-I/scratch/fmckenna/petsc-2.2.1/include/mpiuni -DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX

PETSCINC = -I/scratch/fmckenna/petsc-2.2.1/include -D_PETSC -I/scratch/fmckenna/petsc-2.2.1/bmake/linux-gnu -DPETSC_USE_BOPT_g -DPETSC_USE_EXTERN_CXX


# $(FE)/system_of_eqn/linearSOE/petsc/PetscSparseSeqSolver.o \


PETSC_LIB = $(FE)/system_of_eqn/linearSOE/petsc/PetscSOE.o \
$(FE)/system_of_eqn/linearSOE/petsc/PetscSolver.o \
$(FE)/system_of_eqn/linearSOE/petsc/PetscSparseSeqSolver.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgebak.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgebal.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgeev.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgehd2.o \
$(HOME)/OpenSees/OTHER/LAPACK/dgehrd.o \
$(HOME)/OpenSees/OTHER/LAPACK/dhseqr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dlahrd.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorg2r.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorghr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dorgqr.o \
$(HOME)/OpenSees/OTHER/LAPACK/dpotrf.o \
$(HOME)/OpenSees/OTHER/LAPACK/dpotrs.o \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetscsnes.a \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetscksp.a \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetscdm.a \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetscmat.a \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetscvec.a \
/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/libpetsc.a

# -L/scratch/fmckenna/petsc-2.2.1/lib/libg/linux-gnu/
# -lpetscsnes -lpetscksp -lpetscdm -lpetscmat -lpetscvec -lpetsc

endif

ifeq ($(PROGRAMMING_MODE), THREADS)
MACHINE_SPECIFIC_LIBS = -lpthread
else
MACHINE_SPECIFIC_LIBS = -ldl -lieee -lm -lc -lgfortran -Wl,-rpath,/usr/lib -lpng $(MPILIB)
endif


# %---------------------------------------------------------%
# | SECTION 8: INCLUDE FILES |
# | |
# | The following macros specify include files needed for |
# | compilation. |
# %---------------------------------------------------------%


MACHINE_INCLUDES = $(MPIINC) \
-I/usr/local/BerkeleyDB.4.0/include \
-I/usr/include/mysql \
-I$(HOME)/include \
-I$(UMFPACKdir) \
-I$(SUPERLUdir) \
-I$(SUPERLU_DISTdir) $(PETSCINC)


# this file contains all the OpenSees/SRC includes
include $(FE)/Makefile.incl

#TCL_INCLUDES = -I/home/fmk/tcl8.3.3/generic

INCLUDES = $(MACHINE_INCLUDES) $(TCL_INCLUDES) $(FE_INCLUDES)

[fmk]

you need to add the .o files that you have commented (the missing ones in the message) out into say the MACHINE_NUMERICAL_LIBS =

[bartverleye]

Thanks for the hint.

However, if I do that, I get:
ranlib: /home/bver018/opensees/trunk/SRC/system_of_eqn/linearSOE/sparseGEN/DistributedSparseGenColLinSOE.o: File format not recognized

Any clue?

Thanks.