RUNPATH/LD_LIBRARY_PATH Fixes

CMakeLists.txt

@@ -70,9 +70,9 @@ endif("${isSystemDir}" STREQUAL "-1")
 
 ## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them).
 # set runpath, too
-# if(NOT APPLE)
-#   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags")
-# endif()
+if(NOT APPLE)
+  set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags")
+endif()
 
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${THIRD_PARTY_ROOT}/lib")
 
@@ -203,6 +203,11 @@ endif()
 # MPI (mostly for booting, but we also use MPI communication in some places now)
 find_package(MPI REQUIRED)
 include_directories(${MPI_INCLUDE_PATH})
+# Add MPI libs dirs to RPATH
+foreach(lib ${MPI_CXX_LIBRARIES})
+  get_filename_component(libdir "${lib}" PATH)
+  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${libdir}")
+endforeach()
 
 macro(add_grappa_exe target exe )
   add_executable(${target} EXCLUDE_FROM_ALL ${ARGN})

applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../../util/igor_common.rb'
+require_relative_to_symlink '../../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/NPB/MPI/igor_mpi_intsort.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/graph500/grappa/igor_ckpt_gen.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 def expand_flags(*names)
   names.map{|n| "--#{n}=%{#{n}}"}.join(' ')

applications/graph500/grappa/igor_grappa_bfs.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/graph500/grappa/igor_new_bfs.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/graph500/grappa/igor_new_cc.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/graph500/mpi/igor_mpi_bfs.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../../util/igor_common.rb'
+require_relative_to_symlink '../../../util/igor_common.rb'
 
 Igor do
   include Isolatable

applications/pagerank/igor_grappa_pagerank.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../util/igor_common.rb'
+require_relative_to_symlink '../../util/igor_common.rb'
 
 def expand_flags(*names)
   names.map{|n| "--#{n}=%{#{n}}"}.join(' ')

applications/uts/igor_grappa_uts.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../../util/igor_common.rb'
+require_relative_to_symlink '../../util/igor_common.rb'
 
 def expand_flags(*names)
   names.map{|n| "--#{n}=%{#{n}}"}.join(' ')

bin/CMakeLists.txt

@@ -1,9 +1,33 @@
-# file(COPY . DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-file(GLOB scripts "*")
-foreach(file ${scripts})
+
+set(symlinked_scripts
+  distcc_make
+  distcc_ninja
+  grappa_srun
+  launch_distcc.sh
+  srun_epilog.sh
+)
+
+set(configured_scripts
+  srun_prolog.rb
+)
+
+foreach(file ${symlinked_scripts})
   get_filename_component(name ${file} NAME)
-  file(RELATIVE_PATH relative_file ${CMAKE_CURRENT_BINARY_DIR} ${file})
   execute_process(COMMAND
-    ln -sf ${relative_file} ${CMAKE_CURRENT_BINARY_DIR}/${name}
+    ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${CMAKE_CURRENT_BINARY_DIR}/${name}
   )
 endforeach()
+
+set(GRAPPA_LD_DIRS "")
+if(VAMPIR_ROOT)
+  set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${VAMPIR_ROOT}/lib")
+endif()
+if(THIRD_PARTY_ROOT)
+  set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${THIRD_PARTY_ROOT}/lib")
+else()
+  set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${EXTERN_BUILD_DIR}/lib")  
+endif()
+
+foreach(f ${configured_scripts})
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${f}.in ${CMAKE_CURRENT_BINARY_DIR}/${f} @ONLY)
+endforeach()

bin/grappa_srun

@@ -39,7 +39,7 @@ srun_flags = %w[ --cpu_bind=rank --label --kill-on-bad-exit ] \
           << "--task-epilog=#{DIR}/srun_epilog.sh"
 
 # "source" prolog
-require_relative "srun_prolog.rb"
+require "#{DIR}/srun_prolog.rb"
 
 setarch = ""
 

bin/srun_prolog.rb → bin/srun_prolog.rb.in

@@ -4,6 +4,11 @@
   interactive = true
 end
 
+unless ENV["IGOR_OVERRIDE_LIBRARY_PATH"]
+  # prepend VampirTrace & third-party dirs to LD_LIBRARY_PATH
+  ENV["LD_LIBRARY_PATH"] = "@GRAPPA_LD_DIRS@:#{ENV["LD_LIBRARY_PATH"]}"
+end
+
 ## set up Google logging defaults
 ENV["GLOG_logtostderr"] = "1"
 ENV["GLOG_v"] = "0"

system/Grappa.cpp

@@ -185,13 +185,18 @@ static void stats_dump_sighandler( int signum ) {
 // function to call when google logging library detect a failure
 namespace Grappa {
   namespace impl {
+
+    bool freeze_on_error = false;
+
     /// called on failures to backtrace and pause for debugger
-    void failure_function() {
+    void failure_function(int ignore = 0) {
+      fprintf(stderr, "!! failure function\n");
       google::FlushLogFiles(google::GLOG_INFO);
-      google::DumpStackTrace();
-      gasnett_freezeForDebuggerErr();
+      // google::DumpStackTrace();
+      if (freeze_on_error) gasnett_freezeForDebuggerErr();
       gasnet_exit(1);
     }
+
   }
 }
 
@@ -207,7 +212,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt
   // }
 
   // make sure gasnet is ready to backtrace
-  gasnett_backtrace_init( (*argv_p)[0] );
+  // gasnett_backtrace_init( (*argv_p)[0] );
 
   // help generate unique profile filename
   Grappa::impl::set_exe_name( (*argv_p)[0] );
@@ -217,8 +222,11 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt
 
   // activate logging
   google::InitGoogleLogging( *argv_p[0] );
-  google::InstallFailureFunction( &Grappa::impl::failure_function );
-  google::OverrideDefaultSignalHandler( &gasnet_pause_sighandler );
+
+  char * env_freeze = getenv("GASNET_FREEZE_ON_ERROR");
+  if (env_freeze && strncmp(env_freeze,"1",1) == 0) freeze_on_error = true;
+
+  google::OverrideDefaultSignalHandler( &Grappa::impl::failure_function );
 
   DVLOG(1) << "Initializing Grappa library....";
 #ifdef HEAPCHECK_ENABLE
@@ -248,7 +256,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt
   sigemptyset( &sigabrt_sa.sa_mask );
   sigabrt_sa.sa_flags = 0;
   sigabrt_sa.sa_handler = &gasnet_pause_sighandler;
-  CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed.";
+  // CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed.";
 
   // Asynchronous IO
   // initialize completed stack
@@ -385,6 +393,8 @@ void Grappa_activate()
 {
   DVLOG(1) << "Activating Grappa library....";
   global_communicator.activate();
+  google::InstallFailureSignalHandler();
+
   locale_shared_memory.activate();
   global_task_manager.activate();
   Grappa::comm_barrier();

system/LocaleSharedMemory.cpp

@@ -38,7 +38,7 @@ namespace Grappa {
 namespace impl {
 
 /// called on failures to backtrace and pause for debugger
-extern void failure_function();
+extern void failure_function(int ignore = 0);
 
 /// how much memory do we expect to allocate?
 extern int64_t global_memory_size_bytes;

system/RDMAAggregator.cpp

@@ -33,7 +33,7 @@
 namespace Grappa {
   namespace impl {
     // defined in Grappa.cpp
-    extern void failure_function();
+    extern void failure_function(int ignore = 0);
   }
 }
 

system/tests/igor_context_switch.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../util/igor_common.rb'
+require_relative_to_symlink '../util/igor_common.rb'
 
 Igor do
   include Isolatable

system/tests/igor_datastructs.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../util/igor_common.rb'
+require_relative_to_symlink '../util/igor_common.rb'
 
 Igor do
   database '~/exp/pgas.sqlite', :queue

system/tests/igor_tests.rb

@@ -2,7 +2,7 @@
 require 'igor'
 
 # inherit parser, sbatch_flags
-require_relative '../util/igor_common.rb'
+require_relative_to_symlink '../util/igor_common.rb'
 
 Igor do
   database '~/exp/test.sqlite', :vector

util/igor_common.rb

@@ -69,7 +69,7 @@ def isolate(exes, shared_dir=nil)
         end
       end
       FileUtils.cp(exe, @ldir)
-      libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH} ldd #{exe}"`
+      libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH}:$LD_LIBRARY_PATH ldd #{exe}"`
                 .split(/\n/)
                 .map{|s| s[/.*> (.*) \(.*/,1] }
                 .select{|s| s != nil and s != "" and
@@ -105,7 +105,7 @@ def command(c=nil)
             echo $l; sbcast #{@ldir}/$l #{tdir}/${l};
           done;
         fi;
-        #{c}
+        LD_LIBRARY_PATH=#{tdir} IGOR_OVERRIDE_LIBRARY_PATH=1 #{c}
       ].tr("\n "," ")
     end
     return super(c)