segfault on startup in apex code when debuging HPX application

[biddisco]

I'm not sure what the problem is, but I'm getting a segfault on start when I debug a large application (possible stack overflow?). I'm posting this output (from frame 8 where apex is visible) in the hope that it is useful in determining what has gone wrong.

(gdb) print this
$7 = {
apex::event_listener = {
_vptr.event_listener = 0x2aaab0736d50 <vtable for apex::otf2_listener+16>
},
members of apex::otf2_listener:
_terminate = false,
_region_mutex = {
std::__mutex_base = {
_M_mutex = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
}, },
_string_mutex = {
std::__mutex_base = {
_M_mutex = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
}, },
_metric_mutex = {
std::__mutex_base = {
_M_mutex = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
}, },
_comm_mutex = {
std::__mutex_base = {
_M_mutex = {
__data = {
__lock = 0,
__count = 0,
__owner = 0,
__nusers = 0,
__kind = 0,
__spins = 0,
__elision = 0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = '\000' <repeats 39 times>,
__align = 0
}
}, },
_event_threads = std::unordered_set with 2 elements = {
[0] = 1,
[1] = 0
},
_archive_mutex = {
rwlock = {
__data = {
__lock = 0,
__nr_readers = 2,
__readers_wakeup = 0,
__writer_wakeup = 0,
__nr_readers_queued = 0,
__nr_writers_queued = 0,
__writer = 0,
__shared = 0,
__pad1 = 0,
__pad2 = 0,
__flags = 0
},
__size = "\000\000\000\000\002", '\000' <repeats 50 times>,
__align = 8589934592
}
},
static globalOffset = 1488875457697621182,
static flush_callbacks = {
otf2_pre_flush = 0x2aaaafc18210 <apex::otf2_listener::pre_flush(void, unsigned char, unsigned long, void*, bool)>,
otf2_post_flush = 0x2aaaafc18232 <apex::otf2_listener::post_flush(void*, unsigned char, unsigned long)>
},
archive = 0x2aaaba558200,
comm_evt_writer = 0x2aaab8acc1d0,
global_def_writer = 0x0,
global_string_indices = std::map with 1 elements = {
[{
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab8a96690 ""
},
_M_string_length = 0,
{
_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0
}
}] = 0
},
global_region_indices = std::map with 2 elements = {
[{
address = 0,
name = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab9fe2a80 "APEX MAIN THREAD"
},
_M_string_length = 16,
{
_M_local_buf = "\020\000\000\000\000\000\000\000\300?\247\270\252*\000",
_M_allocated_capacity = 16
}
},
_resolved_name = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab8a724d8 ""
},
_M_string_length = 0,
{
_M_local_buf = "\000\000\000\000\000\000\000\000\000Gr\260\252*\000",
_M_allocated_capacity = 0
}
},
has_name = true,
_guid = 0
}] = 0,
[{
address = 0,
name = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaabd6160b8 "locality"
},
_M_string_length = 8,
{
_M_local_buf = "locality\000\000\000\000\000\000\000",
_M_allocated_capacity = 8751735890099007340
}
},
_resolved_name = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaabd6160d8 ""
},
_M_string_length = 0,
{
_M_local_buf = '\000' <repeats 15 times>,
_M_allocated_capacity = 0
}
},
has_name = true,
_guid = 0
}] = 1
},
hostname_indices = std::map with 0 elements,
global_metric_indices = std::map with 0 elements,
static empty = ,
index_filename = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab9fe28a0 "OTF2_archive/.locality."
},
_M_string_length = 23,
{
_M_local_buf = "\036", '\000' <repeats 14 times>,
_M_allocated_capacity = 30
}
},
lock_filename_prefix = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab9fe2a40 "OTF2_archive/.regions.lock."
},
_M_string_length = 27,
{
_M_local_buf = "\036", '\000' <repeats 14 times>,
_M_allocated_capacity = 30
}
},
region_filename_prefix = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab9fe29c0 "OTF2_archive/.regions."
},
_M_string_length = 22,
{
_M_local_buf = "\036", '\000' <repeats 14 times>,
_M_allocated_capacity = 30
}
},
metric_filename_prefix = {
static npos = 18446744073709551615,
_M_dataplus = {
<std::allocator> = {
<__gnu_cxx::new_allocator> = {}, },
members of std::__cxx11::basic_string<char, std::char_traits, std::allocator >::_Alloc_hider:
_M_p = 0x2aaab9fe2a00 "OTF2_archive/.metrics."
},
_M_string_length = 22,
{
_M_local_buf = "\036", '\000' <repeats 14 times>,
_M_allocated_capacity = 30
}
},
static my_saved_node_id = 0,
static my_saved_node_count = 2,
rank_thread_map = std::map with 0 elements,
rank_region_map = std::map with 0 elements,
rank_metric_map = std::map with 0 elements,
reduced_region_map = std::map with 0 elements,
reduced_metric_map = std::map with 0 elements
}
(gdb) bt
#0 0x00002aaaafb983b0 in std::equal_to::operator() (this=0x2aaab8a7b730, __x=@0x2aaabd3fe9f4: 2, __y=@0x8: <error reading variable: Cannot access memory at address 0x8>) at /opt/gcc/6.2.0/snos/include/g++/bits/stl_function.h:356
#1 0x00002aaaafc20015 in std::__detail::_Equal_helper<int, int, std::__detail::_Identity, std::equal_to, unsigned long, false>::_S_equals (__eq=..., __extract=..., __k=@0x2aaabd3fe9f4: 2, __n=0x0) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable_policy.h:1331
#2 0x00002aaaafc1f4ec in std::__detail::_Hashtable_base<int, int, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, true, true> >::_M_equals (this=0x2aaab8a7b730, __k=@0x2aaabd3fe9f4: 2, __c=2, __n=0x0) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable_policy.h:1702
#3 0x00002aaaafc1dc87 in std::_Hashtable<int, int, std::allocator, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, true, true> >::_M_find_before_node (this=0x2aaab8a7b730, __n=0, __k=@0x2aaabd3fe9f4: 2, __code=2) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable.h:1420
#4 0x00002aaaafc1bbda in std::_Hashtable<int, int, std::allocator, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, true, true> >::_M_find_node (this=0x2aaab8a7b730, __bkt=0, __key=@0x2aaabd3fe9f4: 2, __c=2) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable.h:634
#5 0x00002aaaafc1bab6 in std::_Hashtable<int, int, std::allocator, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, true, true> >::_M_insert<int, std::__detail::_AllocNode<std::allocator<std::__detail::_Hash_node<int, false> > > >(int&&, std::__detail::_AllocNode<std::allocator<std::__detail::_Hash_node<int, false> > > const&, std::integral_constant<bool, true>) (this=0x2aaab8a7b730, __v=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x6ccc56e, DIE 0x6dba024>, __node_gen=...) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable.h:1686
#6 0x00002aaaafc19e0d in std::__detail::_Insert<int, int, std::allocator, std::__detail::_Identity, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, true, true>, true, true>::insert(int&&) (this=0x2aaab8a7b730, __v=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x6ccc56e, DIE 0x6dbdcee>) at /opt/gcc/6.2.0/snos/include/g++/bits/hashtable_policy.h:810
#7 0x00002aaaafc18cce in std::unordered_set<int, std::hash, std::equal_to, std::allocator >::insert(int&&) (this=0x2aaab8a7b730, __x=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x6ccc56e, DIE 0x6dbdee2>) at /opt/gcc/6.2.0/snos/include/g++/bits/unordered_set.h:417
#8 0x00002aaaafc0d139 in apex::otf2_listener::getEvtWriter (this=0x2aaab8a7b680) at /scratch/snx3000/biddisco/src/hvtkm/hpx/apex/src/apex/otf2_listener.cpp:268
#9 0x00002aaaafc131ab in apex::otf2_listener::on_start (this=0x2aaab8a7b680, id=0x2aaabdc13060) at /scratch/snx3000/biddisco/src/hvtkm/hpx/apex/src/apex/otf2_listener.cpp:1053
#10 0x00002aaaafb8acab in apex::start (timer_name=..., guid=0) at /scratch/snx3000/biddisco/src/hvtkm/hpx/apex/src/apex/apex.cpp:372
#11 0x00002aaaafb8c6cb in apex::register_thread (name=...) at /scratch/snx3000/biddisco/src/hvtkm/hpx/apex/src/apex/apex.cpp:869
#12 0x00002aaaaed8f6ce in hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss_ex (this=0x2aaab9f54000, locality=..., context=0x2aaaafc30f61 <_ZN3hpx7threads10coroutines6detailL18default_stack_sizeE+881> "worker-thread", num=10, postfix=0x2aaaafcd2261 <_ZStL6ignore+1023> "", service_thread=false, ec=...) at /scratch/snx3000/biddisco/src/hvtkm/hpx/src/runtime_impl.cpp:708
#13 0x00002aaaaed8f89c in hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss (this=0x2aaab9f54000, context=0x2aaaafc30f61 <_ZN3hpx7threads10coroutines6detailL18default_stack_sizeE+881> "worker-thread", num=10, postfix=0x2aaaafcd2261 <_ZStL6ignore+1023> "", service_thread=false) at /scratch/snx3000/biddisco/src/hvtkm/hpx/src/runtime_impl.cpp:666
#14 0x00002aaaaeded588 in operator()<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(char const, long unsigned int, char const*, bool), hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&, char const*&, long unsigned int, char const*, bool&> (v0=..., f=(void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> > * const, const char , unsigned long, const char , bool)) 0x2aaaaed8f824 <hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss(char const, unsigned long, char const, bool)>, this=0x2aaabd3fee27) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/invoke.hpp:94
#15 operator()<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(char const*, long unsigned int, char const*, bool), hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&, char const&, long unsigned int, char const*, bool&> (v0=@0x2aaab8acbdf0: 0x2aaab9f54000, f=(void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> > * const, const char , unsigned long, const char , bool)) 0x2aaaaed8f824 <hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss(char const, unsigned long, char const, bool)>, this=) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/invoke.hpp:107
#16 invoke<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::&)(char const*, unsigned long, char const*, bool), hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&, char const&, unsigned long, char const*, bool&> (f=@0x2aaab8acbde0: (void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> > * const, const char , unsigned long, const char , bool)) 0x2aaaaed8f824 <hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss(char const, unsigned long, char const, bool)>) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/invoke.hpp:138
#17 hpx::util::detail::bound_impl<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(char const*, unsigned long, char const*, bool), hpx::util::tuple<hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, char const, hpx::util::detail::placeholder<1ul>, hpx::util::detail::placeholder<2ul>, bool>, hpx::util::tuple<unsigned long&&, char const*&&>, 0ul, 1ul, 2ul, 3ul, 4ul>(void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::&)(char const, unsigned long, char const*, bool), hpx::util::tuple<hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, char const, hpx::util::detail::placeholder<1ul>, hpx::util::detail::placeholder<2ul>, bool>&, hpx::util::tuple<unsigned long&&, char const*&&>&&, hpx::util::detail::pack_c<unsigned long, 0ul, 1ul, 2ul, 3ul, 4ul>) (f=@0x2aaab8acbde0: (void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::)(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> > * const, const char , unsigned long, const char , bool)) 0x2aaaaed8f824 <hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss(char const, unsigned long, char const, bool)>, bound=..., unbound=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x34602e, DIE 0x4d1f06>) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/bind.hpp:232
#18 0x00002aaaaede68f7 in hpx::util::detail::bound<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&&, char const&, hpx::util::detail::placeholder<1ul> const&, hpx::util::detail::placeholder<2ul> const&, bool&&))(char const*, unsigned long, char const*, bool)>::operator()<unsigned long, char const*>(unsigned long&&, char const*&&) (this=0x2aaab8acbde0, vs#0=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x34602e, DIE 0x4be765>, vs#1=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x34602e, DIE 0x4c4e95>) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/bind.hpp:300
#19 0x00002aaaaedd1c0e in operator()<hpx::util::detail::bound<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&&, char const*&, const hpx::util::detail::placeholder<1ul>&, const hpx::util::detail::placeholder<2ul>&, bool&&))(char const*, long unsigned int, char const*, bool)>&, long unsigned int, char const*> (f=..., this=) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/invoke.hpp:36
#20 invoke_r<void, hpx::util::detail::bound<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&&, char const*&, const hpx::util::detail::placeholder<1ul>&, const hpx::util::detail::placeholder<2ul>&, bool&&))(char const*, long unsigned int, char const*, bool)>&, long unsigned int, char const*> (f=...) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/invoke.hpp:146
#21 hpx::util::detail::callable_vtable<void (unsigned long, char const*)>::_invoke<hpx::util::detail::bound<void (hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::(hpx::runtime_impl<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >&&, char const*&, hpx::util::detail::placeholder<1ul> const&, hpx::util::detail::placeholder<2ul> const&, bool&&))(char const*, unsigned long, char const*, bool)> >(void**, unsigned long&&, char const*&&) (f=0x2aaab9f54980, vs#0=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x34602e, DIE 0x4be765>, vs#1=<unknown type in /scratch/snx3000/biddisco/build/hvtkm/lib/libhpxd.so.1, CU 0x34602e, DIE 0x4c4e95>) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/detail/vtable/callable_vtable.hpp:61
#22 0x00002aaaaf52e682 in operator() (vs#1=0x2aaaafcd2261 <_ZStL6ignore+1023> "", vs#0=10, this=0x2aaab9f54978) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/util/detail/basic_function.hpp:197
#23 hpx::threads::policies::callback_notifier::on_start_thread (this=0x2aaab9f54978, num_thread=10) at /scratch/snx3000/biddisco/src/hvtkm/hpx/hpx/runtime/threads/policies/callback_notifier.hpp:39
#24 0x00002aaaaf57207c in hpx::threads::detail::init_tss_helper<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::init_tss_helper (this=0x2aaabd3ff2b0, pool=..., thread_num=10) at /scratch/snx3000/biddisco/src/hvtkm/hpx/src/runtime/threads/detail/thread_pool.cpp:546
#25 0x00002aaaaf54eb40 in hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >::thread_func (this=0x2aaab8a7afc8, num_thread=10, topology=..., startup=...) at /scratch/snx3000/biddisco/src/hvtkm/hpx/src/runtime/threads/detail/thread_pool.cpp:608
#26 0x00002aaaaf59e951 in boost::_mfi::mf3<void, hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, unsigned long, hpx::threads::topology const&, boost::barrier&>::operator() (this=0x2aaaba558fb8, p=0x2aaab8a7afc8, a1=10, a2=..., a3=...) at /users/biddisco/apps/daint/boost/1.63.0/gnu_620/include/boost-1_63/boost/bind/mem_fn_template.hpp:393
#27 0x00002aaaaf59e239 in boost::_bi::list4<boost::_bi::value<hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >>, boost::_bi::value, boost::_bi::value<std::reference_wrapper<hpx::threads::topology const> >, boost::_bi::value<std::reference_wrapperboost::barrier > >::operator()<boost::_mfi::mf3<void, hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, unsigned long, hpx::threads::topology const&, boost::barrier&>, boost::_bi::list0> (this=0x2aaaba558fc8, f=..., a=...) at /users/biddisco/apps/daint/boost/1.63.0/gnu_620/include/boost-1_63/boost/bind/bind.hpp:463
#28 0x00002aaaaf59daf7 in boost::_bi::bind_t<void, boost::_mfi::mf3<void, hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, unsigned long, hpx::threads::topology const&, boost::barrier&>, boost::_bi::list4<boost::_bi::value<hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >>, boost::_bi::value, boost::_bi::value<std::reference_wrapper<hpx::threads::topology const> >, boost::_bi::value<std::reference_wrapperboost::barrier > > >::operator() (this=0x2aaaba558fb8) at /users/biddisco/apps/daint/boost/1.63.0/gnu_620/include/boost-1_63/boost/bind/bind.hpp:1294
#29 0x00002aaaaf59d5a0 in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf3<void, hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >, unsigned long, hpx::threads::topology const&, boost::barrier&>, boost::_bi::list4<boost::_bi::value<hpx::threads::detail::thread_pool<hpx::threads::policies::local_priority_queue_scheduler<boost::mutex, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_fifo, hpx::threads::policies::lockfree_lifo> >*>, boost::_bi::value, boost::_bi::value<std::reference_wrapper<hpx::threads::topology const> >, boost::_bi::value<std::reference_wrapperboost::barrier > > > >::run (this=0x2aaaba558e00) at /users/biddisco/apps/daint/boost/1.63.0/gnu_620/include/boost-1_63/boost/thread/detail/thread.hpp:116
#30 0x00002aaab17193a7 in boost::(anonymous namespace)::thread_proxy (param=0x2aaaba558e00) at libs/thread/src/pthread/thread.cpp:171
(gdb)

[biddisco]

I had some other crashes that occurred even when I disabled APEX in my code. After a lengthy investigation, I wiped the build dir and recompiled all projects from scratch. The code now runs without the aforementioned crash, so I conclude that some stale cmake cache var or other generated file was causing problems. This issue can be closed. should it recur, I will come back to it.