Unable to find injection library libsanitizer-collection.so while trying to use compute-sanitizer

Question

compute-sanitizer --tool memcheck my_cuda_program is what I am trying to do. I am trying this because I got Thread 1 my_cuda_program received signal CUDA_EXCEPTION_5, Warp Out-of-range Address. when I ran my program via cuda-gdb.

However, I get

========= COMPUTE-SANITIZER
========= Unable to find injection library libsanitizer-collection.so

this as a output. However, libsanitizer-collection.so exists in my /usr/lib/nvidia-cuda-toolkit.

My nvcc version is 12.4, which is why I am using compute sanitizer. I compile my program using cmake, which looks like this

cmake_minimum_required(VERSION 3.29.2)
project(nnbody VERSION 0.1.0 LANGUAGES CUDA CXX)

include(CTest)
enable_testing()

add_subdirectory(third-party/yaml-cpp) 

enable_language(CUDA)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")

set(CMAKE_CUDA_STANDARD 20)
set(CMAKE_CUDA_STANDARD_REQUIRED ON)
# set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -arch=sm_60")

# check openmp
find_package(OpenMP REQUIRED)
if (OPENMP_FOUND)
  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
  set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
else ()
  message (FATAL_ERROR "The compiler does not support OpenMP.")
endif()

# check BLAS and LAPACK
find_package(BLAS REQUIRED)
find_package(LAPACK REQUIRED)
find_package(FFTW3 REQUIRED)

set(VTK_DIR "/usr/local/include/vtk-9.3/")
find_package(VTK REQUIRED)

file(GLOB ALL_FILES_SRC
     "src/*.cuh"
     "src/*.cu"
     "src/*.h"
     "src/*.hpp"
     "src/*.c"
     "src/*.cpp"
)

set_source_files_properties(ALL_FILES_SRC LANGUAGE CUDA)

add_executable(nnbody ${ALL_FILES_SRC})
target_link_libraries(nnbody PRIVATE ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ${FFTW3_LIBRARIES} ${VTK_LIBRARIES} yaml-cpp)
set_property(TARGET nnbody PROPERTY CUDA_ARCHITECTURES 60 86)


set(CPACK_PROJECT_NAME ${PROJECT_NAME})
set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
include(CPack)

I don't know what to try..

Answer 1

I had the same bug where /usr/bin/compute-sanitizer would try to load libsanitizer-collection.so from /usr/bin, but it is located at /usr/lib/nvidia-cuda-toolkit/compute-sanitizer/libsanitizer-collection.so. (*buntu 22.04, NVIDIA driver version 550, CUDA 12.4)

You can confirm by using strace to see where compute-sanitizer tries to find the library:

$ strace compute-sanitizer ./main 2>&1 | grep libsanitizer-collection.so
stat("/usr/bin/libsanitizer-collection.so", 0x7ffe926f6c20) = -1 ENOENT (No such file or directory)
write(1, "========= Unable to find injecti"..., 70========= Unable to find injection library libsanitizer-collection.so

The actual location of libsanitizer-collection.so can be determined with the locate command:

$ locate libsanitizer-collection.so
/usr/lib/nvidia-cuda-toolkit/compute-sanitizer/libsanitizer-collection.so

I solved the problem by moving compute-sanitizer to the /usr/lib/nvidia-cuda-toolkit/compute-sanitizer directory and linking it back to /usr/bin.

sudo mv /usr/bin/compute-sanitizer /usr/lib/nvidia-cuda-toolkit/compute-sanitizer/compute-sanitizer
sudo ln -s /usr/lib/nvidia-cuda-toolkit/compute-sanitizer/compute-sanitizer /usr/bin/compute-sanitizer

I then get a new error, but that might be unrelated:

========= COMPUTE-SANITIZER
========= Target application terminated before first instrumented API call
========= Error: couldn't find exit code.