diff --git a/src/stub_launcher.cc b/src/stub_launcher.cc index c676c5b4..4bc8979c 100644 --- a/src/stub_launcher.cc +++ b/src/stub_launcher.cc @@ -36,6 +36,10 @@ #ifdef _WIN32 #include // getpid() +#else +#include + +#include #endif extern char** environ; @@ -955,6 +959,32 @@ StubLauncher::WaitForStubProcess() #endif } +#ifndef _WIN32 +namespace { + +// True when waitpid status is a crash. SIGKILL is excluded: an unhealthy stub +// is terminated that way on purpose. +bool +StubDiedOnCrashSignal(int status) +{ + if (!WIFSIGNALED(status)) { + return false; + } + switch (WTERMSIG(status)) { + case SIGSEGV: + case SIGABRT: + case SIGBUS: + case SIGILL: + case SIGFPE: + return true; + default: + return false; + } +} + +} // namespace +#endif + bool StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds) { @@ -966,14 +996,23 @@ StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds) return true; } + auto log_crash_signal = [this](int status) { + if (StubDiedOnCrashSignal(status)) { + LOG_MESSAGE( + TRITONSERVER_LOG_ERROR, + (std::string("Python backend stub '") + model_instance_name_ + + "' exited on signal " + std::to_string(WTERMSIG(status))) + .c_str()); + } + }; + for (int64_t elapsed = 0; elapsed < timeout_seconds; ++elapsed) { - int status; + int status = 0; pid_t ret = waitpid(stub_pid_, &status, WNOHANG); - if (ret == stub_pid_) { - stub_pid_ = 0; - return true; - } - if (ret == -1) { + if (ret == stub_pid_ || ret == -1) { + if (ret == stub_pid_) { + log_crash_signal(status); + } stub_pid_ = 0; return true; } @@ -982,9 +1021,12 @@ StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds) // Stub may have exited during the last sleep(1); recheck before killing. { - int status; + int status = 0; pid_t ret = waitpid(stub_pid_, &status, WNOHANG); if (ret == stub_pid_ || ret == -1) { + if (ret == stub_pid_) { + log_crash_signal(status); + } stub_pid_ = 0; return true; } diff --git a/src/stub_launcher.h b/src/stub_launcher.h index ed6abbe7..7e2d2dd4 100644 --- a/src/stub_launcher.h +++ b/src/stub_launcher.h @@ -162,6 +162,7 @@ class StubLauncher { void WaitForStubProcess(); // Wait for stub process with timeout. Returns true if the stub exited. + // A stub that exits on SIGSEGV, SIGABRT, SIGBUS, SIGILL, or SIGFPE is logged. bool WaitForStubProcessWithTimeout(int64_t timeout_seconds); #ifndef _WIN32