fix: Log when the Python backend stub exits on a crash signal

src/stub_launcher.cc

@@ -36,6 +36,10 @@
 
 #ifdef _WIN32
 #include <process.h>  // getpid()
+#else
+#include <sys/wait.h>
+
+#include <csignal>
 #endif
 
 extern char** environ;
@@ -955,6 +959,32 @@ StubLauncher::WaitForStubProcess()
 #endif
 }
 
+#ifndef _WIN32
+namespace {
+
+// True when waitpid status is a crash. SIGKILL is excluded: an unhealthy stub
+// is terminated that way on purpose.
+bool
+StubDiedOnCrashSignal(int status)
+{
+  if (!WIFSIGNALED(status)) {
+    return false;
+  }
+  switch (WTERMSIG(status)) {
+    case SIGSEGV:
+    case SIGABRT:
+    case SIGBUS:
+    case SIGILL:
+    case SIGFPE:
+      return true;
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+#endif
+
 bool
 StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds)
 {
@@ -966,14 +996,23 @@ StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds)
     return true;
   }
 
+  auto log_crash_signal = [this](int status) {
+    if (StubDiedOnCrashSignal(status)) {
+      LOG_MESSAGE(
+          TRITONSERVER_LOG_ERROR,
+          (std::string("Python backend stub '") + model_instance_name_ +
+           "' exited on signal " + std::to_string(WTERMSIG(status)))
+              .c_str());
+    }
+  };
+
   for (int64_t elapsed = 0; elapsed < timeout_seconds; ++elapsed) {
-    int status;
+    int status = 0;
     pid_t ret = waitpid(stub_pid_, &status, WNOHANG);
-    if (ret == stub_pid_) {
-      stub_pid_ = 0;
-      return true;
-    }
-    if (ret == -1) {
+    if (ret == stub_pid_ || ret == -1) {
+      if (ret == stub_pid_) {
+        log_crash_signal(status);
+      }
       stub_pid_ = 0;
       return true;
     }
@@ -982,9 +1021,12 @@ StubLauncher::WaitForStubProcessWithTimeout(int64_t timeout_seconds)
 
   // Stub may have exited during the last sleep(1); recheck before killing.
   {
-    int status;
+    int status = 0;
     pid_t ret = waitpid(stub_pid_, &status, WNOHANG);
     if (ret == stub_pid_ || ret == -1) {
+      if (ret == stub_pid_) {
+        log_crash_signal(status);
+      }
       stub_pid_ = 0;
       return true;
     }

src/stub_launcher.h

@@ -162,6 +162,7 @@ class StubLauncher {
   void WaitForStubProcess();
 
   // Wait for stub process with timeout. Returns true if the stub exited.
+  // A stub that exits on SIGSEGV, SIGABRT, SIGBUS, SIGILL, or SIGFPE is logged.
   bool WaitForStubProcessWithTimeout(int64_t timeout_seconds);
 
 #ifndef _WIN32