From 3edc1972c87a78843effec8779b3c2bcadfc5c01 Mon Sep 17 00:00:00 2001 From: Brian Hanson Date: Mon, 13 Apr 2026 02:46:08 +0000 Subject: [PATCH] crash handler, model guard, status temp/fan, timeout fix - SIGSEGV/SIGABRT crash handler in ui/main.cc prints stack trace to stderr - Fixed onroad crash: guard update_model() against empty model position data (was dereferencing end()-1 on empty list when modeld not running in bench) - Status window: added device temperature and fan speed - Interactive timeout returns to splash/onroad (not ClearPilotPanel) - bench_cmd dump detects crash loops via UI process uptime check - bench_cmd wait_ready timeout increased to 20s - Restored camerad to bench ignore list (not needed for UI testing) - Updated CLAUDE.md with crash debugging procedures Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 34 ++++++++++++++++++++++++------- selfdrive/clearpilot/bench_cmd.py | 2 +- selfdrive/ui/main.cc | 25 +++++++++++++++++++++++ selfdrive/ui/qt/window.cc | 27 +++++++++++++++++++++--- selfdrive/ui/qt/window.h | 2 ++ selfdrive/ui/ui.cc | 3 +++ 6 files changed, 82 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fe1255a..4721d27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -102,23 +102,42 @@ Bench mode allows testing the onroad UI without a car connected. It runs a fake **IMPORTANT**: Do NOT use `echo` to write bench params — `echo` appends a newline which causes param parsing to fail silently (e.g. gear stays in park). Always use the `bench_cmd.py` tool. ```bash -# 1. Start in bench mode +# 1. Build first +chown -R comma:comma /data/openpilot +su - comma -c "bash /data/openpilot/build_only.sh" + +# 2. Start in bench mode su - comma -c "bash /data/openpilot/launch_openpilot.sh --bench" -# 2. Wait for UI to be ready (polls RPC, up to 10s) +# 3. Wait for UI to be ready (polls RPC every 1s, up to 20s) su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd wait_ready" -# 3. Control vehicle state +# 4. Control vehicle state su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd gear D" su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd speed 20" su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd speedlimit 45" su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd cruise 55" su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd engaged 1" -# 4. Inspect UI widget tree (RPC call, instant response) +# 5. Inspect UI widget tree (RPC call, instant response) su - comma -c "PYTHONPATH=/data/openpilot python3 -m selfdrive.clearpilot.bench_cmd dump" ``` +### Debugging Crashes + +The UI has a SIGSEGV/SIGABRT crash handler (`selfdrive/ui/main.cc`) that prints a stack trace to stderr, captured in the per-process log: + +```bash +# Check for crash traces +grep -A 30 "CRASH" /data/log2/$(ls -t /data/log2/ | head -1)/ui.log + +# Resolve addresses to source lines +addr2line -e /data/openpilot/selfdrive/ui/ui -f 0xADDRESS + +# bench_cmd dump detects crash loops automatically: +# if UI process uptime < 5s, it reports "likely crash looping" +``` + ### UI Introspection RPC The UI process runs a ZMQ REP server at `ipc:///tmp/clearpilot_ui_rpc`. Send `"dump"` to get a recursive widget tree showing class name, visibility, geometry, and stacked layout current indices. This is the primary debugging tool for understanding what the UI is displaying. @@ -147,10 +166,11 @@ The UI process runs a ZMQ REP server at `ipc:///tmp/clearpilot_ui_rpc`. Send `"d | `launch_openpilot.sh` | Accepts `--bench` flag, exports BENCH_MODE env var | | `selfdrive/ui/qt/window.cc` | UI RPC server (`ipc:///tmp/clearpilot_ui_rpc`), widget tree dump | -### Known Issues +### Resolved Issues -- The UI segfaults (exit code -11) when switching to the onroad camera view without a camera feed. The `CameraWidget` / `AnnotatedCameraWidget` crash on empty VisionIPC buffers. camerad is blocked in bench mode, but the `OnroadWindow` still contains a `CameraWidget` that crashes when made visible. Need to either provide fake VisionIPC frames or make the camera widget gracefully handle missing data. -- The `showDriverView` function in `home.cc` was previously overriding park/drive transitions every frame. Fixed to only act when not in started+parked state, but transitions to onroad (drive) view still trigger the camera crash. +- **SIGSEGV in onroad view (fixed)**: `update_model()` in `ui.cc` dereferenced empty model position data when modeld wasn't running. Fixed by guarding against empty `plan_position.getX()`. The root cause was found using the crash handler + `addr2line`. +- **`showDriverView` overriding transitions (fixed)**: was forcing `slayout` to onroad/home every frame at 20Hz, overriding park/drive logic. Fixed to only act when not in started state. +- **Sidebar appearing during onroad transition (fixed)**: `MainWindow::closeSettings()` was re-enabling the sidebar. Fixed by not calling `closeSettings` during `offroadTransition`. ## Session Logging diff --git a/selfdrive/clearpilot/bench_cmd.py b/selfdrive/clearpilot/bench_cmd.py index 8d6c599..991062b 100644 --- a/selfdrive/clearpilot/bench_cmd.py +++ b/selfdrive/clearpilot/bench_cmd.py @@ -57,7 +57,7 @@ def ui_dump(): ctx.term() -def wait_ready(timeout=10): +def wait_ready(timeout=20): """Wait until the UI shows ReadyWindow, up to timeout seconds.""" start = time.time() while time.time() - start < timeout: diff --git a/selfdrive/ui/main.cc b/selfdrive/ui/main.cc index 4903a3d..1086a56 100755 --- a/selfdrive/ui/main.cc +++ b/selfdrive/ui/main.cc @@ -1,4 +1,9 @@ #include +#include +#include +#include +#include +#include #include #include @@ -8,7 +13,27 @@ #include "selfdrive/ui/qt/util.h" #include "selfdrive/ui/qt/window.h" +// CLEARPILOT: crash handler — prints stack trace to stderr on SIGSEGV/SIGABRT +static void crash_handler(int sig) { + const char *sig_name = (sig == SIGSEGV) ? "SIGSEGV" : (sig == SIGABRT) ? "SIGABRT" : "SIGNAL"; + fprintf(stderr, "\n=== CRASH: %s (signal %d) ===\n", sig_name, sig); + + void *frames[64]; + int count = backtrace(frames, 64); + fprintf(stderr, "Backtrace (%d frames):\n", count); + backtrace_symbols_fd(frames, count, STDERR_FILENO); + fprintf(stderr, "=== END CRASH ===\n"); + fflush(stderr); + + // Re-raise to get default behavior (core dump / exit) + signal(sig, SIG_DFL); + raise(sig); +} + int main(int argc, char *argv[]) { + signal(SIGSEGV, crash_handler); + signal(SIGABRT, crash_handler); + setpriority(PRIO_PROCESS, 0, -20); qInstallMessageHandler(swagLogMessageHandler); diff --git a/selfdrive/ui/qt/window.cc b/selfdrive/ui/qt/window.cc index d111502..c37c752 100755 --- a/selfdrive/ui/qt/window.cc +++ b/selfdrive/ui/qt/window.cc @@ -50,10 +50,11 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) { } }); QObject::connect(device(), &Device::interactiveTimeout, [=]() { - if (main_layout->currentWidget() == settingsWindow || - main_layout->currentWidget() == statusWindow) { - closeSettings(); + // CLEARPILOT: on timeout, return to splash/onroad (not ClearPilotPanel) + if (main_layout->currentWidget() != homeWindow) { + main_layout->setCurrentWidget(homeWindow); } + homeWindow->offroadTransition(!uiState()->scene.started); }); // load fonts @@ -229,6 +230,8 @@ StatusWindow::StatusWindow(QWidget *parent) : QFrame(parent) { storage_label = makeRow("Storage"); ram_label = makeRow("Memory"); load_label = makeRow("Load"); + temp_label = makeRow("Temperature"); + fan_label = makeRow("Fan Speed"); ip_label = makeRow("IP Address"); wifi_label = makeRow("WiFi"); vpn_label = makeRow("VPN"); @@ -273,6 +276,24 @@ void StatusWindow::refresh() { load_label->setText(QString("%1 %2 %3").arg(parts[0], parts[1], parts[2])); } + // Temperature + QString temps = shellCmd("cat /sys/class/thermal/thermal_zone*/temp 2>/dev/null | sort -rn | head -1"); + if (!temps.isEmpty()) { + float temp_c = temps.toLong() / 1000.0; + temp_label->setText(QString("%1°C").arg(temp_c, 0, 'f', 1)); + temp_label->setStyleSheet(temp_c > 70 ? "color: #ff4444; font-size: 38px;" : + temp_c > 55 ? "color: #ffaa00; font-size: 38px;" : + "color: white; font-size: 38px;"); + } + + // Fan speed + QString fan = shellCmd("cat /sys/class/hwmon/hwmon*/fan1_input 2>/dev/null | head -1"); + if (fan.isEmpty()) { + // Try reading from deviceState param as fallback + fan = shellCmd("cat /dev/shm/params/d/LastFanSpeed 2>/dev/null"); + } + fan_label->setText(fan.isEmpty() ? "—" : fan + " RPM"); + // IP + WiFi QString ip = shellCmd("ip route get 1.1.1.1 2>/dev/null | head -1 | awk '{print $7}'"); ip_label->setText(ip.isEmpty() ? "No connection" : ip); diff --git a/selfdrive/ui/qt/window.h b/selfdrive/ui/qt/window.h index ea0b5e6..0fbb8e4 100755 --- a/selfdrive/ui/qt/window.h +++ b/selfdrive/ui/qt/window.h @@ -29,6 +29,8 @@ private: QLabel *storage_label; QLabel *ram_label; QLabel *load_label; + QLabel *temp_label; + QLabel *fan_label; QLabel *ip_label; QLabel *wifi_label; QLabel *vpn_label; diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index 67cccde..28a8536 100755 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -93,6 +93,9 @@ void update_model(UIState *s, if (plan_position.getX().size() < model.getPosition().getX().size()) { plan_position = model.getPosition(); } + // CLEARPILOT: guard against empty model data (bench mode, no modeld running) + if (plan_position.getX().size() == 0) return; + float max_distance = scene.unlimited_road_ui_length ? *(plan_position.getX().end() - 1) : std::clamp(*(plan_position.getX().end() - 1), MIN_DRAW_DISTANCE, MAX_DRAW_DISTANCE);