clearpilot/system/loggerd/deleter.py

#!/usr/bin/env python3
import os
import shutil
import sys
import threading
from openpilot.system.hardware.hw import Paths
from openpilot.common.swaglog import cloudlog
from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
from openpilot.system.loggerd.uploader import listdir_by_creation
from openpilot.system.loggerd.xattr_cache import getxattr

# CLEARPILOT: bumped from 5 GB to 9 GB so dashcam footage has headroom
MIN_BYTES = 9 * 1024 * 1024 * 1024
MIN_PERCENT = 10

DELETE_LAST = ['boot', 'crash']

# CLEARPILOT: dashcam footage directory (trip dirs YYYYMMDD-HHMMSS/ with .mp4 segments)
VIDEOS_DIR = '/data/media/0/videos'
# CLEARPILOT: max total size for /data/log2 session logs
LOG2_MAX_BYTES = 4 * 1024 * 1024 * 1024

PRESERVE_ATTR_NAME = 'user.preserve'
PRESERVE_ATTR_VALUE = b'1'
PRESERVE_COUNT = 5


def has_preserve_xattr(d: str) -> bool:
  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE


def get_preserved_segments(dirs_by_creation: list[str]) -> list[str]:
  preserved = []
  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
    if n == PRESERVE_COUNT:
      break
    date_str, _, seg_str = d.rpartition("--")

    # ignore non-segment directories
    if not date_str:
      continue
    try:
      seg_num = int(seg_str)
    except ValueError:
      continue

    # preserve segment and its prior
    preserved.append(d)
    preserved.append(f"{date_str}--{seg_num - 1}")

  return preserved


def delete_oldest_video():
  """CLEARPILOT: prune dashcam footage when disk space is low.
  Trip directories are /data/media/0/videos/YYYYMMDD-HHMMSS/ containing .mp4
  segments. Deletes entire oldest trip directory first. If only one trip
  remains (the active one), deletes individual segments oldest-first within
  it. Also cleans up legacy flat .mp4 files.
  Returns True if something was deleted."""
  try:
    if not os.path.isdir(VIDEOS_DIR):
      return False

    legacy_files = []
    trip_dirs = []
    for entry in os.listdir(VIDEOS_DIR):
      path = os.path.join(VIDEOS_DIR, entry)
      if os.path.isfile(path) and entry.endswith('.mp4'):
        legacy_files.append(entry)
      elif os.path.isdir(path):
        trip_dirs.append(entry)

    if legacy_files:
      legacy_files.sort()
      delete_path = os.path.join(VIDEOS_DIR, legacy_files[0])
      print(f"CLP deleter: deleting legacy video {delete_path}", file=sys.stderr, flush=True)
      os.remove(delete_path)
      return True

    if not trip_dirs:
      return False

    trip_dirs.sort()  # timestamp names = chronological order

    if len(trip_dirs) > 1:
      delete_path = os.path.join(VIDEOS_DIR, trip_dirs[0])
      print(f"CLP deleter: deleting trip {delete_path}", file=sys.stderr, flush=True)
      shutil.rmtree(delete_path)
      return True

    # Only one trip left (likely active) — drop its oldest segment
    trip_path = os.path.join(VIDEOS_DIR, trip_dirs[0])
    segments = sorted(f for f in os.listdir(trip_path) if f.endswith('.mp4'))
    if not segments:
      return False
    delete_path = os.path.join(trip_path, segments[0])
    print(f"CLP deleter: deleting segment {delete_path}", file=sys.stderr, flush=True)
    os.remove(delete_path)
    return True
  except OSError as e:
    print(f"CLP deleter: issue deleting video from {VIDEOS_DIR}: {e}", file=sys.stderr, flush=True)
    return False


def cleanup_log2():
  """CLEARPILOT: keep /data/log2 session logs under LOG2_MAX_BYTES total.
  Deletes oldest dated session directories first (the 'current' symlink/dir
  is preserved). Runs even when disk space is fine."""
  log_base = "/data/log2"
  if not os.path.isdir(log_base):
    return
  dirs = []
  for entry in sorted(os.listdir(log_base)):
    if entry == "current":
      continue
    path = os.path.join(log_base, entry)
    if os.path.isdir(path) and not os.path.islink(path):
      try:
        size = sum(f.stat().st_size for f in os.scandir(path) if f.is_file())
      except OSError:
        size = 0
      dirs.append((entry, path, size))
  total = sum(s for _, _, s in dirs)
  current = os.path.join(log_base, "current")
  if os.path.isdir(current):
    try:
      total += sum(f.stat().st_size for f in os.scandir(current) if f.is_file())
    except OSError:
      pass
  while total > LOG2_MAX_BYTES and dirs:
    entry, path, size = dirs.pop(0)
    try:
      print(f"CLP deleter: deleting log session {path} ({size // 1024 // 1024} MB)",
            file=sys.stderr, flush=True)
      shutil.rmtree(path)
      total -= size
    except OSError as e:
      print(f"CLP deleter: issue deleting log {path}: {e}", file=sys.stderr, flush=True)


def deleter_thread(exit_event):
  while not exit_event.is_set():
    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT

    if out_of_percent or out_of_bytes:
      # CLEARPILOT: drop oldest dashcam footage first, fall back to log segments
      if delete_oldest_video():
        exit_event.wait(.1)
        continue

      dirs = listdir_by_creation(Paths.log_root())

      # skip deleting most recent N preserved segments (and their prior segment)
      preserved_dirs = get_preserved_segments(dirs)

      # remove the earliest directory we can
      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
        delete_path = os.path.join(Paths.log_root(), delete_dir)

        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
          continue

        try:
          cloudlog.info(f"deleting {delete_path}")
          if os.path.isfile(delete_path):
            os.remove(delete_path)
          else:
            shutil.rmtree(delete_path)
          break
        except OSError:
          cloudlog.exception(f"issue deleting {delete_path}")
      exit_event.wait(.1)
    else:
      # CLEARPILOT: keep /data/log2 quota even when disk space is fine
      cleanup_log2()
      exit_event.wait(30)


def main():
  deleter_thread(threading.Event())


if __name__ == "__main__":
  main()