smartbotic-database/packaging/deb/systemd/smartbotic-database.service

[Unit]
Description=Smartbotic Database Service
After=network.target

[Service]
# v1.7.5: Type=notify so dependents block at startup until the service is
# *actually* ready (recovery complete + migrations applied + gRPC listening),
# not just "process spawned." Pre-1.7.5 this was Type=simple, which made
# `After=smartbotic-database.service` on consumers ineffective: tools,
# gateway, server etc. would launch immediately and silently fall back to
# defaults when their startup DB reads timed out against an in-recovery
# database. The service code already calls `sd_notify(READY=1)` at the
# correct point in `service/src/main.cpp` after `service.start()`; with
# Type=notify systemd actually waits for that signal before considering
# the unit active and unblocking dependents.
Type=notify
NotifyAccess=main

# v1.8.1: Cap glibc per-thread arenas to two. With Type=notify + threads
# in eviction / WAL sync / snapshot / gRPC / replication, glibc allocates
# `8 × num_cpus` arenas by default and small frees go on per-thread
# free-lists that never `madvise(MADV_DONTNEED)`. On Zoe (22 threads) the
# committed heap drifted 6.5 GB above the 5 GB the DB itself was tracking
# — a fragmentation gap large enough to OOM-kill the process every ~3h
# even though `estimatedMemoryBytes_` was correct. Two arenas is enough
# for the writer + reader thread split; the small extra contention is
# negligible vs. the RSS savings (measured 7.97 → 5.57 GB on Zoe).
# Operators can override per-deployment by setting MALLOC_ARENA_MAX
# elsewhere; the binary also calls `mallopt(M_ARENA_MAX, 2)` at startup
# as a fallback for non-systemd deploys.
Environment="MALLOC_ARENA_MAX=2"

# v1.9.4: tune jemalloc (which the binary links against, overriding glibc
# malloc/free). `background_thread:true` starts a dedicated thread that
# decays dirty pages on a wallclock timer regardless of allocation
# pressure — fixes the "trim never reclaims because the trailing chunk
# is always in use" pattern we hit with glibc under sustained gRPC
# load. `dirty_decay_ms:1000` says hold idle dirty pages for at most
# one second before madvising them back to the OS — generous enough to
# absorb workload bursts, tight enough to keep RSS close to working set.
# `muzzy_decay_ms:1000` same idea for pages glibc-equivalent would call
# "muzzy" (released but recoverable). Ignored on glibc.
Environment="MALLOC_CONF=background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000"

# Recovery on a large dataset (700k+ docs, multi-MB WAL) routinely takes
# 60–90s. Default systemd start timeout (90s) is too tight; bump to 5 min.
# The service emits `EXTEND_TIMEOUT_USEC=` during long recovery anyway, but
# the static cap is the safety net.
TimeoutStartSec=300
User=smartbotic-db
Group=smartbotic-db
ExecStart=/usr/bin/smartbotic-database --config /etc/smartbotic-database/config.json
WorkingDirectory=/var/lib/smartbotic-database
Restart=on-failure
RestartSec=3

# v1.8.2: stop watchdog must cover the final-on-shutdown snapshot. On Zoe
# (1.85M docs / ~5 GB tracked) serialize takes ~70 s. Pre-1.8.2 this was
# 30 s, so `systemctl restart` always SIGKILLed mid-write — peak RSS hit
# 11 GB during the doomed serialize, the new snapshot never made it to
# disk, and the next boot fell back to WAL-only replay. The shutdown path
# also calls `EXTEND_TIMEOUT_USEC` from `DatabaseService::stop()` so this
# static cap is a safety net, not the primary control.
TimeoutStopSec=300

LimitNOFILE=65536

[Install]
WantedBy=multi-user.target