feat(config): conf.d drop-in directory with RFC 7396 merge_patch (v1.7.0)

Server now loads /etc/smartbotic-database/config.json first, then merges
*.json files from /etc/smartbotic-database/conf.d/ in lexicographic
order. Drop-ins use RFC 7396 semantics: objects merge recursively,
arrays and scalars replace.

Lets consumer projects (shadowman-cpp, callerai, etc.) ship their own
tuning in their deb postinst without fighting the upstream config.json
as a dpkg conffile. Recommended naming: 00-defaults.json,
50-<project>.json, 99-local.json.

Syntax errors in any file cause exit(11) - operator typos should be
loud. Unknown keys pass through to individual config consumers which
may log WARN but don't block startup.

- New config_loader.{hpp,cpp} exposes loadLayeredConfig().
- DatabaseService::loadConfig() split into file-parse + parseConfig(json)
  so the same Config derivation is reused for layered input.
- server.postinst creates /etc/smartbotic-database/conf.d at 0755.

VERSION

@@ -1 +1 @@
-1.6.3
+1.7.0

packaging/deb/scripts/server.postinst

@@ -17,6 +17,10 @@ case "$1" in
 
         install -d -o smartbotic-db -g smartbotic-db -m 0750 "$DATA_DIR"
         install -d -o root -g smartbotic-db -m 0750 /etc/smartbotic-database
+        # v1.7.0: drop-in config directory. *.json files here are merged
+        # lexicographically into /etc/smartbotic-database/config.json at
+        # service start (RFC 7396 merge_patch semantics).
+        install -d -o root -g smartbotic-db -m 0755 /etc/smartbotic-database/conf.d
 
         # ── Migration from shadowman-database ──
         OLD_SM_DATA="/var/lib/shadowman/data/database"

service/CMakeLists.txt

@@ -29,6 +29,7 @@ set(DATABASE_SERVICE_SOURCES
     src/views/projection.cpp
     src/views/view_manager.cpp
     src/config/collection_config_manager.cpp
+    src/config/config_loader.cpp
 )
 
 # Create executable

service/src/config/config_loader.cpp

@@ -0,0 +1,119 @@
+#include "config_loader.hpp"
+
+#include <algorithm>
+#include <fstream>
+#include <spdlog/spdlog.h>
+#include <stdexcept>
+#include <system_error>
+
+namespace fs = std::filesystem;
+
+namespace smartbotic::database {
+
+namespace {
+
+nlohmann::json parseJsonFile(const fs::path& path) {
+    std::ifstream f(path);
+    if (!f) {
+        throw std::runtime_error("cannot open config file: " + path.string());
+    }
+
+    nlohmann::json out;
+    try {
+        f >> out;
+    } catch (const nlohmann::json::parse_error& e) {
+        // nlohmann carries the byte offset in e.what(); include the filename
+        // so the operator can jump straight to the bad file.
+        throw std::runtime_error("JSON syntax error in " + path.string() +
+                                 ": " + e.what());
+    }
+    return out;
+}
+
+std::vector<fs::path> listJsonDropins(const fs::path& dir) {
+    std::vector<fs::path> out;
+    std::error_code ec;
+    if (!fs::exists(dir, ec) || !fs::is_directory(dir, ec)) {
+        return out;
+    }
+    for (const auto& entry : fs::directory_iterator(dir, ec)) {
+        if (!entry.is_regular_file(ec)) continue;
+        if (entry.path().extension() != ".json") continue;
+        out.push_back(entry.path());
+    }
+    // Sort by filename (not full path) — matches operator expectation of
+    // "00-…, 50-…, 99-…" ordering regardless of the parent directory layout.
+    std::sort(out.begin(), out.end(), [](const fs::path& a, const fs::path& b) {
+        return a.filename().string() < b.filename().string();
+    });
+    return out;
+}
+
+// Shallow list of top-level keys, for a non-secret-leaking debug summary.
+std::string topLevelKeysSummary(const nlohmann::json& config) {
+    if (!config.is_object()) return "<non-object>";
+    std::string out;
+    bool first = true;
+    for (auto it = config.begin(); it != config.end(); ++it) {
+        if (!first) out += ", ";
+        out += it.key();
+        first = false;
+    }
+    return out;
+}
+
+} // anonymous namespace
+
+nlohmann::json loadLayeredConfig(const fs::path& baseFile,
+                                 const fs::path& dropinDir,
+                                 std::vector<fs::path>* mergedFiles) {
+    nlohmann::json config = nlohmann::json::object();
+
+    // Base file (optional).
+    std::error_code ec;
+    const bool baseExists = fs::exists(baseFile, ec);
+    if (baseExists) {
+        spdlog::info("Loading base config: {}", baseFile.string());
+        config = parseJsonFile(baseFile);
+        if (!config.is_object()) {
+            throw std::runtime_error("base config must be a JSON object: " +
+                                     baseFile.string());
+        }
+        if (mergedFiles) mergedFiles->push_back(baseFile);
+    } else {
+        spdlog::info("No base config at {}, starting with empty config",
+                     baseFile.string());
+    }
+
+    // Drop-ins.
+    auto dropins = listJsonDropins(dropinDir);
+    for (const auto& p : dropins) {
+        spdlog::info("Merging drop-in: {}", p.string());
+        nlohmann::json patch = parseJsonFile(p);
+        if (!patch.is_object()) {
+            throw std::runtime_error("drop-in config must be a JSON object: " +
+                                     p.string());
+        }
+        // RFC 7396: arrays and scalars replace; objects merge recursively.
+        config.merge_patch(patch);
+        if (mergedFiles) mergedFiles->push_back(p);
+    }
+
+    if (dropins.empty()) {
+        spdlog::info("No drop-ins found in {}", dropinDir.string());
+    } else {
+        spdlog::info("Config assembled from {} file(s) ({} base + {} drop-in(s))",
+                     (baseExists ? 1u : 0u) + dropins.size(),
+                     baseExists ? 1 : 0,
+                     dropins.size());
+    }
+
+    // Log the top-level key set at DEBUG — values may contain secrets, keys
+    // are safe and give operators enough to diagnose "did my drop-in land?".
+    spdlog::debug("Effective config top-level keys: {}",
+                  topLevelKeysSummary(config));
+
+    return config;
+}
+
+} // namespace smartbotic::database

service/src/config/config_loader.hpp

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <filesystem>
+#include <nlohmann/json.hpp>
+#include <string>
+#include <vector>
+
+namespace smartbotic::database {
+
+/**
+ * Load a layered configuration:
+ *   - baseFile is loaded first (if it exists — a missing base is allowed,
+ *     giving an empty JSON object)
+ *   - All *.json files in dropinDir are merged in lexicographic filename
+ *     order using nlohmann::json::merge_patch() (RFC 7396: objects merge
+ *     recursively; arrays and scalars replace).
+ *
+ * Rationale: consumer projects (shadowman-cpp, callerai, …) can ship their
+ * own tuning via a drop-in without fighting the upstream config.json as a
+ * dpkg conffile. Recommended naming: 00-defaults.json, 50-<project>.json,
+ * 99-local.json.
+ *
+ * @param baseFile     The main config.json (may not exist).
+ * @param dropinDir    The conf.d directory (may not exist).
+ * @param mergedFiles  Out parameter — populated with the absolute paths of
+ *                     every file that contributed to the final config, in
+ *                     merge order. Optional.
+ * @return             The merged JSON object.
+ * @throws std::runtime_error on any JSON syntax error. The caller is
+ *                     expected to translate this to exit code 11.
+ */
+nlohmann::json loadLayeredConfig(const std::filesystem::path& baseFile,
+                                 const std::filesystem::path& dropinDir,
+                                 std::vector<std::filesystem::path>* mergedFiles = nullptr);
+
+} // namespace smartbotic::database

service/src/database_service.cpp

@@ -279,11 +279,14 @@ DatabaseService::Config DatabaseService::loadConfig(const std::filesystem::path&
 
     nlohmann::json json;
     file >> json;
+    return parseConfig(json);
+}
 
+DatabaseService::Config DatabaseService::parseConfig(const nlohmann::json& json) {
     Config config;
 
     // Check for both "storage" and "database" keys for backward compatibility
-    nlohmann::json* dbConfig = nullptr;
+    const nlohmann::json* dbConfig = nullptr;
     if (json.contains("database")) {
         dbConfig = &json["database"];
     } else if (json.contains("storage")) {
@@ -291,7 +294,7 @@ DatabaseService::Config DatabaseService::loadConfig(const std::filesystem::path&
     }
 
     if (dbConfig) {
-        auto& db = *dbConfig;
+        const auto& db = *dbConfig;
 
         config.nodeId = db.value("node_id", config.nodeId);
         config.bindAddress = db.value("bind_address", config.bindAddress);

service/src/database_service.hpp

@@ -161,6 +161,14 @@ public:
      */
     static Config loadConfig(const std::filesystem::path& configPath);
 
+    /**
+     * Parse configuration from an already-loaded JSON object. Used by the
+     * conf.d drop-in loader (see config/config_loader.hpp) after it has
+     * merged /etc/smartbotic-database/config.json with every *.json in
+     * /etc/smartbotic-database/conf.d/.
+     */
+    static Config parseConfig(const nlohmann::json& json);
+
 private:
     void setupComponents();
     void startGrpcServer();

service/src/main.cpp

@@ -1,4 +1,5 @@
 #include "database_service.hpp"
+#include "config/config_loader.hpp"
 #include "persistence/persistence_manager.hpp"
 
 #include <spdlog/spdlog.h>
@@ -10,6 +11,7 @@
 #include <iostream>
 #include <optional>
 #include <string>
+#include <vector>
 
 #ifdef HAVE_SYSTEMD
 #include <systemd/sd-daemon.h>
@@ -158,14 +160,30 @@ int main(int argc, char* argv[]) {
         // Initialize logging
         initLogging("database");
 
-        // Find configuration file
+        // Find configuration file (base) + sibling conf.d/ drop-in directory
         auto configPath = findConfigFile(argc, argv);
-        spdlog::info("Loading configuration from {}", configPath.string());
+        auto confDir = configPath.parent_path() / "conf.d";
+        spdlog::info("Loading configuration from {} (drop-ins: {})",
+                     configPath.string(), confDir.string());
 
-        // Load configuration
+        // Load configuration. The loader accepts a missing base file (empty {})
+        // and a missing conf.d/. If either exists we parse it; syntax errors
+        // in any file are fatal with exit code 11 (config invalid).
         smartbotic::database::DatabaseService::Config config;
-        if (std::filesystem::exists(configPath)) {
-            config = smartbotic::database::DatabaseService::loadConfig(configPath);
+        const bool baseExists = std::filesystem::exists(configPath);
+        const bool dropinsExist = std::filesystem::exists(confDir);
+
+        if (baseExists || dropinsExist) {
+            try {
+                std::vector<std::filesystem::path> mergedFiles;
+                nlohmann::json merged = smartbotic::database::loadLayeredConfig(
+                    configPath, confDir, &mergedFiles);
+                config = smartbotic::database::DatabaseService::parseConfig(merged);
+            } catch (const std::exception& e) {
+                std::cerr << "Config error: " << e.what() << std::endl;
+                spdlog::error("Config error: {}", e.what());
+                return 11;
+            }
         } else {
             spdlog::warn("Configuration file not found, using defaults");
             // Set default data directory