|
|
@@ -16,6 +16,7 @@
|
|
|
// candidate file, read the flag, learn whether a peer namespace has it.
|
|
|
|
|
|
#include <atomic>
|
|
|
+#include <chrono>
|
|
|
#include <filesystem>
|
|
|
#include <fstream>
|
|
|
#include <iostream>
|
|
|
@@ -50,6 +51,20 @@ struct TmpFm {
|
|
|
std::string dir;
|
|
|
std::unique_ptr<FileManager> fm;
|
|
|
|
|
|
+ explicit TmpFm(std::unordered_map<std::string, uint32_t> ttlByType) {
|
|
|
+ static std::atomic<int> counter{0};
|
|
|
+ dir = "/tmp/sbdb-file-project-test-" + std::to_string(::getpid()) + "-ttl" +
|
|
|
+ std::to_string(counter.fetch_add(1));
|
|
|
+ std::error_code ec;
|
|
|
+ fs::remove_all(dir, ec);
|
|
|
+ fs::create_directories(dir);
|
|
|
+ FileManager::Config cfg;
|
|
|
+ cfg.filesDir = dir;
|
|
|
+ cfg.defaultTtlSecondsByType = std::move(ttlByType);
|
|
|
+ fm = std::make_unique<FileManager>(cfg);
|
|
|
+ fm->start();
|
|
|
+ }
|
|
|
+
|
|
|
TmpFm() {
|
|
|
static std::atomic<int> counter{0};
|
|
|
dir = "/tmp/sbdb-file-project-test-" + std::to_string(::getpid()) + "-" +
|
|
|
@@ -86,6 +101,23 @@ struct TmpFm {
|
|
|
out << json;
|
|
|
}
|
|
|
|
|
|
+ fs::path blobPathFor(const std::string& checksum) const {
|
|
|
+ std::string hash = checksum;
|
|
|
+ if (hash.rfind("sha256:", 0) == 0) hash = hash.substr(7);
|
|
|
+ const std::string prefix = hash.size() >= 2 ? hash.substr(0, 2) : "00";
|
|
|
+ return fs::path(dir) / "blobs" / prefix / hash;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Rewrite a record's expiry in place, to age a file without sleeping.
|
|
|
+ void setExpiry(const std::string& id, uint64_t expiresAt) const {
|
|
|
+ auto rp = recordPath(id);
|
|
|
+ nlohmann::json j;
|
|
|
+ { std::ifstream in(rp); in >> j; }
|
|
|
+ j["expiresAt"] = expiresAt;
|
|
|
+ std::ofstream out(rp);
|
|
|
+ out << j.dump(2);
|
|
|
+ }
|
|
|
+
|
|
|
nlohmann::json rawRecord(const std::string& id) const {
|
|
|
std::ifstream in(recordPath(id));
|
|
|
nlohmann::json j;
|
|
|
@@ -216,6 +248,168 @@ void test_stamping_leaves_existing_projects_alone() {
|
|
|
"and it stays in its own project, not moved to default");
|
|
|
}
|
|
|
|
|
|
+// v2.8.0 — file TTL.
|
|
|
+//
|
|
|
+// The assertion that matters most is `test_expiry_respects_shared_blobs`. Blobs
|
|
|
+// are content-addressed and deduplicated across projects, so "delete the file"
|
|
|
+// cannot mean "unlink the blob": two records may point at the same bytes. Expiry
|
|
|
+// therefore routes through deleteFile(), which decrements the refcount and
|
|
|
+// unlinks only on the last reference. Getting that wrong destroys live data
|
|
|
+// belonging to a file that has not expired.
|
|
|
+
|
|
|
+uint64_t now_ms_test() {
|
|
|
+ return static_cast<uint64_t>(
|
|
|
+ std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
|
+ std::chrono::system_clock::now().time_since_epoch()).count());
|
|
|
+}
|
|
|
+
|
|
|
+void test_ttl_zero_never_expires() {
|
|
|
+ TmpFm fm;
|
|
|
+ auto info = mk("keep.bin", "acme");
|
|
|
+ info.expiresAt = 0;
|
|
|
+ auto id = fm->storeFile({1, 2, 3}, info).id;
|
|
|
+ check(fm->getFileInfo(id).has_value(), "ttl 0 file is readable");
|
|
|
+ check(fm->purgeExpired() == 0, "ttl 0 file is not swept");
|
|
|
+ check(fm->getFileInfo(id).has_value(), "and survives a sweep");
|
|
|
+}
|
|
|
+
|
|
|
+void test_expired_file_is_invisible_before_the_sweep() {
|
|
|
+ TmpFm fm;
|
|
|
+ auto info = mk("gone.bin", "acme");
|
|
|
+ info.expiresAt = now_ms_test() - 1000; // already expired
|
|
|
+ auto id = fm->storeFile({4, 5, 6}, info).id;
|
|
|
+
|
|
|
+ // No sweep has run yet - expiry must still be immediate to every caller.
|
|
|
+ check(!fm->getFileInfo(id).has_value(), "expired file reads as absent at once");
|
|
|
+ check(!fm->getFileInfoIn("acme", id).has_value(), "and through the scoped accessor");
|
|
|
+ check(fm->listFiles("acme").totalCount == 0, "and is absent from listings");
|
|
|
+ bool threw = false;
|
|
|
+ try { (void)fm->readFileIn("acme", id); } catch (const std::exception&) { threw = true; }
|
|
|
+ check(threw, "and its bytes are not served");
|
|
|
+}
|
|
|
+
|
|
|
+void test_expiry_deletes_the_blob_not_just_the_record() {
|
|
|
+ TmpFm fm;
|
|
|
+ auto info = mk("bytes.bin", "acme");
|
|
|
+ info.expiresAt = now_ms_test() - 1;
|
|
|
+ std::vector<uint8_t> payload{9, 9, 9, 9, 9};
|
|
|
+ auto res = fm->storeFile(payload, info);
|
|
|
+
|
|
|
+ auto blob = fm.blobPathFor(res.checksum);
|
|
|
+ check(fs::exists(blob), "blob written");
|
|
|
+ check(fs::exists(fm.recordPath(res.id)), "record written");
|
|
|
+
|
|
|
+ check(fm->purgeExpired() == 1, "sweep deletes the expired file");
|
|
|
+ check(!fs::exists(fm.recordPath(res.id)), "metadata record removed");
|
|
|
+ check(!fs::exists(blob),
|
|
|
+ "THE BLOB IS REMOVED - reclaiming metadata alone would leak the bytes");
|
|
|
+ check(fm->getRefCount(res.checksum) == 0, "refcount cleared");
|
|
|
+}
|
|
|
+
|
|
|
+void test_expiry_respects_shared_blobs() {
|
|
|
+ TmpFm fm;
|
|
|
+ std::vector<uint8_t> shared{7, 7, 7, 7};
|
|
|
+
|
|
|
+ auto a = mk("expiring.bin", "acme");
|
|
|
+ a.expiresAt = now_ms_test() - 1; // expires now
|
|
|
+ auto ra = fm->storeFile(shared, a);
|
|
|
+
|
|
|
+ auto b = mk("permanent.bin", "acme");
|
|
|
+ b.expiresAt = 0; // never
|
|
|
+ auto rb = fm->storeFile(shared, b);
|
|
|
+
|
|
|
+ check(ra.checksum == rb.checksum, "identical bytes share one blob");
|
|
|
+ check(fm->getRefCount(ra.checksum) == 2, "two references to that blob");
|
|
|
+
|
|
|
+ auto blob = fm.blobPathFor(ra.checksum);
|
|
|
+ check(fm->purgeExpired() == 1, "only the expired record is swept");
|
|
|
+ check(!fs::exists(fm.recordPath(ra.id)), "expired record gone");
|
|
|
+ check(fs::exists(fm.recordPath(rb.id)), "the permanent record survives");
|
|
|
+ check(fs::exists(blob),
|
|
|
+ "the SHARED BLOB survives - unlinking it would destroy live data");
|
|
|
+ check(fm->getRefCount(ra.checksum) == 1, "refcount decremented to one");
|
|
|
+ check(fm->readFileIn("acme", rb.id).size() == shared.size(),
|
|
|
+ "and the surviving file still serves its bytes");
|
|
|
+
|
|
|
+ // Now expire the survivor too: the blob must finally go.
|
|
|
+ fm.setExpiry(rb.id, now_ms_test() - 1);
|
|
|
+ check(fm->purgeExpired() == 1, "the last reference is swept");
|
|
|
+ check(!fs::exists(blob), "blob unlinked once the last reference goes");
|
|
|
+ check(fm->getRefCount(ra.checksum) == 0, "refcount cleared");
|
|
|
+}
|
|
|
+
|
|
|
+void test_sweep_is_idempotent() {
|
|
|
+ TmpFm fm;
|
|
|
+ auto info = mk("x.bin", "acme");
|
|
|
+ info.expiresAt = now_ms_test() - 1;
|
|
|
+ fm->storeFile({1}, info);
|
|
|
+ check(fm->purgeExpired() == 1, "first sweep deletes it");
|
|
|
+ check(fm->purgeExpired() == 0, "second sweep is a no-op");
|
|
|
+}
|
|
|
+
|
|
|
+void test_legacy_records_never_expire() {
|
|
|
+ TmpFm fm;
|
|
|
+ // Pre-2.8.0 record: no expiresAt key at all.
|
|
|
+ fm.writeLegacyRecord("old", R"({"id":"old","name":"a","fileType":"document","project":"acme"})");
|
|
|
+ check(fm->getFileInfo("old").has_value(), "legacy record readable");
|
|
|
+ check(fm->purgeExpired() == 0,
|
|
|
+ "a record with no expiresAt must never be swept - absent means never");
|
|
|
+}
|
|
|
+
|
|
|
+// v2.8.0 — per-fileType default retention: the analogue of a collection's
|
|
|
+// defaultTtlSeconds, so retention is set once by an operator instead of relied
|
|
|
+// on at every upload site.
|
|
|
+void test_default_ttl_by_type() {
|
|
|
+ TmpFm fm({{"generated", 3600}});
|
|
|
+ auto a = fm->storeFile({1}, mk("g.bin", "acme", "generated"));
|
|
|
+ auto ga = fm->getFileInfo(a.id);
|
|
|
+ check(ga && ga->expiresAt > 0, "a type with a default gets an expiry");
|
|
|
+
|
|
|
+ auto b = fm->storeFile({2}, mk("d.bin", "acme", "document"));
|
|
|
+ auto gb = fm->getFileInfo(b.id);
|
|
|
+ check(gb && gb->expiresAt == 0, "a type with no default keeps forever");
|
|
|
+}
|
|
|
+
|
|
|
+void test_default_ttl_matching_order() {
|
|
|
+ // Most specific wins: project+type, then type, then project wildcard, then
|
|
|
+ // the global wildcard.
|
|
|
+ TmpFm fm({{"acme:generated", 10}, {"generated", 20}, {"acme:*", 30}, {"*", 40}});
|
|
|
+ check(fm->defaultTtlFor("acme", "generated") == 10, "project+type is most specific");
|
|
|
+ check(fm->defaultTtlFor("other", "generated") == 20, "type matches any project");
|
|
|
+ check(fm->defaultTtlFor("acme", "plugin") == 30, "project wildcard covers other types");
|
|
|
+ check(fm->defaultTtlFor("other", "plugin") == 40, "global wildcard is the last resort");
|
|
|
+ check(fm->defaultTtlFor("other", "") == 40, "empty type still falls through");
|
|
|
+}
|
|
|
+
|
|
|
+void test_explicit_expiry_overrides_the_default() {
|
|
|
+ TmpFm fm({{"generated", 3600}});
|
|
|
+ auto info = mk("pinned.bin", "acme", "generated");
|
|
|
+ info.expiresAt = now_ms_test() + 60000; // caller decided
|
|
|
+ auto r = fm->storeFile({3}, info);
|
|
|
+ auto got = fm->getFileInfo(r.id);
|
|
|
+ check(got && got->expiresAt == info.expiresAt,
|
|
|
+ "an explicit expiry is not overwritten by the type default");
|
|
|
+}
|
|
|
+
|
|
|
+void test_never_expires_opts_out_of_the_default() {
|
|
|
+ TmpFm fm({{"generated", 1}});
|
|
|
+ auto info = mk("keep.bin", "acme", "generated");
|
|
|
+ info.neverExpires = true; // explicit ttl=0 from the wire
|
|
|
+ auto r = fm->storeFile({4}, info);
|
|
|
+ auto got = fm->getFileInfo(r.id);
|
|
|
+ check(got && got->expiresAt == 0,
|
|
|
+ "an explicit never-expire beats the configured default");
|
|
|
+ check(fm->purgeExpired() == 0, "and the sweeper leaves it alone");
|
|
|
+}
|
|
|
+
|
|
|
+void test_no_config_means_no_expiry() {
|
|
|
+ TmpFm fm; // no defaults at all
|
|
|
+ auto r = fm->storeFile({5}, mk("x.bin", "acme", "generated"));
|
|
|
+ auto got = fm->getFileInfo(r.id);
|
|
|
+ check(got && got->expiresAt == 0,
|
|
|
+ "with no retention configured nothing expires - the default default");
|
|
|
+}
|
|
|
+
|
|
|
} // namespace
|
|
|
|
|
|
int main() {
|
|
|
@@ -228,6 +422,17 @@ int main() {
|
|
|
test_dedup_flag_does_not_leak_across_projects();
|
|
|
test_stamp_projects_is_idempotent();
|
|
|
test_stamping_leaves_existing_projects_alone();
|
|
|
+ test_ttl_zero_never_expires();
|
|
|
+ test_expired_file_is_invisible_before_the_sweep();
|
|
|
+ test_expiry_deletes_the_blob_not_just_the_record();
|
|
|
+ test_expiry_respects_shared_blobs();
|
|
|
+ test_sweep_is_idempotent();
|
|
|
+ test_legacy_records_never_expire();
|
|
|
+ test_default_ttl_by_type();
|
|
|
+ test_default_ttl_matching_order();
|
|
|
+ test_explicit_expiry_overrides_the_default();
|
|
|
+ test_never_expires_opts_out_of_the_default();
|
|
|
+ test_no_config_means_no_expiry();
|
|
|
|
|
|
std::cout << "passed: " << g_pass << ", failed: " << g_fail << "\n";
|
|
|
return g_fail == 0 ? 0 : 1;
|
