|
@@ -19,6 +19,16 @@ grpc::Status readOnlyStatus(const std::string& rpcName, const DatabaseService& s
|
|
|
rpcName + " rejected: database is in read-only mode. " + svc.readOnlyReason());
|
|
rpcName + " rejected: database is in read-only mode. " + svc.readOnlyReason());
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+// Returns a Status for rejecting writes when memory pressure is EMERGENCY.
|
|
|
|
|
+// Shares the RESOURCE_EXHAUSTED code with gRPC concurrency cap so clients
|
|
|
|
|
+// can reuse the same retry path.
|
|
|
|
|
+grpc::Status memoryEmergencyStatus(const std::string& rpcName, const MemoryStore& store) {
|
|
|
|
|
+ return grpc::Status(grpc::StatusCode::RESOURCE_EXHAUSTED,
|
|
|
|
|
+ rpcName + " rejected: memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store.pressurePercent()) + "% of max, threshold " +
|
|
|
|
|
+ std::to_string(store.emergencyPercent()) + "%). Retry after backoff.");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
// RAII guard for a streaming-RPC concurrency counter (v1.6.2).
|
|
// RAII guard for a streaming-RPC concurrency counter (v1.6.2).
|
|
|
// Increments on construction (if under limit), decrements on destruction.
|
|
// Increments on construction (if under limit), decrements on destruction.
|
|
|
// Check ok() before using — false means the limit was reached and the
|
|
// Check ok() before using — false means the limit was reached and the
|
|
@@ -88,6 +98,9 @@ grpc::Status DatabaseGrpcImpl::Insert(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("Insert", service_);
|
|
return readOnlyStatus("Insert", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("Insert", store_);
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -194,6 +207,12 @@ grpc::Status DatabaseGrpcImpl::Update(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
response->set_success(false);
|
|
response->set_success(false);
|
|
|
response->set_error("cannot write to view '" + request->collection() + "': views are read-only");
|
|
response->set_error("cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -273,6 +292,12 @@ grpc::Status DatabaseGrpcImpl::PatchDocument(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
response->set_success(false);
|
|
response->set_success(false);
|
|
|
response->set_error("cannot write to view '" + request->collection() + "': views are read-only");
|
|
response->set_error("cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -326,6 +351,9 @@ grpc::Status DatabaseGrpcImpl::Upsert(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("Upsert", service_);
|
|
return readOnlyStatus("Upsert", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("Upsert", store_);
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -400,6 +428,9 @@ grpc::Status DatabaseGrpcImpl::Delete(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("Delete", service_);
|
|
return readOnlyStatus("Delete", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("Delete", store_);
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -482,6 +513,12 @@ grpc::Status DatabaseGrpcImpl::RestoreVersion(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
uint64_t newVersion = store_.restoreToVersion(
|
|
uint64_t newVersion = store_.restoreToVersion(
|
|
|
request->collection(), request->id(),
|
|
request->collection(), request->id(),
|
|
|
request->version(), request->actor());
|
|
request->version(), request->actor());
|
|
@@ -507,6 +544,12 @@ grpc::Status DatabaseGrpcImpl::RestoreToDate(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
auto [restoredVersion, newVersion] = store_.restoreToDate(
|
|
auto [restoredVersion, newVersion] = store_.restoreToDate(
|
|
|
request->collection(), request->id(),
|
|
request->collection(), request->id(),
|
|
|
request->timestamp(), request->actor());
|
|
request->timestamp(), request->actor());
|
|
@@ -533,6 +576,9 @@ grpc::Status DatabaseGrpcImpl::BatchInsert(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("BatchInsert", service_);
|
|
return readOnlyStatus("BatchInsert", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("BatchInsert", store_);
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -601,6 +647,9 @@ grpc::Status DatabaseGrpcImpl::BatchDelete(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("BatchDelete", service_);
|
|
return readOnlyStatus("BatchDelete", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("BatchDelete", store_);
|
|
|
|
|
+ }
|
|
|
if (view_manager_.isView(request->collection())) {
|
|
if (view_manager_.isView(request->collection())) {
|
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
|
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
"cannot write to view '" + request->collection() + "': views are read-only");
|
|
@@ -747,6 +796,9 @@ grpc::Status DatabaseGrpcImpl::SetAdd(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("SetAdd", service_);
|
|
return readOnlyStatus("SetAdd", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("SetAdd", store_);
|
|
|
|
|
+ }
|
|
|
bool added = store_.setAdd(request->collection(), request->set_id(), request->member());
|
|
bool added = store_.setAdd(request->collection(), request->set_id(), request->member());
|
|
|
response->set_added(added);
|
|
response->set_added(added);
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
@@ -760,6 +812,9 @@ grpc::Status DatabaseGrpcImpl::SetRemove(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("SetRemove", service_);
|
|
return readOnlyStatus("SetRemove", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("SetRemove", store_);
|
|
|
|
|
+ }
|
|
|
bool removed = store_.setRemove(request->collection(), request->set_id(), request->member());
|
|
bool removed = store_.setRemove(request->collection(), request->set_id(), request->member());
|
|
|
response->set_removed(removed);
|
|
response->set_removed(removed);
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
@@ -797,6 +852,9 @@ grpc::Status DatabaseGrpcImpl::CreateCollection(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("CreateCollection", service_);
|
|
return readOnlyStatus("CreateCollection", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("CreateCollection", store_);
|
|
|
|
|
+ }
|
|
|
CollectionOptions opts;
|
|
CollectionOptions opts;
|
|
|
// Proto3 defaults bools to false, but we want autoCreateId to default to true
|
|
// Proto3 defaults bools to false, but we want autoCreateId to default to true
|
|
|
// Since proto3 can't distinguish "not set" from "false", we always default to true
|
|
// Since proto3 can't distinguish "not set" from "false", we always default to true
|
|
@@ -828,6 +886,9 @@ grpc::Status DatabaseGrpcImpl::DropCollection(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("DropCollection", service_);
|
|
return readOnlyStatus("DropCollection", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("DropCollection", store_);
|
|
|
|
|
+ }
|
|
|
bool dropped = store_.dropCollection(request->name());
|
|
bool dropped = store_.dropCollection(request->name());
|
|
|
response->set_dropped(dropped);
|
|
response->set_dropped(dropped);
|
|
|
|
|
|
|
@@ -884,6 +945,9 @@ grpc::Status DatabaseGrpcImpl::UploadFile(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("UploadFile", service_);
|
|
return readOnlyStatus("UploadFile", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("UploadFile", store_);
|
|
|
|
|
+ }
|
|
|
pb::FileChunk chunk;
|
|
pb::FileChunk chunk;
|
|
|
std::vector<uint8_t> fileData;
|
|
std::vector<uint8_t> fileData;
|
|
|
pb::FileMetadata metadata;
|
|
pb::FileMetadata metadata;
|
|
@@ -980,6 +1044,9 @@ grpc::Status DatabaseGrpcImpl::DeleteFile(
|
|
|
if (service_.isReadOnly()) {
|
|
if (service_.isReadOnly()) {
|
|
|
return readOnlyStatus("DeleteFile", service_);
|
|
return readOnlyStatus("DeleteFile", service_);
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ return memoryEmergencyStatus("DeleteFile", store_);
|
|
|
|
|
+ }
|
|
|
bool deleted = files_.deleteFile(request->id());
|
|
bool deleted = files_.deleteFile(request->id());
|
|
|
response->set_deleted(deleted);
|
|
response->set_deleted(deleted);
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
@@ -1270,6 +1337,12 @@ grpc::Status DatabaseGrpcImpl::CreateView(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
try {
|
|
try {
|
|
|
ViewInfo v;
|
|
ViewInfo v;
|
|
|
v.name = request->name();
|
|
v.name = request->name();
|
|
@@ -1320,6 +1393,12 @@ grpc::Status DatabaseGrpcImpl::DropView(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
try {
|
|
try {
|
|
|
std::string err;
|
|
std::string err;
|
|
|
bool ok = view_manager_.dropView(request->name(), err);
|
|
bool ok = view_manager_.dropView(request->name(), err);
|
|
@@ -1524,6 +1603,12 @@ grpc::Status DatabaseGrpcImpl::ConfigureCollection(
|
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
response->set_error("database is in read-only mode: " + service_.readOnlyReason());
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
try {
|
|
try {
|
|
|
CollectionCfg cfg;
|
|
CollectionCfg cfg;
|
|
|
cfg.timestampPrecision = request->config().timestamp_precision();
|
|
cfg.timestampPrecision = request->config().timestamp_precision();
|
|
@@ -1567,6 +1652,14 @@ grpc::Status DatabaseGrpcImpl::MigrateCollectionTimestamps(
|
|
|
response->set_rows_skipped(0);
|
|
response->set_rows_skipped(0);
|
|
|
return grpc::Status::OK;
|
|
return grpc::Status::OK;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (store_.pressure() == MemoryPressure::Emergency) {
|
|
|
|
|
+ response->set_success(false);
|
|
|
|
|
+ response->set_error("memory pressure emergency (" +
|
|
|
|
|
+ std::to_string(store_.pressurePercent()) + "%); retry after backoff");
|
|
|
|
|
+ response->set_rows_migrated(0);
|
|
|
|
|
+ response->set_rows_skipped(0);
|
|
|
|
|
+ return grpc::Status::OK;
|
|
|
|
|
+ }
|
|
|
try {
|
|
try {
|
|
|
const std::string& collection = request->collection();
|
|
const std::string& collection = request->collection();
|
|
|
const std::string& from = request->from_precision();
|
|
const std::string& from = request->from_precision();
|