Implemented per-collection TTL property [CBL-9299]

C/Cpp_include/c4Collection.hh

@@ -90,6 +90,12 @@ struct C4Collection
     virtual C4Timestamp nextDocExpiration() const = 0;
     virtual int64_t     purgeExpiredDocs()        = 0;
 
+    /// Sets the default time-to-live, in milliseconds, or disables it if the value is 0.
+    /// Every document subsequently saved or deleted will have its expiration set accordingly.
+    virtual void setTTL(C4TimeInterval ms) = 0;
+    /// Gets the time-to-live value, in milliseconds, or 0 if there is none.
+    virtual C4TimeInterval getTTL() const = 0;
+
     // Queries & Indexes:
 
     /// Same as the C4Database method, but the query will refer to this collection by default.

C/c4CAPI.cc

@@ -345,6 +345,19 @@ int64_t c4coll_purgeExpiredDocs(C4Collection* coll, C4Error* C4NULLABLE outError
     return tryCatch<int64_t>(outError, [=] { return coll->purgeExpiredDocs(); });
 }
 
+C4TimeInterval c4coll_getTTL(C4Collection* coll) C4API {
+    return tryCatch<C4TimeInterval>(
+            nullptr, [=] { return _usuallyTrue(c4coll_isValid(coll)) ? coll->getTTL() : C4TimeInterval::None; });
+}
+
+bool c4coll_setTTL(C4Collection* coll, C4TimeInterval ttl, C4Error* C4NULLABLE outError) C4API {
+    returnIfCollectionInvalid(coll, outError, false);
+    return tryCatch<bool>(outError, [=] {
+        coll->setTTL(ttl);
+        return true;
+    });
+}
+
 bool c4coll_createIndex(C4Collection* coll, C4String name, C4String indexSpec, C4QueryLanguage queryLanguage,
                         C4IndexType indexType, const C4IndexOptions* C4NULLABLE indexOptions,
                         C4Error* C4NULLABLE outError) noexcept {

C/include/c4Base.h

@@ -84,6 +84,11 @@ DEFINE_ENUM_ADD_SUB_INT(C4SequenceNumber)
 enum class C4Timestamp : int64_t { None = 0, Error = -1 };
 DEFINE_ENUM_ADD_SUB_INT(C4Timestamp)
 
+/** A time interval, in units of milliseconds. */
+enum class C4TimeInterval : int64_t { None = 0 };
+
+inline C4Timestamp operator+(C4Timestamp t, C4TimeInterval i) { return t + int64_t(i); }
+
 C4API_BEGIN_DECLS
 #else
 /** A database sequence number, representing the order in which a revision was created. */
@@ -93,6 +98,9 @@ typedef uint64_t C4SequenceNumber;
         Measured in milliseconds since the Unix epoch (1/1/1970, midnight UTC.)
         A value of 0 represents "no expiration". */
 typedef int64_t C4Timestamp;
+
+/** A time interval, in units of milliseconds. */
+typedef int64_t C4TimeInterval;
 #endif
 
 

C/include/c4Collection.h

@@ -280,6 +280,20 @@ CBL_CORE_API C4Timestamp c4coll_nextDocExpiration(C4Collection*) C4API;
     \note The caller must use a lock for Database when this function is called.*/
 NODISCARD CBL_CORE_API int64_t c4coll_purgeExpiredDocs(C4Collection*, C4Error* C4NULLABLE) C4API;
 
+/** Returns the collection's current time-to-live; 0 means no TTL is set.
+    \note The caller must use a lock for Database when this function is called. */
+NODISCARD CBL_CORE_API C4TimeInterval c4coll_getTTL(C4Collection* collection) C4API;
+
+/** Sets the collection's time-to-live.
+    When a TTL is configured, every document saved or deleted gets an expiration time based on the TTL.
+    \note The caller must use a lock for Database when this function is called.
+    @param collection  The collection
+    @param ttl  The time-to-live in _milliseconds_; 0 for none
+    @param outError Information about any error that occurred
+    @returns True on success, false on error. */
+NODISCARD CBL_CORE_API bool c4coll_setTTL(C4Collection* collection, C4TimeInterval ttl,
+                                          C4Error* C4NULLABLE outError) C4API;
+
 
 /** @} */
 /** @} */  // end Collections group

C/scripts/c4.txt

@@ -325,6 +325,8 @@ c4coll_purgeExpiredDocs
 c4coll_createIndex
 c4coll_deleteIndex
 c4coll_getIndexesInfo
+c4coll_getTTL
+c4coll_setTTL
 
 c4db_copyNamed
 c4db_deleteNamed

C/tests/c4DatabaseTest.cc

@@ -525,14 +525,21 @@ N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Database Changes", "[Database][Enumerato
 static constexpr int secs = 1000;
 static constexpr int ms   = 1;
 
-static bool docExists(C4Database* db, slice docID) {
-    C4Error err;
-    auto    defaultColl = c4db_getDefaultCollection(db, nullptr);
-    auto    doc         = c4::make_ref(c4coll_getDoc(defaultColl, docID, true, kDocGetCurrentRev, &err));
+static bool docExists(C4Collection* coll, slice docID) {
+    C4Error err{};
+    auto    doc = c4::make_ref(c4coll_getDoc(coll, docID, true, kDocGetCurrentRev, &err));
     if ( doc ) return true;
     CHECK(err == C4Error{LiteCoreDomain, kC4ErrorNotFound});
     return false;
-};
+}
+
+static bool docExists(C4Database* db, slice docID) { return docExists(c4db_getDefaultCollection(db, nullptr), docID); };
+
+static void waitForExpiration(C4Collection* coll, slice docID, C4TimeInterval ttl) {
+    C4Log("---- Waiting %lldms for '%.*s' to expire...", (long long)ttl, FMTSLICE(docID));
+    this_thread::sleep_for(int64_t(ttl) * 1ms);
+    CHECK_BEFORE(15s, !docExists(coll, docID));
+}
 
 N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Database Expired", "[Database][C][Expiration]") {
     C4Error err;
@@ -847,6 +854,65 @@ N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Expire documents while deleting collecti
     }
 }
 
+N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Collection TTL", "[Database][C][Expiration]") {
+    C4CollectionSpec darthSpec{"vader"_sl, "darth"_sl};
+    auto             darthColl = REQUIRED(c4db_createCollection(db, darthSpec, nullptr));
+
+    // Check default TTL, and create a non-expiring doc:
+    CHECK(c4coll_getTTL(darthColl) == 0);
+    createRev(darthColl, "keep_me"_sl, kRevID, kFleeceBody);
+    CHECK(c4coll_getDocExpiration(darthColl, "keep_me"_sl, WITH_ERROR()) == 0);
+
+    // Set TTL to 2 sec and check that it persists:
+    REQUIRE(c4coll_setTTL(darthColl, 2000, WITH_ERROR()));
+    CHECK(c4coll_getTTL(darthColl) == 2000);
+    reopenDB();
+    darthColl = REQUIRED(c4db_createCollection(db, darthSpec, nullptr));
+    REQUIRE(c4coll_getTTL(darthColl) == 2000);
+
+    // Create a doc, and verify its expiration is set:
+    auto before = c4_now();
+    createRev(darthColl, "expire_me"_sl, kRevID, kFleeceBody);
+    auto after = c4_now();
+    auto exp   = c4coll_getDocExpiration(darthColl, "expire_me"_sl, ERROR_INFO());
+    CHECK(exp >= before + 2000);
+    CHECK(exp <= after + 2000);
+    REQUIRE(c4coll_nextDocExpiration(darthColl) == exp);
+
+    // Create another doc, delete it, and verify the expiration is still correct:
+    {
+        TransactionHelper t(db);
+        before = c4_now();
+        createRev(darthColl, "expire_me_deleted"_sl, kRevID, kFleeceBody);
+        createNewRev(darthColl, "expire_me_deleted"_sl, kRevID, kFleeceBody, kRevDeleted);
+        after = c4_now();
+        exp   = c4coll_getDocExpiration(darthColl, "expire_me_deleted"_sl, ERROR_INFO());
+        CHECK(exp >= before + 2000);
+        CHECK(exp <= after + 2000);
+    }
+
+    // Wait for the expiration time to pass and verify the docs are deleted:
+    waitForExpiration(darthColl, "expire_me", 2000);
+    waitForExpiration(darthColl, "expire_me_deleted", 10);
+    CHECK(docExists(darthColl, "keep_me"));
+}
+
+N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Collection TTL in txn", "[Database][C][Expiration]") {
+    // Verify that the Housekeeper correctly determines when it should expire docs, even if it's
+    // initialized during a transaction in which a doc with a TTL is saved. (The CollectionImpl
+    // has to assist it with this: the Housekeeper on its own wouldn't see the new expiration
+    // in its 'backgroundDB' because the transaction in the main DB hasn't committed yet.)
+    C4CollectionSpec darthSpec{"vader"_sl, "darth"_sl};
+    auto             darthColl = REQUIRED(c4db_createCollection(db, darthSpec, nullptr));
+    REQUIRE(c4coll_setTTL(darthColl, 2000, WITH_ERROR()));
+
+    {
+        TransactionHelper t(db);
+        createRev(darthColl, "expire_me"_sl, kRevID, kFleeceBody);
+    }
+    waitForExpiration(darthColl, "expire_me", 2000);
+}
+
 N_WAY_TEST_CASE_METHOD(C4DatabaseTest, "Database BlobStore", "[Database][C][Blob]") {
     C4BlobStore* blobs = c4db_getBlobStore(db, ERROR_INFO());
     REQUIRE(blobs != nullptr);

LiteCore/Database/CollectionImpl.hh

@@ -109,10 +109,6 @@ namespace litecore {
             if ( _sequenceTracker ) _sequenceTracker->useLocked()->beginTransaction();
         }
 
-        bool changedDuringTransaction() {
-            return _sequenceTracker && _sequenceTracker->useLocked()->changedDuringTransaction();
-        }
-
         void transactionEnding(ExclusiveTransaction* transaction, bool committing) {
             if ( _sequenceTracker ) {
                 auto st = _sequenceTracker->useLocked();
@@ -314,26 +310,30 @@ namespace litecore {
         }
 
         void moveDocument(slice docID, C4Collection* toCollection, slice newDocID) override {
+            auto                    dst = (CollectionImpl*)toCollection;
             C4Database::Transaction t(getDatabase());
             if ( newDocID ) C4Document::requireValidDocID(newDocID);
-            keyStore().moveTo(docID, ((CollectionImpl*)toCollection)->keyStore(), dbImpl()->transaction(), newDocID);
+            keyStore().moveTo(docID, dst->keyStore(), dbImpl()->transaction(), newDocID);
             // DOES NOT NOTIFY SEQUENCE TRACKER! (should it?)
             t.commit();
+            if ( auto ttl = dst->getTTL(); ttl != C4TimeInterval::None ) dst->documentExpirationChanged(c4_now() + ttl);
         }
 
-        void documentSaved(C4Document* doc) {
+        void documentSaved(C4Document* doc, bool newSequence) {
             // CBL-1089
             // Conflicted documents are not eligible to be replicated,
             // so ignore them.  Later when the conflict is resolved
             // there will be logic to replicate them (see TreeDocument::resolveConflict)
-            if ( _sequenceTracker && !(doc->selectedRev().flags & kRevIsConflict) ) {
+            if ( newSequence && _sequenceTracker && !(doc->selectedRev().flags & kRevIsConflict) ) {
                 Assert(doc->selectedRev().sequence == doc->sequence());  // The new revision must be selected
                 auto st = _sequenceTracker->useLocked();
                 st->documentChanged(doc->docID(),
                                     doc->getSelectedRevIDGlobalForm(),  // entire version vector
                                     doc->selectedRev().sequence, doc->getRevisionBody().size,
                                     SequenceTracker::RevisionFlags(doc->selectedRev().flags));
             }
+
+            if ( auto ttl = getTTL(); ttl != C4TimeInterval::None ) documentExpirationChanged(c4_now() + ttl);
         }
 
 #pragma mark - PURGING / EXPIRING DOCS:
@@ -346,12 +346,7 @@ namespace litecore {
                 if ( !keyStore().setExpiration(docID, expiration) ) return false;
                 t.commit();
             }
-
-            if ( expiration > C4Timestamp::None ) {
-                if ( _expiryStarted ) _housekeeper->documentExpirationChanged(expiration);
-                else
-                    startHousekeeping(HousekeeperTask::Expiry);
-            }
+            documentExpirationChanged(expiration);
             return true;
         }
 
@@ -378,12 +373,34 @@ namespace litecore {
             return count;
         }
 
+        void setTTL(C4TimeInterval ttl) override {
+            C4Database::Transaction t(dbImpl());
+            auto&                   ks = keyStore();
+            if ( ttl != ks.getTTL() ) {
+                ks.setTTL(ttl);
+                t.commit();
+            }
+        }
+
+        C4TimeInterval getTTL() const override { return keyStore().getTTL(); }
+
+        void documentExpirationChanged(C4Timestamp expiration) {
+            if ( expiration > C4Timestamp::None ) {
+                if ( !_expiryStarted ) startHousekeeping(HousekeeperTask::Expiry);
+                if ( _expiryStarted ) _housekeeper->documentExpirationChanged(expiration);
+            }
+        }
+
         void startHousekeeping(HousekeeperTask task) {
             if ( !isValid() && _expiryStarted && _migrateStarted ) return;
 
-            if ( auto flags = _database->getConfiguration().flags;
-                 (flags & (kC4DB_ReadOnly | kC4DB_NoHousekeeping)) != 0 )
-                return;
+            // Housekeeping only makes sense in writeable dbs. In general `kC4DB_NoHousekeeping` disables it,
+            // but not for expiration; otherwise it would be a mess when a db with no expiry tracker sets a
+            // document expiration -- it would have to find some other instance with a Housekeeper and notify it
+            // (across threads.) [CBL-9299]
+            auto dbFlags = _database->getConfiguration().flags;
+            if ( dbFlags & kC4DB_ReadOnly ) return;
+            if ( (dbFlags & kC4DB_NoHousekeeping) && task != HousekeeperTask::Expiry ) return;
 
             if ( !_housekeeper ) {
                 _housekeeper = new Housekeeper(this);

LiteCore/Database/TreeDocument.cc

@@ -330,13 +330,14 @@ namespace litecore {
                 case litecore::RevTreeRecord::kConflict:
                     return false;
                 case litecore::RevTreeRecord::kNoNewSequence:
+                    asInternal(collection())->documentSaved(this, false);
                     return true;
                 case litecore::RevTreeRecord::kNewSequence:
                     _selected.flags &= ~kRevNew;
                     if ( _revTree.sequence() > _sequence ) {
                         _sequence = _revTree.sequence();
                         if ( _selected.sequence == 0_seq ) _selected.sequence = _sequence;
-                        asInternal(collection())->documentSaved(this);
+                        asInternal(collection())->documentSaved(this, true);
                     }
                     return true;
                 default:

LiteCore/Database/VectorDocument.cc

@@ -700,6 +700,7 @@ namespace litecore {
                     return true;
                 case VectorRecord::kNoNewSequence:
                     _updateDocFields();  // flags may have changed
+                    asInternal(collection())->documentSaved(this, false);
                     return true;
                 case VectorRecord::kConflict:
                     return false;
@@ -713,7 +714,7 @@ namespace litecore {
                                                     ((_flags & kRevDeleted) ? "Deleted" : "Saved"), SPLAT(_docID),
                                                     SPLAT(revID), (uint64_t)_sequence);
                     }
-                    asInternal(collection())->documentSaved(this);
+                    asInternal(collection())->documentSaved(this, true);
                     return true;
             }
             return false;  // unreachable

LiteCore/Storage/BothKeyStore.cc

@@ -61,7 +61,9 @@ namespace litecore {
 
         if ( seq == 0_seq && rec.sequence > 0_seq ) {
             // Conflict. Maybe record is currently in the other KeyStore; if so, delete it & retry
-            expiration_t expiry = other->getExpiration(rec.key);
+            auto expiry = expiration_t::None;
+            if ( target->getTTL() == C4TimeInterval::None )  // don't preserve exp if collection has a TTL
+                expiry = other->getExpiration(rec.key);
             if ( other->del(rec.key, t, rec.sequence, rec.subsequence) ) {
                 // We move a record from one sub-store to the other one by deleting it from
                 // one store and inserting it to the other one while keeping the seqquence.

LiteCore/Storage/BothKeyStore.hh

@@ -83,6 +83,10 @@ namespace litecore {
             return _liveStore->expireRecords(callback) + _deadStore->expireRecords(callback);
         }
 
+        C4TimeInterval getTTL() const override { return _liveStore->getTTL(); }
+
+        void setTTL(C4TimeInterval ms) override { _liveStore->setTTL(ms); }
+
         //// QUERIES & INDEXES:
 
         std::vector<alloc_slice> withDocBodies(const std::vector<slice>& docIDs, WithDocBodyCallback callback) override;

LiteCore/Storage/KeyStore.hh

@@ -180,6 +180,12 @@ namespace litecore {
             @return  The number of records deleted */
         virtual unsigned expireRecords(std::optional<ExpirationCallback> = std::nullopt) = 0;
 
+        /** Sets default time-to-live of docs, in ms */
+        virtual void setTTL(C4TimeInterval ms) = 0;
+
+        /** Gets default time-to-live of docs, in ms */
+        virtual C4TimeInterval getTTL() const = 0;
+
 
         //////// Queries: