From 352b790b6e292c0d921ad0231352d767fde53758 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Thu, 22 Mar 2018 17:31:16 -0700 Subject: Switch FSTLevelDBQueryCache to use transactions (#942) * Start work on leveldb transactions * Style * Working API. Not plumbed in yet * Move files into correct place * Wrangling file locations and associations * Tests pass * Add some comments * style * Fix copyright * Rewrite iterator internals to handle deletion-while-iterating. Also add tests for same * Switch to strings instead of slices * Style * More style fixes * Start switching writegroup over * Swap out write group tracking for transaction usage * Style * Response to feedback before updating docs * Style * Add comment * Initialize version_ * Satisfy the linter * Start switching writegroup over * Swap out write group tracking for transaction usage * Style * Checkpoint before implementing BatchDescription * Style * Initial plumbing for leveldb local parts * Add model::BatchId * Port leveldb_key.{h,cc} * Add string StartsWith * Add leveldb_key_test.cc to the project * Revert back to using leveldb::Slice for read/describe These operations universally operate on keys obtained from leveldb so it's actually unhelpful to force all the callers to make absl::string_views from them. * Everything passing * Drop unused function * Style * STart work on reads * Swap reads in queryCache to use transactions * Fix up tests of querycache * Drop commented out code * Cleanup * Style * Fix up for passing tests * style * Renaming * Style * Start work on ToString for transactions * Add ToString() method to LevelDbTransaction * Style * lint * Fix includes, drop runTransaction * current_transaction -> currentTransaction * LevelDbTransaction::NewIterator now returns a unique_ptr * Style * Revert addition of util::StartsWith * Add log line * Style * Add log line * Style * Add debug log line for commits, drop unused BatchDescription * Fix an include, but mostly try to trigger a travis build --- .../Tests/Local/FSTLevelDBMigrationsTests.mm | 2 +- .../Tests/Local/FSTLevelDBTransactionTests.mm | 8 +- .../Example/Tests/Local/FSTQueryCacheTests.mm | 164 +++++++++++---------- Firestore/Source/Local/FSTLevelDB.h | 3 + Firestore/Source/Local/FSTLevelDB.mm | 7 +- Firestore/Source/Local/FSTLevelDBMigrations.mm | 2 +- Firestore/Source/Local/FSTLevelDBQueryCache.h | 3 +- Firestore/Source/Local/FSTLevelDBQueryCache.mm | 49 +++--- Firestore/Source/Local/FSTLocalStore.mm | 19 ++- .../firestore/local/leveldb_transaction.cc | 5 +- .../firebase/firestore/local/leveldb_transaction.h | 2 +- 11 files changed, 137 insertions(+), 127 deletions(-) (limited to 'Firestore') diff --git a/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm b/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm index 822e3ca..4f0332e 100644 --- a/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm +++ b/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm @@ -15,13 +15,13 @@ */ #import -#include #import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" #import "Firestore/Source/Local/FSTLevelDB.h" #import "Firestore/Source/Local/FSTLevelDBKey.h" #import "Firestore/Source/Local/FSTLevelDBMigrations.h" #import "Firestore/Source/Local/FSTLevelDBQueryCache.h" +#include "leveldb/db.h" #include "Firestore/core/src/firebase/firestore/util/ordered_code.h" diff --git a/Firestore/Example/Tests/Local/FSTLevelDBTransactionTests.mm b/Firestore/Example/Tests/Local/FSTLevelDBTransactionTests.mm index 2a1efb0..c959b4f 100644 --- a/Firestore/Example/Tests/Local/FSTLevelDBTransactionTests.mm +++ b/Firestore/Example/Tests/Local/FSTLevelDBTransactionTests.mm @@ -62,7 +62,7 @@ using firebase::firestore::local::LevelDbTransaction; std::string key = "key1"; transaction.Put(key, "value"); - std::unique_ptr iter(transaction.NewIterator()); + auto iter = transaction.NewIterator(); iter->Seek(key); XCTAssertEqual(key, iter->key()); iter->Next(); @@ -212,7 +212,7 @@ using firebase::firestore::local::LevelDbTransaction; transaction.Put("key_" + std::to_string(i), "value_" + std::to_string(i)); } - std::unique_ptr it(transaction.NewIterator()); + auto it = transaction.NewIterator(); it->Seek("key_0"); for (int i = 0; i < 4; ++i) { XCTAssertTrue(it->Valid()); @@ -234,7 +234,7 @@ using firebase::firestore::local::LevelDbTransaction; // Create a transaction, iterate, deleting key_0. Verify we still iterate key_1. LevelDbTransaction transaction(_db.get()); - std::unique_ptr it(transaction.NewIterator()); + auto it = transaction.NewIterator(); it->Seek("key_0"); XCTAssertTrue(it->Valid()); XCTAssertEqual("key_0", it->key()); @@ -256,7 +256,7 @@ using firebase::firestore::local::LevelDbTransaction; // Create a transaction, iterate to key_1, delete key_2. Verify we still iterate key_3. LevelDbTransaction transaction(_db.get()); - std::unique_ptr it(transaction.NewIterator()); + auto it = transaction.NewIterator(); it->Seek("key_0"); XCTAssertTrue(it->Valid()); XCTAssertEqual("key_0", it->key()); diff --git a/Firestore/Example/Tests/Local/FSTQueryCacheTests.mm b/Firestore/Example/Tests/Local/FSTQueryCacheTests.mm index 0618e9c..6ef927d 100644 --- a/Firestore/Example/Tests/Local/FSTQueryCacheTests.mm +++ b/Firestore/Example/Tests/Local/FSTQueryCacheTests.mm @@ -56,24 +56,29 @@ NS_ASSUME_NONNULL_BEGIN - (void)testReadQueryNotInCache { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"ReadQueryNotInCache"]; XCTAssertNil([self.queryCache queryDataForQuery:_queryRooms]); + [self.persistence commitGroup:group]; } - (void)testSetAndReadAQuery { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"SetAndReadQuery"]; FSTQueryData *queryData = [self queryDataWithQuery:_queryRooms]; - [self addQueryData:queryData]; + [self.queryCache addQueryData:queryData group:group]; FSTQueryData *result = [self.queryCache queryDataForQuery:_queryRooms]; XCTAssertEqualObjects(result.query, queryData.query); XCTAssertEqual(result.targetID, queryData.targetID); XCTAssertEqualObjects(result.resumeToken, queryData.resumeToken); + [self.persistence commitGroup:group]; } - (void)testCanonicalIDCollision { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"CanonicalIDCollision"]; // Type information is currently lost in our canonicalID implementations so this currently an // easy way to force colliding canonicalIDs FSTQuery *q1 = [FSTTestQuery("a") queryByAddingFilter:FSTTestFilter("foo", @"==", @(1))]; @@ -81,136 +86,151 @@ NS_ASSUME_NONNULL_BEGIN XCTAssertEqualObjects(q1.canonicalID, q2.canonicalID); FSTQueryData *data1 = [self queryDataWithQuery:q1]; - [self addQueryData:data1]; + [self.queryCache addQueryData:data1 group:group]; // Using the other query should not return the query cache entry despite equal canonicalIDs. XCTAssertNil([self.queryCache queryDataForQuery:q2]); XCTAssertEqualObjects([self.queryCache queryDataForQuery:q1], data1); FSTQueryData *data2 = [self queryDataWithQuery:q2]; - [self addQueryData:data2]; + [self.queryCache addQueryData:data2 group:group]; XCTAssertEqual([self.queryCache count], 2); XCTAssertEqualObjects([self.queryCache queryDataForQuery:q1], data1); XCTAssertEqualObjects([self.queryCache queryDataForQuery:q2], data2); - [self removeQueryData:data1]; + [self.queryCache removeQueryData:data1 group:group]; XCTAssertNil([self.queryCache queryDataForQuery:q1]); XCTAssertEqualObjects([self.queryCache queryDataForQuery:q2], data2); XCTAssertEqual([self.queryCache count], 1); - [self removeQueryData:data2]; + [self.queryCache removeQueryData:data2 group:group]; XCTAssertNil([self.queryCache queryDataForQuery:q1]); XCTAssertNil([self.queryCache queryDataForQuery:q2]); XCTAssertEqual([self.queryCache count], 0); + [self.persistence commitGroup:group]; } - (void)testSetQueryToNewValue { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"SetQueryToNewValue"]; FSTQueryData *queryData1 = [self queryDataWithQuery:_queryRooms targetID:1 listenSequenceNumber:10 version:1]; - [self addQueryData:queryData1]; + [self.queryCache addQueryData:queryData1 group:group]; FSTQueryData *queryData2 = [self queryDataWithQuery:_queryRooms targetID:1 listenSequenceNumber:10 version:2]; - [self addQueryData:queryData2]; + [self.queryCache addQueryData:queryData2 group:group]; FSTQueryData *result = [self.queryCache queryDataForQuery:_queryRooms]; XCTAssertNotEqualObjects(queryData2.resumeToken, queryData1.resumeToken); XCTAssertNotEqualObjects(queryData2.snapshotVersion, queryData1.snapshotVersion); XCTAssertEqualObjects(result.resumeToken, queryData2.resumeToken); XCTAssertEqualObjects(result.snapshotVersion, queryData2.snapshotVersion); + [self.persistence commitGroup:group]; } - (void)testRemoveQuery { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoveQuery"]; FSTQueryData *queryData1 = [self queryDataWithQuery:_queryRooms]; - [self addQueryData:queryData1]; + [self.queryCache addQueryData:queryData1 group:group]; - [self removeQueryData:queryData1]; + [self.queryCache removeQueryData:queryData1 group:group]; FSTQueryData *result = [self.queryCache queryDataForQuery:_queryRooms]; XCTAssertNil(result); + [self.persistence commitGroup:group]; } - (void)testRemoveNonExistentQuery { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoveNonExistentQuery"]; FSTQueryData *queryData = [self queryDataWithQuery:_queryRooms]; // no-op, but make sure it doesn't throw. - XCTAssertNoThrow([self removeQueryData:queryData]); + XCTAssertNoThrow([self.queryCache removeQueryData:queryData group:group]); + [self.persistence commitGroup:group]; } - (void)testRemoveQueryRemovesMatchingKeysToo { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = + [self.persistence startGroupWithAction:@"RemoveQueryRemovesMatchingKeysToo"]; FSTQueryData *rooms = [self queryDataWithQuery:_queryRooms]; - [self addQueryData:rooms]; + [self.queryCache addQueryData:rooms group:group]; FSTDocumentKey *key1 = FSTTestDocKey(@"rooms/foo"); FSTDocumentKey *key2 = FSTTestDocKey(@"rooms/bar"); - [self addMatchingKey:key1 forTargetID:rooms.targetID]; - [self addMatchingKey:key2 forTargetID:rooms.targetID]; + [self addMatchingKey:key1 forTargetID:rooms.targetID group:group]; + [self addMatchingKey:key2 forTargetID:rooms.targetID group:group]; XCTAssertTrue([self.queryCache containsKey:key1]); XCTAssertTrue([self.queryCache containsKey:key2]); - [self removeQueryData:rooms]; + [self.queryCache removeQueryData:rooms group:group]; XCTAssertFalse([self.queryCache containsKey:key1]); XCTAssertFalse([self.queryCache containsKey:key2]); + [self.persistence commitGroup:group]; } - (void)testAddOrRemoveMatchingKeys { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"AddOrRemoveMatchingKeys"]; FSTDocumentKey *key = FSTTestDocKey(@"foo/bar"); XCTAssertFalse([self.queryCache containsKey:key]); - [self addMatchingKey:key forTargetID:1]; + [self addMatchingKey:key forTargetID:1 group:group]; XCTAssertTrue([self.queryCache containsKey:key]); - [self addMatchingKey:key forTargetID:2]; + [self addMatchingKey:key forTargetID:2 group:group]; XCTAssertTrue([self.queryCache containsKey:key]); - [self removeMatchingKey:key forTargetID:1]; + [self removeMatchingKey:key forTargetID:1 group:group]; XCTAssertTrue([self.queryCache containsKey:key]); - [self removeMatchingKey:key forTargetID:2]; + [self removeMatchingKey:key forTargetID:2 group:group]; XCTAssertFalse([self.queryCache containsKey:key]); + [self.persistence commitGroup:group]; } - (void)testRemoveMatchingKeysForTargetID { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoveMatchingKeysForTargetID"]; FSTDocumentKey *key1 = FSTTestDocKey(@"foo/bar"); FSTDocumentKey *key2 = FSTTestDocKey(@"foo/baz"); FSTDocumentKey *key3 = FSTTestDocKey(@"foo/blah"); - [self addMatchingKey:key1 forTargetID:1]; - [self addMatchingKey:key2 forTargetID:1]; - [self addMatchingKey:key3 forTargetID:2]; + [self addMatchingKey:key1 forTargetID:1 group:group]; + [self addMatchingKey:key2 forTargetID:1 group:group]; + [self addMatchingKey:key3 forTargetID:2 group:group]; XCTAssertTrue([self.queryCache containsKey:key1]); XCTAssertTrue([self.queryCache containsKey:key2]); XCTAssertTrue([self.queryCache containsKey:key3]); - [self removeMatchingKeysForTargetID:1]; + [self.queryCache removeMatchingKeysForTargetID:1 group:group]; XCTAssertFalse([self.queryCache containsKey:key1]); XCTAssertFalse([self.queryCache containsKey:key2]); XCTAssertTrue([self.queryCache containsKey:key3]); - [self removeMatchingKeysForTargetID:2]; + [self.queryCache removeMatchingKeysForTargetID:2 group:group]; XCTAssertFalse([self.queryCache containsKey:key1]); XCTAssertFalse([self.queryCache containsKey:key2]); XCTAssertFalse([self.queryCache containsKey:key3]); + [self.persistence commitGroup:group]; } - (void)testRemoveEmitsGarbageEvents { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoveEmitsGarbageEvents"]; FSTEagerGarbageCollector *garbageCollector = [[FSTEagerGarbageCollector alloc] init]; [garbageCollector addGarbageSource:self.queryCache]; FSTAssertEqualSets([garbageCollector collectGarbage], @[]); @@ -218,65 +238,69 @@ NS_ASSUME_NONNULL_BEGIN FSTQueryData *rooms = [self queryDataWithQuery:FSTTestQuery("rooms")]; FSTDocumentKey *room1 = FSTTestDocKey(@"rooms/bar"); FSTDocumentKey *room2 = FSTTestDocKey(@"rooms/foo"); - [self addQueryData:rooms]; - [self addMatchingKey:room1 forTargetID:rooms.targetID]; - [self addMatchingKey:room2 forTargetID:rooms.targetID]; + [self.queryCache addQueryData:rooms group:group]; + [self addMatchingKey:room1 forTargetID:rooms.targetID group:group]; + [self addMatchingKey:room2 forTargetID:rooms.targetID group:group]; FSTQueryData *halls = [self queryDataWithQuery:FSTTestQuery("halls")]; FSTDocumentKey *hall1 = FSTTestDocKey(@"halls/bar"); FSTDocumentKey *hall2 = FSTTestDocKey(@"halls/foo"); - [self addQueryData:halls]; - [self addMatchingKey:hall1 forTargetID:halls.targetID]; - [self addMatchingKey:hall2 forTargetID:halls.targetID]; + [self.queryCache addQueryData:halls group:group]; + [self addMatchingKey:hall1 forTargetID:halls.targetID group:group]; + [self addMatchingKey:hall2 forTargetID:halls.targetID group:group]; FSTAssertEqualSets([garbageCollector collectGarbage], @[]); - [self removeMatchingKey:room1 forTargetID:rooms.targetID]; + [self removeMatchingKey:room1 forTargetID:rooms.targetID group:group]; FSTAssertEqualSets([garbageCollector collectGarbage], @[ room1 ]); - [self removeQueryData:rooms]; + [self.queryCache removeQueryData:rooms group:group]; FSTAssertEqualSets([garbageCollector collectGarbage], @[ room2 ]); - [self removeMatchingKeysForTargetID:halls.targetID]; + [self.queryCache removeMatchingKeysForTargetID:halls.targetID group:group]; FSTAssertEqualSets([garbageCollector collectGarbage], (@[ hall1, hall2 ])); + [self.persistence commitGroup:group]; } - (void)testMatchingKeysForTargetID { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"MatchingKeysForTargetID"]; FSTDocumentKey *key1 = FSTTestDocKey(@"foo/bar"); FSTDocumentKey *key2 = FSTTestDocKey(@"foo/baz"); FSTDocumentKey *key3 = FSTTestDocKey(@"foo/blah"); - [self addMatchingKey:key1 forTargetID:1]; - [self addMatchingKey:key2 forTargetID:1]; - [self addMatchingKey:key3 forTargetID:2]; + [self addMatchingKey:key1 forTargetID:1 group:group]; + [self addMatchingKey:key2 forTargetID:1 group:group]; + [self addMatchingKey:key3 forTargetID:2 group:group]; FSTAssertEqualSets([self.queryCache matchingKeysForTargetID:1], (@[ key1, key2 ])); FSTAssertEqualSets([self.queryCache matchingKeysForTargetID:2], @[ key3 ]); - [self addMatchingKey:key1 forTargetID:2]; + [self addMatchingKey:key1 forTargetID:2 group:group]; FSTAssertEqualSets([self.queryCache matchingKeysForTargetID:1], (@[ key1, key2 ])); FSTAssertEqualSets([self.queryCache matchingKeysForTargetID:2], (@[ key1, key3 ])); + [self.persistence commitGroup:group]; } - (void)testHighestListenSequenceNumber { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"HighestListenSequenceNumber"]; FSTQueryData *query1 = [[FSTQueryData alloc] initWithQuery:FSTTestQuery("rooms") targetID:1 listenSequenceNumber:10 purpose:FSTQueryPurposeListen]; - [self addQueryData:query1]; + [self.queryCache addQueryData:query1 group:group]; FSTQueryData *query2 = [[FSTQueryData alloc] initWithQuery:FSTTestQuery("halls") targetID:2 listenSequenceNumber:20 purpose:FSTQueryPurposeListen]; - [self addQueryData:query2]; + [self.queryCache addQueryData:query2 group:group]; XCTAssertEqual([self.queryCache highestListenSequenceNumber], 20); // TargetIDs never come down. - [self removeQueryData:query2]; + [self.queryCache removeQueryData:query2 group:group]; XCTAssertEqual([self.queryCache highestListenSequenceNumber], 20); // A query with an empty result set still counts. @@ -284,14 +308,15 @@ NS_ASSUME_NONNULL_BEGIN targetID:42 listenSequenceNumber:100 purpose:FSTQueryPurposeListen]; - [self addQueryData:query3]; + [self.queryCache addQueryData:query3 group:group]; XCTAssertEqual([self.queryCache highestListenSequenceNumber], 100); - [self removeQueryData:query1]; + [self.queryCache removeQueryData:query1 group:group]; XCTAssertEqual([self.queryCache highestListenSequenceNumber], 100); - [self removeQueryData:query3]; + [self.queryCache removeQueryData:query3 group:group]; XCTAssertEqual([self.queryCache highestListenSequenceNumber], 100); + [self.persistence commitGroup:group]; // Verify that the highestTargetID even survives restarts. [self.queryCache shutdown]; @@ -303,6 +328,7 @@ NS_ASSUME_NONNULL_BEGIN - (void)testHighestTargetID { if ([self isTestBaseClass]) return; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoveMatchingKeysForTargetID"]; XCTAssertEqual([self.queryCache highestTargetID], 0); FSTQueryData *query1 = [[FSTQueryData alloc] initWithQuery:FSTTestQuery("rooms") @@ -311,21 +337,21 @@ NS_ASSUME_NONNULL_BEGIN purpose:FSTQueryPurposeListen]; FSTDocumentKey *key1 = FSTTestDocKey(@"rooms/bar"); FSTDocumentKey *key2 = FSTTestDocKey(@"rooms/foo"); - [self addQueryData:query1]; - [self addMatchingKey:key1 forTargetID:1]; - [self addMatchingKey:key2 forTargetID:1]; + [self.queryCache addQueryData:query1 group:group]; + [self addMatchingKey:key1 forTargetID:1 group:group]; + [self addMatchingKey:key2 forTargetID:1 group:group]; FSTQueryData *query2 = [[FSTQueryData alloc] initWithQuery:FSTTestQuery("halls") targetID:2 listenSequenceNumber:20 purpose:FSTQueryPurposeListen]; FSTDocumentKey *key3 = FSTTestDocKey(@"halls/foo"); - [self addQueryData:query2]; - [self addMatchingKey:key3 forTargetID:2]; + [self.queryCache addQueryData:query2 group:group]; + [self addMatchingKey:key3 forTargetID:2 group:group]; XCTAssertEqual([self.queryCache highestTargetID], 2); // TargetIDs never come down. - [self removeQueryData:query2]; + [self.queryCache removeQueryData:query2 group:group]; XCTAssertEqual([self.queryCache highestTargetID], 2); // A query with an empty result set still counts. @@ -333,15 +359,15 @@ NS_ASSUME_NONNULL_BEGIN targetID:42 listenSequenceNumber:100 purpose:FSTQueryPurposeListen]; - [self addQueryData:query3]; + [self.queryCache addQueryData:query3 group:group]; XCTAssertEqual([self.queryCache highestTargetID], 42); - [self removeQueryData:query1]; + [self.queryCache removeQueryData:query1 group:group]; XCTAssertEqual([self.queryCache highestTargetID], 42); - [self removeQueryData:query3]; + [self.queryCache removeQueryData:query3 group:group]; XCTAssertEqual([self.queryCache highestTargetID], 42); - + [self.persistence commitGroup:group]; // Verify that the highestTargetID even survives restarts. [self.queryCache shutdown]; self.queryCache = [self.persistence queryCache]; @@ -393,42 +419,20 @@ NS_ASSUME_NONNULL_BEGIN resumeToken:resumeToken]; } -/** Adds the given query data to the queryCache under test, committing immediately. */ -- (void)addQueryData:(FSTQueryData *)queryData { - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"addQueryData"]; - [self.queryCache addQueryData:queryData group:group]; - [self.persistence commitGroup:group]; -} - -/** Removes the given query data from the queryCache under test, committing immediately. */ -- (void)removeQueryData:(FSTQueryData *)queryData { - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"removeQueryData"]; - [self.queryCache removeQueryData:queryData group:group]; - [self.persistence commitGroup:group]; -} - -- (void)addMatchingKey:(FSTDocumentKey *)key forTargetID:(FSTTargetID)targetID { +- (void)addMatchingKey:(FSTDocumentKey *)key + forTargetID:(FSTTargetID)targetID + group:(FSTWriteGroup *)group { FSTDocumentKeySet *keys = [FSTDocumentKeySet keySet]; keys = [keys setByAddingObject:key]; - - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"addMatchingKeys"]; [self.queryCache addMatchingKeys:keys forTargetID:targetID group:group]; - [self.persistence commitGroup:group]; } -- (void)removeMatchingKey:(FSTDocumentKey *)key forTargetID:(FSTTargetID)targetID { +- (void)removeMatchingKey:(FSTDocumentKey *)key + forTargetID:(FSTTargetID)targetID + group:(FSTWriteGroup *)group { FSTDocumentKeySet *keys = [FSTDocumentKeySet keySet]; keys = [keys setByAddingObject:key]; - - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"removeMatchingKeys"]; [self.queryCache removeMatchingKeys:keys forTargetID:targetID group:group]; - [self.persistence commitGroup:group]; -} - -- (void)removeMatchingKeysForTargetID:(FSTTargetID)targetID { - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"removeMatchingKeysForTargetID"]; - [self.queryCache removeMatchingKeysForTargetID:targetID group:group]; - [self.persistence commitGroup:group]; } @end diff --git a/Firestore/Source/Local/FSTLevelDB.h b/Firestore/Source/Local/FSTLevelDB.h index 77abb3d..b03b65f 100644 --- a/Firestore/Source/Local/FSTLevelDB.h +++ b/Firestore/Source/Local/FSTLevelDB.h @@ -20,6 +20,7 @@ #import "Firestore/Source/Local/FSTPersistence.h" #include "Firestore/core/src/firebase/firestore/core/database_info.h" +#include "Firestore/core/src/firebase/firestore/local/leveldb_transaction.h" #include "leveldb/db.h" @class FSTLocalSerializer; @@ -96,6 +97,8 @@ NS_ASSUME_NONNULL_BEGIN /** The native db pointer, allocated during start. */ @property(nonatomic, assign, readonly) std::shared_ptr ptr; +@property(nonatomic, readonly) firebase::firestore::local::LevelDbTransaction *currentTransaction; + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTLevelDB.mm b/Firestore/Source/Local/FSTLevelDB.mm index a7ee99d..6efbff1 100644 --- a/Firestore/Source/Local/FSTLevelDB.mm +++ b/Firestore/Source/Local/FSTLevelDB.mm @@ -203,6 +203,11 @@ using leveldb::WriteOptions; return database; } +- (LevelDbTransaction *)currentTransaction { + FSTAssert(_transaction != nullptr, @"Attempting to access transaction before one has started"); + return _transaction.get(); +} + #pragma mark - Persistence Factory methods - (id)mutationQueueForUser:(const User &)user { @@ -210,7 +215,7 @@ using leveldb::WriteOptions; } - (id)queryCache { - return [[FSTLevelDBQueryCache alloc] initWithDB:_ptr serializer:self.serializer]; + return [[FSTLevelDBQueryCache alloc] initWithDB:self serializer:self.serializer]; } - (id)remoteDocumentCache { diff --git a/Firestore/Source/Local/FSTLevelDBMigrations.mm b/Firestore/Source/Local/FSTLevelDBMigrations.mm index 7f521d1..cf06c9f 100644 --- a/Firestore/Source/Local/FSTLevelDBMigrations.mm +++ b/Firestore/Source/Local/FSTLevelDBMigrations.mm @@ -66,7 +66,7 @@ static void SaveVersion(FSTLevelDBSchemaVersion version, LevelDbTransaction *tra * It assumes the metadata has already been written and is able to be read in this transaction. */ static void AddTargetCount(LevelDbTransaction *transaction) { - std::unique_ptr it(transaction->NewIterator()); + auto it = transaction->NewIterator(); std::string start_key = [FSTLevelDBTargetKey keyPrefix]; it->Seek(start_key); diff --git a/Firestore/Source/Local/FSTLevelDBQueryCache.h b/Firestore/Source/Local/FSTLevelDBQueryCache.h index d955b02..2cd6758 100644 --- a/Firestore/Source/Local/FSTLevelDBQueryCache.h +++ b/Firestore/Source/Local/FSTLevelDBQueryCache.h @@ -22,6 +22,7 @@ #include "Firestore/core/src/firebase/firestore/local/leveldb_transaction.h" #include "leveldb/db.h" +@class FSTLevelDB; @class FSTLocalSerializer; @class FSTPBTargetGlobal; @protocol FSTGarbageCollector; @@ -53,7 +54,7 @@ NS_ASSUME_NONNULL_BEGIN * * @param db The LevelDB in which to create the cache. */ -- (instancetype)initWithDB:(std::shared_ptr)db +- (instancetype)initWithDB:(FSTLevelDB *)db serializer:(FSTLocalSerializer *)serializer NS_DESIGNATED_INITIALIZER; @end diff --git a/Firestore/Source/Local/FSTLevelDBQueryCache.mm b/Firestore/Source/Local/FSTLevelDBQueryCache.mm index 95b032e..c7eecc4 100644 --- a/Firestore/Source/Local/FSTLevelDBQueryCache.mm +++ b/Firestore/Source/Local/FSTLevelDBQueryCache.mm @@ -16,9 +16,6 @@ #import "Firestore/Source/Local/FSTLevelDBQueryCache.h" -#include -#include - #import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" #import "Firestore/Source/Core/FSTQuery.h" #import "Firestore/Source/Local/FSTLevelDB.h" @@ -28,17 +25,15 @@ #import "Firestore/Source/Local/FSTWriteGroup.h" #import "Firestore/Source/Model/FSTDocumentKey.h" #import "Firestore/Source/Util/FSTAssert.h" +#include "absl/strings/match.h" NS_ASSUME_NONNULL_BEGIN using firebase::firestore::local::LevelDbTransaction; using Firestore::StringView; using leveldb::DB; -using leveldb::Iterator; -using leveldb::ReadOptions; using leveldb::Slice; using leveldb::Status; -using leveldb::WriteOptions; @interface FSTLevelDBQueryCache () @@ -50,8 +45,7 @@ using leveldb::WriteOptions; @end @implementation FSTLevelDBQueryCache { - // The DB pointer is shared with all cooperating LevelDB-related objects. - std::shared_ptr _db; + FSTLevelDB *_db; /** * The last received snapshot version. This is part of `metadata` but we store it separately to @@ -107,7 +101,7 @@ using leveldb::WriteOptions; return proto; } -- (instancetype)initWithDB:(std::shared_ptr)db serializer:(FSTLocalSerializer *)serializer { +- (instancetype)initWithDB:(FSTLevelDB *)db serializer:(FSTLocalSerializer *)serializer { if (self = [super init]) { FSTAssert(db, @"db must not be NULL"); _db = db; @@ -117,7 +111,8 @@ using leveldb::WriteOptions; } - (void)start { - FSTPBTargetGlobal *metadata = [FSTLevelDBQueryCache readTargetMetadataFromDB:_db]; + // TODO(gsoltis): switch this usage of ptr to currentTransaction + FSTPBTargetGlobal *metadata = [FSTLevelDBQueryCache readTargetMetadataFromDB:_db.ptr]; FSTAssert( metadata != nil, @"Found nil metadata, expected schema to be at version 0 which ensures metadata existence"); @@ -148,7 +143,6 @@ using leveldb::WriteOptions; } - (void)shutdown { - _db.reset(); } - (void)saveQueryData:(FSTQueryData *)queryData group:(FSTWriteGroup *)group { @@ -221,9 +215,10 @@ using leveldb::WriteOptions; * Parses the given bytes as an FSTPBTarget protocol buffer and then converts to the equivalent * query data. */ -- (FSTQueryData *)decodedTargetWithSlice:(Slice)slice { - NSData *data = - [[NSData alloc] initWithBytesNoCopy:(void *)slice.data() length:slice.size() freeWhenDone:NO]; +- (FSTQueryData *)decodeTarget:(absl::string_view)encoded { + NSData *data = [[NSData alloc] initWithBytesNoCopy:(void *)encoded.data() + length:encoded.size() + freeWhenDone:NO]; NSError *error; FSTPBTarget *proto = [FSTPBTarget parseFromData:data error:&error]; @@ -239,7 +234,7 @@ using leveldb::WriteOptions; // Note that this is a scan rather than a get because canonicalIDs are not required to be unique // per target. Slice canonicalID = StringView(query.canonicalID); - std::unique_ptr indexItererator(_db->NewIterator([FSTLevelDB standardReadOptions])); + auto indexItererator = _db.currentTransaction->NewIterator(); std::string indexPrefix = [FSTLevelDBQueryTargetKey keyPrefixWithCanonicalID:canonicalID]; indexItererator->Seek(indexPrefix); @@ -247,15 +242,13 @@ using leveldb::WriteOptions; // unique and ordered, so when scanning a table prefixed by exactly one canonicalID, all the // targetIDs will be unique and in order. std::string targetPrefix = [FSTLevelDBTargetKey keyPrefix]; - std::unique_ptr targetIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); + auto targetIterator = _db.currentTransaction->NewIterator(); FSTLevelDBQueryTargetKey *rowKey = [[FSTLevelDBQueryTargetKey alloc] init]; for (; indexItererator->Valid(); indexItererator->Next()) { - Slice indexKey = indexItererator->key(); - // Only consider rows matching exactly the specific canonicalID of interest. - if (!indexKey.starts_with(indexPrefix) || ![rowKey decodeKey:indexKey] || - canonicalID != rowKey.canonicalID) { + if (!absl::StartsWith(indexItererator->key(), indexPrefix) || + ![rowKey decodeKey:indexItererator->key()] || canonicalID != rowKey.canonicalID) { // End of this canonicalID's possible targets. break; } @@ -272,13 +265,13 @@ using leveldb::WriteOptions; FSTFail( @"Dangling query-target reference found: " @"%@ points to %@; seeking there found %@", - [FSTLevelDBKey descriptionForKey:indexKey], [FSTLevelDBKey descriptionForKey:targetKey], - foundKeyDescription); + [FSTLevelDBKey descriptionForKey:indexItererator->key()], + [FSTLevelDBKey descriptionForKey:targetKey], foundKeyDescription); } // Finally after finding a potential match, check that the query is actually equal to the // requested query. - FSTQueryData *target = [self decodedTargetWithSlice:targetIterator->value()]; + FSTQueryData *target = [self decodeTarget:targetIterator->value()]; if ([target.query isEqual:query]) { return target; } @@ -319,12 +312,12 @@ using leveldb::WriteOptions; - (void)removeMatchingKeysForTargetID:(FSTTargetID)targetID group:(FSTWriteGroup *)group { std::string indexPrefix = [FSTLevelDBTargetDocumentKey keyPrefixWithTargetID:targetID]; - std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); + auto indexIterator = _db.currentTransaction->NewIterator(); indexIterator->Seek(indexPrefix); FSTLevelDBTargetDocumentKey *rowKey = [[FSTLevelDBTargetDocumentKey alloc] init]; for (; indexIterator->Valid(); indexIterator->Next()) { - Slice indexKey = indexIterator->key(); + absl::string_view indexKey = indexIterator->key(); // Only consider rows matching this specific targetID. if (![rowKey decodeKey:indexKey] || rowKey.targetID != targetID) { @@ -342,13 +335,13 @@ using leveldb::WriteOptions; - (FSTDocumentKeySet *)matchingKeysForTargetID:(FSTTargetID)targetID { std::string indexPrefix = [FSTLevelDBTargetDocumentKey keyPrefixWithTargetID:targetID]; - std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); + auto indexIterator = _db.currentTransaction->NewIterator(); indexIterator->Seek(indexPrefix); FSTDocumentKeySet *result = [FSTDocumentKeySet keySet]; FSTLevelDBTargetDocumentKey *rowKey = [[FSTLevelDBTargetDocumentKey alloc] init]; for (; indexIterator->Valid(); indexIterator->Next()) { - Slice indexKey = indexIterator->key(); + absl::string_view indexKey = indexIterator->key(); // Only consider rows matching this specific targetID. if (![rowKey decodeKey:indexKey] || rowKey.targetID != targetID) { @@ -365,7 +358,7 @@ using leveldb::WriteOptions; - (BOOL)containsKey:(FSTDocumentKey *)key { std::string indexPrefix = [FSTLevelDBDocumentTargetKey keyPrefixWithResourcePath:key.path]; - std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); + auto indexIterator = _db.currentTransaction->NewIterator(); indexIterator->Seek(indexPrefix); if (indexIterator->Valid()) { diff --git a/Firestore/Source/Local/FSTLocalStore.mm b/Firestore/Source/Local/FSTLocalStore.mm index a5b85cd..f61a20f 100644 --- a/Firestore/Source/Local/FSTLocalStore.mm +++ b/Firestore/Source/Local/FSTLocalStore.mm @@ -31,6 +31,7 @@ #import "Firestore/Source/Local/FSTReferenceSet.h" #import "Firestore/Source/Local/FSTRemoteDocumentCache.h" #import "Firestore/Source/Local/FSTRemoteDocumentChangeBuffer.h" +#import "Firestore/Source/Local/FSTWriteGroup.h" #import "Firestore/Source/Model/FSTDocument.h" #import "Firestore/Source/Model/FSTDocumentDictionary.h" #import "Firestore/Source/Model/FSTDocumentKey.h" @@ -371,6 +372,7 @@ NS_ASSUME_NONNULL_BEGIN - (void)notifyLocalViewChanges:(NSArray *)viewChanges { FSTReferenceSet *localViewReferences = self.localViewReferences; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"NotifyLocalViewChanges"]; for (FSTLocalViewChanges *view in viewChanges) { FSTQueryData *queryData = [self.queryCache queryDataForQuery:view.query]; FSTAssert(queryData, @"Local view changes contain unallocated query."); @@ -378,6 +380,7 @@ NS_ASSUME_NONNULL_BEGIN [localViewReferences addReferencesToKeys:view.addedKeys forID:targetID]; [localViewReferences removeReferencesToKeys:view.removedKeys forID:targetID]; } + [self.persistence commitGroup:group]; } - (nullable FSTMutationBatch *)nextMutationBatchAfterBatchID:(FSTBatchID)batchID { @@ -389,6 +392,7 @@ NS_ASSUME_NONNULL_BEGIN } - (FSTQueryData *)allocateQuery:(FSTQuery *)query { + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"Allocate query"]; FSTQueryData *cached = [self.queryCache queryDataForQuery:query]; FSTTargetID targetID; FSTListenSequenceNumber sequenceNumber = [self.listenSequence next]; @@ -397,18 +401,14 @@ NS_ASSUME_NONNULL_BEGIN // TODO(mcg): freshen last accessed date? targetID = cached.targetID; } else { - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"Allocate query"]; - targetID = _targetIDGenerator.NextId(); cached = [[FSTQueryData alloc] initWithQuery:query targetID:targetID listenSequenceNumber:sequenceNumber purpose:FSTQueryPurposeListen]; [self.queryCache addQueryData:cached group:group]; - - [self.persistence commitGroup:group]; } - + [self.persistence commitGroup:group]; // Sanity check to ensure that even when resuming a query it's not currently active. FSTBoxedTargetID *boxedTargetID = @(targetID); FSTAssert(!self.targetIDs[boxedTargetID], @"Tried to allocate an already allocated query: %@", @@ -448,20 +448,23 @@ NS_ASSUME_NONNULL_BEGIN } - (FSTDocumentKeySet *)remoteDocumentKeysForTarget:(FSTTargetID)targetID { - return [self.queryCache matchingKeysForTargetID:targetID]; + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"RemoteDocumentKeysForTarget"]; + FSTDocumentKeySet *keySet = [self.queryCache matchingKeysForTargetID:targetID]; + [self.persistence commitGroup:group]; + return keySet; } - (void)collectGarbage { + FSTWriteGroup *group = [self.persistence startGroupWithAction:@"Garbage Collection"]; // Call collectGarbage regardless of whether isGCEnabled so the referenceSet doesn't continue to // accumulate the garbage keys. NSSet *garbage = [self.garbageCollector collectGarbage]; if (garbage.count > 0) { - FSTWriteGroup *group = [self.persistence startGroupWithAction:@"Garbage Collection"]; for (FSTDocumentKey *key in garbage) { [self.remoteDocumentCache removeEntryForKey:key group:group]; } - [self.persistence commitGroup:group]; } + [self.persistence commitGroup:group]; } /** diff --git a/Firestore/core/src/firebase/firestore/local/leveldb_transaction.cc b/Firestore/core/src/firebase/firestore/local/leveldb_transaction.cc index d6c9799..f7d39b2 100644 --- a/Firestore/core/src/firebase/firestore/local/leveldb_transaction.cc +++ b/Firestore/core/src/firebase/firestore/local/leveldb_transaction.cc @@ -166,8 +166,9 @@ void LevelDbTransaction::Put(const absl::string_view& key, version_++; } -LevelDbTransaction::Iterator* LevelDbTransaction::NewIterator() { - return new LevelDbTransaction::Iterator(this); +std::unique_ptr +LevelDbTransaction::NewIterator() { + return std::make_unique(this); } Status LevelDbTransaction::Get(const absl::string_view& key, diff --git a/Firestore/core/src/firebase/firestore/local/leveldb_transaction.h b/Firestore/core/src/firebase/firestore/local/leveldb_transaction.h index b219a69..bcc7c91 100644 --- a/Firestore/core/src/firebase/firestore/local/leveldb_transaction.h +++ b/Firestore/core/src/firebase/firestore/local/leveldb_transaction.h @@ -180,7 +180,7 @@ class LevelDbTransaction { * Returns a new Iterator over the pending changes in this transaction, merged * with the existing values already in leveldb. */ - Iterator* NewIterator(); + std::unique_ptr NewIterator(); /** * Commits the transaction. All pending changes are written. The transaction -- cgit v1.2.3