From 3cbdbf2652202a3473271ed298ff50e5797cce68 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Tue, 30 Jan 2018 14:36:22 -0800 Subject: Schema migrations for LevelDB (#728) * Implement schema versions * Style fixes * newlines, copyrights, assumptions * Fix nullability * Raw ptr -> shared_ptr * kVersionTableGlobal -> kVersionGlobalTable * Drop utils, move into static methods * Drop extra include * Add a few more comments * Move version constant into migrations file * formatting? * Fix comment --- .../Example/Firestore.xcodeproj/project.pbxproj | 42 +++++----- .../Tests/Local/FSTLevelDBMigrationsTests.mm | 75 +++++++++++++++++ .../Tests/Local/FSTPersistenceTestHelpers.h | 6 ++ .../Tests/Local/FSTPersistenceTestHelpers.m | 9 +- Firestore/Source/Local/FSTLevelDB.h | 5 ++ Firestore/Source/Local/FSTLevelDB.mm | 13 ++- Firestore/Source/Local/FSTLevelDBKey.h | 8 ++ Firestore/Source/Local/FSTLevelDBKey.mm | 12 +++ Firestore/Source/Local/FSTLevelDBMigrations.h | 45 ++++++++++ Firestore/Source/Local/FSTLevelDBMigrations.mm | 95 ++++++++++++++++++++++ Firestore/Source/Local/FSTLevelDBQueryCache.h | 8 ++ Firestore/Source/Local/FSTLevelDBQueryCache.mm | 82 ++++++++----------- 12 files changed, 327 insertions(+), 73 deletions(-) create mode 100644 Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm create mode 100644 Firestore/Source/Local/FSTLevelDBMigrations.h create mode 100644 Firestore/Source/Local/FSTLevelDBMigrations.mm diff --git a/Firestore/Example/Firestore.xcodeproj/project.pbxproj b/Firestore/Example/Firestore.xcodeproj/project.pbxproj index 1d39703..a12c4fc 100644 --- a/Firestore/Example/Firestore.xcodeproj/project.pbxproj +++ b/Firestore/Example/Firestore.xcodeproj/project.pbxproj @@ -23,6 +23,7 @@ /* End PBXAggregateTarget section */ /* Begin PBXBuildFile section */ + 132E32A6C1989C284BFE10B2 /* FSTLevelDBMigrationsTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 132E36114FC5BA5DBE3CB260 /* FSTLevelDBMigrationsTests.mm */; }; 3B843E4C1F3A182900548890 /* remote_store_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 3B843E4A1F3930A400548890 /* remote_store_spec_test.json */; }; 5436F32420008FAD006E51E3 /* string_printf_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 5436F32320008FAD006E51E3 /* string_printf_test.cc */; }; 54740A571FC914BA00713A1A /* secure_random_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54740A531FC913E500713A1A /* secure_random_test.cc */; }; @@ -67,8 +68,6 @@ AB356EF7200EA5EB0089B766 /* field_value_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB356EF6200EA5EB0089B766 /* field_value_test.cc */; }; AB380CFB2019388600D97691 /* target_id_generator_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CF82019382300D97691 /* target_id_generator_test.cc */; }; AB380CFE201A2F4500D97691 /* string_util_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CFC201A2EE200D97691 /* string_util_test.cc */; }; - AB380D02201BC69F00D97691 /* bits_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380D01201BC69F00D97691 /* bits_test.cc */; }; - AB380D04201BC6E400D97691 /* ordered_code_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380D03201BC6E400D97691 /* ordered_code_test.cc */; }; AB382F7C1FE02A1F007CA955 /* FIRDocumentReferenceTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AB382F7B1FE02A1F007CA955 /* FIRDocumentReferenceTests.m */; }; AB382F7E1FE03059007CA955 /* FIRFieldPathTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AB382F7D1FE03059007CA955 /* FIRFieldPathTests.m */; }; AB7BAB342012B519001E0872 /* geo_point_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB7BAB332012B519001E0872 /* geo_point_test.cc */; }; @@ -78,7 +77,6 @@ AB99452C1FE3018D00DFC1E6 /* FIRQuerySnapshotTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AB99452B1FE3018D00DFC1E6 /* FIRQuerySnapshotTests.m */; }; AB99452E1FE30AC800DFC1E6 /* FIRFieldValueTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AB99452D1FE30AC800DFC1E6 /* FIRFieldValueTests.m */; }; ABAEEF4F1FD5F8B100C966CB /* FIRQueryTests.m in Sources */ = {isa = PBXBuildFile; fileRef = ABAEEF4E1FD5F8B100C966CB /* FIRQueryTests.m */; }; - ABE6637A201FA81900ED349A /* database_id_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB71064B201FA60300344F18 /* database_id_test.cc */; }; ABF341051FE860CA00C48322 /* FSTAPIHelpers.m in Sources */ = {isa = PBXBuildFile; fileRef = ABF341021FE8593500C48322 /* FSTAPIHelpers.m */; }; ABF6506C201131F8005F2C74 /* timestamp_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = ABF6506B201131F8005F2C74 /* timestamp_test.cc */; }; AFE6114F0D4DAECBA7B7C089 /* Pods_Firestore_IntegrationTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B2FA635DF5D116A67A7441CD /* Pods_Firestore_IntegrationTests.framework */; }; @@ -199,6 +197,7 @@ /* Begin PBXFileReference section */ 04DF37A117F88A9891379ED6 /* Pods-Firestore_Tests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Tests.release.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Tests/Pods-Firestore_Tests.release.xcconfig"; sourceTree = ""; }; 12F4357299652983A615F886 /* LICENSE */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text; name = LICENSE; path = ../LICENSE; sourceTree = ""; }; + 132E36114FC5BA5DBE3CB260 /* FSTLevelDBMigrationsTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FSTLevelDBMigrationsTests.mm; sourceTree = ""; }; 32AD40BF6B0E849B07FFD05E /* Pods_SwiftBuildTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_SwiftBuildTest.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 3B843E4A1F3930A400548890 /* remote_store_spec_test.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = remote_store_spec_test.json; sourceTree = ""; }; 42491D7DC8C8CD245CC22B93 /* Pods-SwiftBuildTest.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-SwiftBuildTest.debug.xcconfig"; path = "Pods/Target Support Files/Pods-SwiftBuildTest/Pods-SwiftBuildTest.debug.xcconfig"; sourceTree = ""; }; @@ -254,11 +253,8 @@ AB356EF6200EA5EB0089B766 /* field_value_test.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = field_value_test.cc; sourceTree = ""; }; AB380CF82019382300D97691 /* target_id_generator_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = target_id_generator_test.cc; sourceTree = ""; }; AB380CFC201A2EE200D97691 /* string_util_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = string_util_test.cc; path = ../../core/test/firebase/firestore/util/string_util_test.cc; sourceTree = ""; }; - AB380D01201BC69F00D97691 /* bits_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = bits_test.cc; path = ../../core/test/firebase/firestore/util/bits_test.cc; sourceTree = ""; }; - AB380D03201BC6E400D97691 /* ordered_code_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ordered_code_test.cc; path = ../../core/test/firebase/firestore/util/ordered_code_test.cc; sourceTree = ""; }; AB382F7B1FE02A1F007CA955 /* FIRDocumentReferenceTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FIRDocumentReferenceTests.m; sourceTree = ""; }; AB382F7D1FE03059007CA955 /* FIRFieldPathTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FIRFieldPathTests.m; sourceTree = ""; }; - AB71064B201FA60300344F18 /* database_id_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = database_id_test.cc; sourceTree = ""; }; AB7BAB332012B519001E0872 /* geo_point_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = geo_point_test.cc; path = ../../core/test/firebase/firestore/geo_point_test.cc; sourceTree = ""; }; AB9945251FE2D71100DFC1E6 /* FIRCollectionReferenceTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FIRCollectionReferenceTests.m; sourceTree = ""; }; AB9945271FE2DE0C00DFC1E6 /* FIRSnapshotMetadataTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FIRSnapshotMetadataTests.m; sourceTree = ""; }; @@ -410,10 +406,8 @@ children = ( 548DB926200D590300E00ABC /* assert_test.cc */, 54740A521FC913E500713A1A /* autoid_test.cc */, - AB380D01201BC69F00D97691 /* bits_test.cc */, 548DB928200D59F600E00ABC /* comparison_test.cc */, 54C2294E1FECABAE007D065B /* log_test.cc */, - AB380D03201BC6E400D97691 /* ordered_code_test.cc */, 54740A531FC913E500713A1A /* secure_random_test.cc */, 5436F32320008FAD006E51E3 /* string_printf_test.cc */, AB380CFC201A2EE200D97691 /* string_util_test.cc */, @@ -426,6 +420,7 @@ children = ( AB380CF7201937B800D97691 /* core */, AB356EF5200E9D1A0089B766 /* model */, + 54764FAD1FAA0C650085E60A /* Port */, 54740A561FC913EB00713A1A /* util */, 54764FAE1FAA21B90085E60A /* FSTGoogleTestTests.mm */, AB7BAB332012B519001E0872 /* geo_point_test.cc */, @@ -433,6 +428,13 @@ name = GoogleTests; sourceTree = ""; }; + 54764FAD1FAA0C650085E60A /* Port */ = { + isa = PBXGroup; + children = ( + ); + name = Port; + sourceTree = ""; + }; 6003F581195388D10070C39A = { isa = PBXGroup; children = ( @@ -555,7 +557,6 @@ children = ( AB356EF6200EA5EB0089B766 /* field_value_test.cc */, ABF6506B201131F8005F2C74 /* timestamp_test.cc */, - AB71064B201FA60300344F18 /* database_id_test.cc */, ); name = model; path = ../../core/test/firebase/firestore/model; @@ -595,31 +596,32 @@ DE51B1621F0D48AC0013853F /* Local */ = { isa = PBXGroup; children = ( + 61E1D8AF1FCF6AF500753285 /* StringViewTests.mm */, + DE51B16A1F0D48AC0013853F /* FSTLocalStoreTests.h */, + DE51B1701F0D48AC0013853F /* FSTMutationQueueTests.h */, + DE51B1721F0D48AC0013853F /* FSTPersistenceTestHelpers.h */, + DE51B1741F0D48AC0013853F /* FSTQueryCacheTests.h */, + DE51B1771F0D48AC0013853F /* FSTRemoteDocumentCacheTests.h */, DE51B1631F0D48AC0013853F /* FSTEagerGarbageCollectorTests.m */, - DE51B1641F0D48AC0013853F /* FSTLevelDBKeyTests.mm */, DE51B1651F0D48AC0013853F /* FSTLevelDBLocalStoreTests.m */, - DE51B1661F0D48AC0013853F /* FSTLevelDBMutationQueueTests.mm */, DE51B1671F0D48AC0013853F /* FSTLevelDBQueryCacheTests.m */, - DE51B1681F0D48AC0013853F /* FSTLevelDBRemoteDocumentCacheTests.mm */, DE51B1691F0D48AC0013853F /* FSTLocalSerializerTests.m */, - DE51B16A1F0D48AC0013853F /* FSTLocalStoreTests.h */, DE51B16B1F0D48AC0013853F /* FSTLocalStoreTests.m */, DE51B16C1F0D48AC0013853F /* FSTMemoryLocalStoreTests.m */, DE51B16D1F0D48AC0013853F /* FSTMemoryMutationQueueTests.m */, DE51B16E1F0D48AC0013853F /* FSTMemoryQueryCacheTests.m */, DE51B16F1F0D48AC0013853F /* FSTMemoryRemoteDocumentCacheTests.m */, - DE51B1701F0D48AC0013853F /* FSTMutationQueueTests.h */, DE51B1711F0D48AC0013853F /* FSTMutationQueueTests.m */, - DE51B1721F0D48AC0013853F /* FSTPersistenceTestHelpers.h */, DE51B1731F0D48AC0013853F /* FSTPersistenceTestHelpers.m */, - DE51B1741F0D48AC0013853F /* FSTQueryCacheTests.h */, DE51B1751F0D48AC0013853F /* FSTQueryCacheTests.m */, DE51B1761F0D48AC0013853F /* FSTReferenceSetTests.m */, - DE51B1771F0D48AC0013853F /* FSTRemoteDocumentCacheTests.h */, DE51B1781F0D48AC0013853F /* FSTRemoteDocumentCacheTests.m */, DE51B1791F0D48AC0013853F /* FSTRemoteDocumentChangeBufferTests.m */, + DE51B1641F0D48AC0013853F /* FSTLevelDBKeyTests.mm */, + DE51B1661F0D48AC0013853F /* FSTLevelDBMutationQueueTests.mm */, + DE51B1681F0D48AC0013853F /* FSTLevelDBRemoteDocumentCacheTests.mm */, DE51B17A1F0D48AC0013853F /* FSTWriteGroupTests.mm */, - 61E1D8AF1FCF6AF500753285 /* StringViewTests.mm */, + 132E36114FC5BA5DBE3CB260 /* FSTLevelDBMigrationsTests.mm */, ); path = Local; sourceTree = ""; @@ -1225,7 +1227,6 @@ files = ( DE2EF0881F3D0B6E003D0CDC /* FSTTreeSortedDictionaryTests.m in Sources */, DE51B1FD1F0D492C0013853F /* FSTSpecTests.m in Sources */, - ABE6637A201FA81900ED349A /* database_id_test.cc in Sources */, ABAEEF4F1FD5F8B100C966CB /* FIRQueryTests.m in Sources */, ABF341051FE860CA00C48322 /* FSTAPIHelpers.m in Sources */, DE51B1CC1F0D48C00013853F /* FIRGeoPointTests.m in Sources */, @@ -1276,7 +1277,6 @@ DE51B1E01F0D490D0013853F /* FSTMemoryQueryCacheTests.m in Sources */, DE51B1E91F0D490D0013853F /* FSTLevelDBMutationQueueTests.mm in Sources */, 54764FAF1FAA21B90085E60A /* FSTGoogleTestTests.mm in Sources */, - AB380D04201BC6E400D97691 /* ordered_code_test.cc in Sources */, DE51B1E61F0D490D0013853F /* FSTRemoteDocumentCacheTests.m in Sources */, 61E1D8B11FCF6C5700753285 /* StringViewTests.mm in Sources */, DE51B1D91F0D490D0013853F /* FSTEagerGarbageCollectorTests.m in Sources */, @@ -1293,7 +1293,6 @@ DE51B1FC1F0D492C0013853F /* FSTMockDatastore.m in Sources */, DE51B1CE1F0D48CD0013853F /* FSTEventManagerTests.m in Sources */, DE51B1E41F0D490D0013853F /* FSTQueryCacheTests.m in Sources */, - AB380D02201BC69F00D97691 /* bits_test.cc in Sources */, DE51B1CD1F0D48CD0013853F /* FSTDatabaseInfoTests.m in Sources */, AB382F7E1FE03059007CA955 /* FIRFieldPathTests.m in Sources */, 548DB929200D59F600E00ABC /* comparison_test.cc in Sources */, @@ -1302,6 +1301,7 @@ 54740A571FC914BA00713A1A /* secure_random_test.cc in Sources */, DE51B1DD1F0D490D0013853F /* FSTLocalStoreTests.m in Sources */, D5B25474286C9800CE42B8C2 /* FSTTestDispatchQueue.m in Sources */, + 132E32A6C1989C284BFE10B2 /* FSTLevelDBMigrationsTests.mm in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm b/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm new file mode 100644 index 0000000..8ef0e94 --- /dev/null +++ b/Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#include + +#import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" +#import "Firestore/Source/Local/FSTLevelDBMigrations.h" +#import "Firestore/Source/Local/FSTLevelDBQueryCache.h" + +#import "Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.h" + +NS_ASSUME_NONNULL_BEGIN + +using leveldb::DB; +using leveldb::Options; +using leveldb::Status; + +@interface FSTLevelDBMigrationsTests : XCTestCase +@end + +@implementation FSTLevelDBMigrationsTests { + std::shared_ptr _db; +} + +- (void)setUp { + Options options; + options.error_if_exists = true; + options.create_if_missing = true; + + NSString *dir = [FSTPersistenceTestHelpers levelDBDir]; + DB *db; + Status status = DB::Open(options, [dir UTF8String], &db); + XCTAssert(status.ok(), @"Failed to create db: %s", status.ToString().c_str()); + _db.reset(db); +} + +- (void)tearDown { + _db.reset(); +} + +- (void)testAddsTargetGlobal { + FSTPBTargetGlobal *metadata = [FSTLevelDBQueryCache readTargetMetadataFromDB:_db]; + XCTAssertNil(metadata, @"Not expecting metadata yet, we should have an empty db"); + [FSTLevelDBMigrations runMigrationsOnDB:_db]; + metadata = [FSTLevelDBQueryCache readTargetMetadataFromDB:_db]; + XCTAssertNotNil(metadata, @"Migrations should have added the metadata"); +} + +- (void)testSetsVersionNumber { + FSTLevelDBSchemaVersion initial = [FSTLevelDBMigrations schemaVersionForDB:_db]; + XCTAssertEqual(0, initial, "No version should be equivalent to 0"); + + // Pick an arbitrary high migration number and migrate to it. + [FSTLevelDBMigrations runMigrationsOnDB:_db]; + FSTLevelDBSchemaVersion actual = [FSTLevelDBMigrations schemaVersionForDB:_db]; + XCTAssertGreaterThan(actual, 0, @"Expected to migrate to a schema version > 0"); +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.h b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.h index 936bacf..5859d4b 100644 --- a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.h +++ b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.h @@ -23,6 +23,12 @@ NS_ASSUME_NONNULL_BEGIN @interface FSTPersistenceTestHelpers : NSObject +/** + * @return The directory where a leveldb instance can store data files. Any files that existed + * there will be deleted first. + */ ++ (NSString *)levelDBDir; + /** * Creates and starts a new FSTLevelDB instance for testing, destroying any previous contents * if they existed. diff --git a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.m b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.m index c773b12..e9e129d 100644 --- a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.m +++ b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.m @@ -26,10 +26,9 @@ NS_ASSUME_NONNULL_BEGIN @implementation FSTPersistenceTestHelpers -+ (FSTLevelDB *)levelDBPersistence { ++ (NSString *)levelDBDir { NSError *error; NSFileManager *files = [NSFileManager defaultManager]; - NSString *dir = [NSTemporaryDirectory() stringByAppendingPathComponent:@"FSTPersistenceTestHelpers"]; if ([files fileExistsAtPath:dir]) { @@ -40,12 +39,18 @@ NS_ASSUME_NONNULL_BEGIN format:@"Failed to clean up leveldb path %@: %@", dir, error]; } } + return dir; +} + ++ (FSTLevelDB *)levelDBPersistence { + NSString *dir = [self levelDBDir]; FSTDatabaseID *databaseID = [FSTDatabaseID databaseIDWithProject:@"p" database:@"d"]; FSTSerializerBeta *remoteSerializer = [[FSTSerializerBeta alloc] initWithDatabaseID:databaseID]; FSTLocalSerializer *serializer = [[FSTLocalSerializer alloc] initWithRemoteSerializer:remoteSerializer]; FSTLevelDB *db = [[FSTLevelDB alloc] initWithDirectory:dir serializer:serializer]; + NSError *error; BOOL success = [db start:&error]; if (!success) { [NSException raise:NSInternalInconsistencyException diff --git a/Firestore/Source/Local/FSTLevelDB.h b/Firestore/Source/Local/FSTLevelDB.h index 762054b..6819116 100644 --- a/Firestore/Source/Local/FSTLevelDB.h +++ b/Firestore/Source/Local/FSTLevelDB.h @@ -23,6 +23,7 @@ namespace leveldb { class DB; +class ReadOptions; class Status; } #endif @@ -70,6 +71,10 @@ NS_ASSUME_NONNULL_BEGIN #ifdef __cplusplus // What follows is the Objective-C++ extension to the API. +/** + * @return A standard set of read options + */ ++ (const leveldb::ReadOptions)standardReadOptions; /** * Creates an NSError based on the given status if the status is not ok. diff --git a/Firestore/Source/Local/FSTLevelDB.mm b/Firestore/Source/Local/FSTLevelDB.mm index 83b932c..d163ed5 100644 --- a/Firestore/Source/Local/FSTLevelDB.mm +++ b/Firestore/Source/Local/FSTLevelDB.mm @@ -20,6 +20,7 @@ #import "FIRFirestoreErrors.h" #import "Firestore/Source/Core/FSTDatabaseInfo.h" +#import "Firestore/Source/Local/FSTLevelDBMigrations.h" #import "Firestore/Source/Local/FSTLevelDBMutationQueue.h" #import "Firestore/Source/Local/FSTLevelDBQueryCache.h" #import "Firestore/Source/Local/FSTLevelDBRemoteDocumentCache.h" @@ -36,6 +37,7 @@ static NSString *const kReservedPathComponent = @"firestore"; using leveldb::DB; using leveldb::Options; +using leveldb::ReadOptions; using leveldb::Status; using leveldb::WriteOptions; @@ -50,6 +52,15 @@ using leveldb::WriteOptions; @implementation FSTLevelDB +/** + * For now this is paranoid, but perhaps disable that in production builds. + */ ++ (const ReadOptions)standardReadOptions { + ReadOptions options; + options.verify_checksums = true; + return options; +} + - (instancetype)initWithDirectory:(NSString *)directory serializer:(FSTLocalSerializer *)serializer { if (self = [super init]) { @@ -115,8 +126,8 @@ using leveldb::WriteOptions; if (!database) { return NO; } - _ptr.reset(database); + [FSTLevelDBMigrations runMigrationsOnDB:_ptr]; return YES; } diff --git a/Firestore/Source/Local/FSTLevelDBKey.h b/Firestore/Source/Local/FSTLevelDBKey.h index 2e9b9b2..e5e7fbb 100644 --- a/Firestore/Source/Local/FSTLevelDBKey.h +++ b/Firestore/Source/Local/FSTLevelDBKey.h @@ -82,6 +82,14 @@ NS_ASSUME_NONNULL_BEGIN @end +/** A key to a singleton row storing the version of the schema. */ +@interface FSTLevelDBVersionKey : NSObject + +/** Returns the key pointing to the singleton row storing the schema version. */ ++ (std::string)key; + +@end + /** A key in the mutations table. */ @interface FSTLevelDBMutationKey : NSObject diff --git a/Firestore/Source/Local/FSTLevelDBKey.mm b/Firestore/Source/Local/FSTLevelDBKey.mm index 9850fff..41aea39 100644 --- a/Firestore/Source/Local/FSTLevelDBKey.mm +++ b/Firestore/Source/Local/FSTLevelDBKey.mm @@ -29,6 +29,7 @@ using firebase::firestore::util::OrderedCode; using Firestore::StringView; using leveldb::Slice; +static const char *kVersionGlobalTable = "version"; static const char *kMutationsTable = "mutation"; static const char *kDocumentMutationsTable = "document_mutation"; static const char *kMutationQueuesTable = "mutation_queue"; @@ -448,6 +449,17 @@ NSString *InvalidKey(const Slice &key) { @end +@implementation FSTLevelDBVersionKey + ++ (std::string)key { + std::string result; + WriteTableName(&result, kVersionGlobalTable); + WriteTerminator(&result); + return result; +} + +@end + @implementation FSTLevelDBMutationKey { std::string _userID; } diff --git a/Firestore/Source/Local/FSTLevelDBMigrations.h b/Firestore/Source/Local/FSTLevelDBMigrations.h new file mode 100644 index 0000000..46c7c93 --- /dev/null +++ b/Firestore/Source/Local/FSTLevelDBMigrations.h @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#include + +#ifdef __cplusplus + +namespace leveldb { +class DB; +} +#endif + +NS_ASSUME_NONNULL_BEGIN + +typedef int32_t FSTLevelDBSchemaVersion; + +@interface FSTLevelDBMigrations : NSObject + +/** + * Returns the current version of the schema for the given database + */ ++ (FSTLevelDBSchemaVersion)schemaVersionForDB:(std::shared_ptr)db; + +/** + * Runs any migrations needed to bring the given database up to the current schema version + */ ++ (void)runMigrationsOnDB:(std::shared_ptr)db; + +@end + +NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTLevelDBMigrations.mm b/Firestore/Source/Local/FSTLevelDBMigrations.mm new file mode 100644 index 0000000..49af893 --- /dev/null +++ b/Firestore/Source/Local/FSTLevelDBMigrations.mm @@ -0,0 +1,95 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Firestore/Source/Local/FSTLevelDBMigrations.h" + +#include +#include + +#import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" +#import "Firestore/Source/Local/FSTLevelDB.h" +#import "Firestore/Source/Local/FSTLevelDBKey.h" +#import "Firestore/Source/Local/FSTLevelDBQueryCache.h" +#import "Firestore/Source/Local/FSTWriteGroup.h" + +NS_ASSUME_NONNULL_BEGIN + +// Current version of the schema defined in this file. +static FSTLevelDBSchemaVersion kSchemaVersion = 1; + +using leveldb::DB; +using leveldb::Status; +using leveldb::Slice; +using leveldb::WriteOptions; + +/** + * Ensures that the global singleton target metadata row exists in LevelDB. + * @param db The db in which to require the row. + */ +static void EnsureTargetGlobal(std::shared_ptr db, FSTWriteGroup *group) { + FSTPBTargetGlobal *targetGlobal = [FSTLevelDBQueryCache readTargetMetadataFromDB:db]; + if (!targetGlobal) { + [group setMessage:[FSTPBTargetGlobal message] forKey:[FSTLevelDBTargetGlobalKey key]]; + } +} + +/** + * Save the given version number as the current version of the schema of the database. + * @param version The version to save + * @param group The transaction in which to save the new version number + */ +static void SaveVersion(FSTLevelDBSchemaVersion version, FSTWriteGroup *group) { + std::string key = [FSTLevelDBVersionKey key]; + std::string version_string = std::to_string(version); + [group setData:version_string forKey:key]; +} + +@implementation FSTLevelDBMigrations + ++ (FSTLevelDBSchemaVersion)schemaVersionForDB:(std::shared_ptr)db { + std::string key = [FSTLevelDBVersionKey key]; + std::string version_string; + Status status = db->Get([FSTLevelDB standardReadOptions], key, &version_string); + if (status.IsNotFound()) { + return 0; + } else { + return stoi(version_string); + } +} + ++ (void)runMigrationsOnDB:(std::shared_ptr)db { + FSTWriteGroup *group = [FSTWriteGroup groupWithAction:@"Migrations"]; + FSTLevelDBSchemaVersion currentVersion = [self schemaVersionForDB:db]; + // Each case in this switch statement intentionally falls through. This lets us + // start at the current schema version and apply any migrations that have not yet + // been applied, to bring us up to current, as defined by the kSchemaVersion constant. + switch (currentVersion) { + case 0: + EnsureTargetGlobal(db, group); + // Fallthrough + default: + if (currentVersion < kSchemaVersion) { + SaveVersion(kSchemaVersion, group); + } + } + if (!group.isEmpty) { + [group writeToDB:db]; + } +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTLevelDBQueryCache.h b/Firestore/Source/Local/FSTLevelDBQueryCache.h index 6d5cd60..67c6575 100644 --- a/Firestore/Source/Local/FSTLevelDBQueryCache.h +++ b/Firestore/Source/Local/FSTLevelDBQueryCache.h @@ -28,12 +28,20 @@ class DB; @class FSTLocalSerializer; @protocol FSTGarbageCollector; +@class FSTPBTargetGlobal; NS_ASSUME_NONNULL_BEGIN /** Cached Queries backed by LevelDB. */ @interface FSTLevelDBQueryCache : NSObject +#ifdef __cplusplus +/** + * Retrieves the global singleton metadata row from the given database, if it exists. + */ ++ (nullable FSTPBTargetGlobal *)readTargetMetadataFromDB:(std::shared_ptr)db; +#endif + - (instancetype)init NS_UNAVAILABLE; /** The garbage collector to notify about potential garbage keys. */ diff --git a/Firestore/Source/Local/FSTLevelDBQueryCache.mm b/Firestore/Source/Local/FSTLevelDBQueryCache.mm index 46af918..b3f4822 100644 --- a/Firestore/Source/Local/FSTLevelDBQueryCache.mm +++ b/Firestore/Source/Local/FSTLevelDBQueryCache.mm @@ -22,6 +22,7 @@ #import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" #import "Firestore/Source/Core/FSTQuery.h" +#import "Firestore/Source/Local/FSTLevelDB.h" #import "Firestore/Source/Local/FSTLevelDBKey.h" #import "Firestore/Source/Local/FSTLocalSerializer.h" #import "Firestore/Source/Local/FSTQueryData.h" @@ -39,17 +40,6 @@ using leveldb::Slice; using leveldb::Status; using leveldb::WriteOptions; -/** - * Returns a standard set of read options. - * - * For now this is paranoid, but perhaps disable that in production builds. - */ -static ReadOptions GetStandardReadOptions() { - ReadOptions options; - options.verify_checksums = true; - return options; -} - @interface FSTLevelDBQueryCache () /** A write-through cached copy of the metadata for the query cache. */ @@ -70,6 +60,29 @@ static ReadOptions GetStandardReadOptions() { FSTSnapshotVersion *_lastRemoteSnapshotVersion; } ++ (nullable FSTPBTargetGlobal *)readTargetMetadataFromDB:(std::shared_ptr)db { + std::string key = [FSTLevelDBTargetGlobalKey key]; + std::string value; + Status status = db->Get([FSTLevelDB standardReadOptions], key, &value); + if (status.IsNotFound()) { + return nil; + } else if (!status.ok()) { + FSTFail(@"metadataForKey: failed loading key %s with status: %s", key.c_str(), + status.ToString().c_str()); + } + + NSData *data = + [[NSData alloc] initWithBytesNoCopy:(void *)value.data() length:value.size() freeWhenDone:NO]; + + NSError *error; + FSTPBTargetGlobal *proto = [FSTPBTargetGlobal parseFromData:data error:&error]; + if (!proto) { + FSTFail(@"FSTPBTargetGlobal failed to parse: %@", error); + } + + return proto; +} + - (instancetype)initWithDB:(std::shared_ptr)db serializer:(FSTLocalSerializer *)serializer { if (self = [super init]) { FSTAssert(db, @"db must not be NULL"); @@ -80,11 +93,10 @@ static ReadOptions GetStandardReadOptions() { } - (void)start { - std::string key = [FSTLevelDBTargetGlobalKey key]; - FSTPBTargetGlobal *metadata = [self metadataForKey:key]; - if (!metadata) { - metadata = [FSTPBTargetGlobal message]; - } + FSTPBTargetGlobal *metadata = [FSTLevelDBQueryCache readTargetMetadataFromDB:_db]; + FSTAssert( + metadata != nil, + @"Found nil metadata, expected schema to be at version 0 which ensures metadata existence"); _lastRemoteSnapshotVersion = [self.serializer decodedVersion:metadata.lastRemoteSnapshotVersion]; self.metadata = metadata; @@ -156,34 +168,6 @@ static ReadOptions GetStandardReadOptions() { [group removeMessageForKey:indexKey]; } -/** - * Looks up the query global metadata associated with the given key. - * - * @return the parsed protocol buffer message or nil if the row referenced by the given key does - * not exist. - */ -- (nullable FSTPBTargetGlobal *)metadataForKey:(const std::string &)key { - std::string value; - Status status = _db->Get(GetStandardReadOptions(), key, &value); - if (status.IsNotFound()) { - return nil; - } else if (!status.ok()) { - FSTFail(@"metadataForKey: failed loading key %s with status: %s", key.c_str(), - status.ToString().c_str()); - } - - NSData *data = - [[NSData alloc] initWithBytesNoCopy:(void *)value.data() length:value.size() freeWhenDone:NO]; - - NSError *error; - FSTPBTargetGlobal *proto = [FSTPBTargetGlobal parseFromData:data error:&error]; - if (!proto) { - FSTFail(@"FSTPBTargetGlobal failed to parse: %@", error); - } - - return proto; -} - /** * Parses the given bytes as an FSTPBTarget protocol buffer and then converts to the equivalent * query data. @@ -206,7 +190,7 @@ static ReadOptions GetStandardReadOptions() { // Note that this is a scan rather than a get because canonicalIDs are not required to be unique // per target. Slice canonicalID = StringView(query.canonicalID); - std::unique_ptr indexItererator(_db->NewIterator(GetStandardReadOptions())); + std::unique_ptr indexItererator(_db->NewIterator([FSTLevelDB standardReadOptions])); std::string indexPrefix = [FSTLevelDBQueryTargetKey keyPrefixWithCanonicalID:canonicalID]; indexItererator->Seek(indexPrefix); @@ -214,7 +198,7 @@ static ReadOptions GetStandardReadOptions() { // unique and ordered, so when scanning a table prefixed by exactly one canonicalID, all the // targetIDs will be unique and in order. std::string targetPrefix = [FSTLevelDBTargetKey keyPrefix]; - std::unique_ptr targetIterator(_db->NewIterator(GetStandardReadOptions())); + std::unique_ptr targetIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); FSTLevelDBQueryTargetKey *rowKey = [[FSTLevelDBQueryTargetKey alloc] init]; for (; indexItererator->Valid(); indexItererator->Next()) { @@ -286,7 +270,7 @@ static ReadOptions GetStandardReadOptions() { - (void)removeMatchingKeysForTargetID:(FSTTargetID)targetID group:(FSTWriteGroup *)group { std::string indexPrefix = [FSTLevelDBTargetDocumentKey keyPrefixWithTargetID:targetID]; - std::unique_ptr indexIterator(_db->NewIterator(GetStandardReadOptions())); + std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); indexIterator->Seek(indexPrefix); FSTLevelDBTargetDocumentKey *rowKey = [[FSTLevelDBTargetDocumentKey alloc] init]; @@ -309,7 +293,7 @@ static ReadOptions GetStandardReadOptions() { - (FSTDocumentKeySet *)matchingKeysForTargetID:(FSTTargetID)targetID { std::string indexPrefix = [FSTLevelDBTargetDocumentKey keyPrefixWithTargetID:targetID]; - std::unique_ptr indexIterator(_db->NewIterator(GetStandardReadOptions())); + std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); indexIterator->Seek(indexPrefix); FSTDocumentKeySet *result = [FSTDocumentKeySet keySet]; @@ -332,7 +316,7 @@ static ReadOptions GetStandardReadOptions() { - (BOOL)containsKey:(FSTDocumentKey *)key { std::string indexPrefix = [FSTLevelDBDocumentTargetKey keyPrefixWithResourcePath:key.path]; - std::unique_ptr indexIterator(_db->NewIterator(GetStandardReadOptions())); + std::unique_ptr indexIterator(_db->NewIterator([FSTLevelDB standardReadOptions])); indexIterator->Seek(indexPrefix); if (indexIterator->Valid()) { -- cgit v1.2.3