nostr
/
damus
mirror of git://jb55.com/damus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate NostrDB files to shared app group file container 

This change was made so that NostrDB data can be accessed from different build targets such as the notification service extension.

Upon initialization of NostrDB, it will check both DB file locations (the old documents directory, and the new shared app group container). If it sees the DB is present on the old location, and not on the new location, it will move the files to the new location. In any other condition it will keep the files intact to prevent data loss.

In order to avoid any conflicts between the damusApp's Ndb instance and the extension's Ndb instance when writing or moving the file, a new parameter called "owns_db_file" was added, and set to "false" for the extension. This ensures that the extension will not attempt to move DB files or create a new DB file on its own. Only the main app can move or create the DB file.

Testing
-------

PASS

Device: iPhone 15 Pro simulator
iOS: 17.0.1
Damus: This commit
Steps:
1. Run with the debugger attached to the extension target.
2. Using Apple's push notification testing dashboard, send a test push notification with a real payload (that includes the nostr event under `nostr_event`. Payload generated by strfry-push-notify).
3. Watch logs. It should show a message like "Got push notification from <DISPLAY_NAME>", where `DISPLAY_NAME` is the correct profile name of the user who generated the event. PASS

Regression testing
------------------

Device: iPhone 13 Mini (Real device)
iOS: 17.1.1
Damus: This commit
Other preconditions:
- Damus is at 1.6 (29) at the start of the test
- NostrDB filled with real data on the old location
Steps:
1. Flash (upgrade) the new Damus version (this commit) (This will be the first time upgrading, shared file container is empty)
2. Try to use the app normally. Scroll and navigate to several locations. Interact with some notes. App should be stable, work, and appear to have profile names already (i.e. It shouldn't start with a bunch of npubs in the place of profile names on known contacts). PASS
3. Downgrade back to the App store version (v1.6 (29))
4. Try to use the app normally. Scroll and navigate, interact, etc. App should work and be stable, but profile name cache is expected to be lost (i.e. shows npubs for a bit until profile is reloaded into NostrDB). PASS
5. Upgrade app again to the version in this commit.
6. Repeat step 2. Everything should work as normal and all profiles should be preloaded from the start. PASS

Closes: https://github.com/damus-io/damus/issues/1744
This commit is contained in:
Daniel D’Aquino 2023-11-20 17:31:35 -08:00 committed by William Casarin
parent 87860a7151
commit a1e6be214e
10 changed files with 105 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
DamusNotificationService/NotificationService.swift
View File

@ -16,6 +16,8 @@ class NotificationService: UNNotificationServiceExtension {
override func didReceive(_ request: UNNotificationRequest, withContentHandler contentHandler: @escaping (UNNotificationContent) -> Void) {
self.contentHandler = contentHandler
let ndb: Ndb? = try? Ndb(owns_db_file: false)
// Modify the notification content here...
guard let nostrEventInfoDictionary = request.content.userInfo["nostr_event"] as? [AnyHashable: Any],
let nostrEventInfo = NostrEventInfoFromPushNotification.from(dictionary: nostrEventInfoDictionary) else {
@ -23,6 +25,12 @@ class NotificationService: UNNotificationServiceExtension {
return;
}
// Log that we got a push notification
if let pubkey = Pubkey(hex: nostrEventInfo.pubkey),
let txn = ndb?.lookup_profile(pubkey) {
Log.debug("Got push notification from %s (%s)", for: .push_notifications, (txn.unsafeUnownedValue?.profile?.display_name ?? "Unknown"), nostrEventInfo.pubkey)
}
if let improvedContent = NotificationFormatter.shared.formatMessage(event: nostrEventInfo) {
contentHandler(improvedContent)
}

2
damus.xcodeproj/project.pbxproj
View File

@ -531,6 +531,7 @@
D7DBD41F2B02F15E002A6197 /* NostrKind.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3BEFD32819DE8F00B3DE84 /* NostrKind.swift */; };
D7DBD4202B0307C7002A6197 /* NostrEventInfoFromPushNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = D70A3B182B02DD2D008BD568 /* NostrEventInfoFromPushNotification.swift */; };
D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */; };
D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C2B10272A7B0F5C008AA43E /* Log.swift */; };
D7FF94002AC7AC5300FD969D /* RelayURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7FF93FF2AC7AC5200FD969D /* RelayURL.swift */; };
E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; };
E990020F2955F837003BBC5A /* EditMetadataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E990020E2955F837003BBC5A /* EditMetadataView.swift */; };
@ -3236,6 +3237,7 @@
D7CCFC072B05833200323D86 /* NdbNote.swift in Sources */,
D7CE1B3F2B0BE719002EDAD4 /* Enum.swift in Sources */,
D7CE1B422B0BE719002EDAD4 /* Offset.swift in Sources */,
D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */,
D7CE1B232B0BE1EE002EDAD4 /* bolt11.c in Sources */,
D7CE1B182B0BDFDD002EDAD4 /* mdb.c in Sources */,
D7CCFC162B05894300323D86 /* Pubkey.swift in Sources */,

2
damus/ContentView.swift
View File

@ -608,7 +608,7 @@ struct ContentView: View {
func connect() {
// nostrdb
let ndb = Ndb()!
let ndb = try! Ndb()!
let pool = RelayPool(ndb: ndb)
let model_cache = RelayModelCache()

2
damus/TestData.swift
View File

@ -64,7 +64,7 @@ var test_damus_state: DamusState = ({
}
print("opening \(tempDir!)")
let ndb = Ndb(path: tempDir)!
let ndb = try! Ndb(path: tempDir)!
let our_pubkey = test_pubkey
let pool = RelayPool(ndb: ndb)
let settings = UserSettingsStore()

1
damus/Util/Log.swift
View File

@ -13,6 +13,7 @@ enum LogCategory: String {
case nav
case render
case storage
case push_notifications
}
/// Damus structured logger

2
damus/Views/SaveKeysView.swift
View File

@ -10,7 +10,7 @@ import Security
struct SaveKeysView: View {
let account: CreateAccountModel
let pool: RelayPool = RelayPool(ndb: Ndb()!)
let pool: RelayPool = RelayPool(ndb: try! Ndb()!)
@State var pub_copied: Bool = false
@State var priv_copied: Bool = false
@State var loading: Bool = false

2
damusTests/Mocking/MockDamusState.swift
View File

@ -24,7 +24,7 @@ func generate_test_damus_state(
}
print("opening \(tempDir!)")
let ndb = Ndb(path: tempDir)!
let ndb = try! Ndb(path: tempDir)!
let our_pubkey = test_pubkey
let pool = RelayPool(ndb: ndb)
let settings = UserSettingsStore()

2
damusTests/ProfileViewTests.swift
View File

@ -27,7 +27,7 @@ final class ProfileViewTests: XCTestCase {
let pk4 = Pubkey(hex: "cc590e46363d0fa66bb27081368d01f169b8ffc7c614629d4e9eef6c88b38670")!
let pk5 = Pubkey(hex: "f2aa579bb998627e04a8f553842a09446360c9d708c6141dd119c479f6ab9d29")!
let ndb = Ndb(path: Ndb.db_path)!
let ndb = try! Ndb(path: Ndb.db_path)!
let txn = NdbTxn(ndb: ndb)
let damus_name = "17ldvg64:nq5mhr77"

96
nostrdb/Ndb.swift
View File

@ -6,29 +6,63 @@
//
import Foundation
import OSLog
fileprivate let APPLICATION_GROUP_IDENTIFIER = "group.com.damus"
class Ndb {
let ndb: ndb_t
static var db_path: String {
let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString
return remove_file_prefix(path!)
let owns_db_file: Bool // Determines whether this class should be allowed to create or move the db file.
// NostrDB used to be stored on the app container's document directory
static private var old_db_path: String? {
guard let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString else {
return nil
}
return remove_file_prefix(path)
}
static var db_path: String? {
// Use the `group.com.damus` container, so that it can be accessible from other targets
// e.g. The notification service extension needs to access Ndb data, which is done through this shared file container.
guard let containerURL = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: APPLICATION_GROUP_IDENTIFIER) else {
return nil
}
return remove_file_prefix(containerURL.absoluteString)
}
static private var db_files: [String] = ["data.mdb", "lock.mdb"]
static var empty: Ndb {
Ndb(ndb: ndb_t(ndb: nil))
}
init?(path: String? = nil) {
//try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb")
//try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb")
init?(path: String? = nil, owns_db_file: Bool = true) throws {
var ndb_p: OpaquePointer? = nil
let ingest_threads: Int32 = 4
var mapsize: Int = 1024 * 1024 * 1024 * 32
if path == nil && owns_db_file {
// `nil` path indicates the default path will be used.
// The default path changed over time, so migrate the database to the new location if needed
do {
try Self.migrate_db_location_if_needed()
}
catch {
// If it fails to migrate, the app can still run without serious consequences. Log instead.
Log.error("Error migrating NostrDB to new file container", for: .storage)
}
}
guard let db_path = Self.db_path,
owns_db_file || Self.db_files_exist(path: db_path) else {
return nil // If the caller claims to not own the DB file, and the DB files do not exist, then we should not initialize Ndb
}
let path = path.map(remove_file_prefix) ?? Ndb.db_path
guard let path = path.map(remove_file_prefix) ?? Ndb.db_path else {
throw Errors.cannot_find_db_path
}
let ok = path.withCString { testdir in
var ok = false
@ -45,10 +79,49 @@ class Ndb {
return nil
}
self.owns_db_file = owns_db_file
self.ndb = ndb_t(ndb: ndb_p)
}
private static func migrate_db_location_if_needed() throws {
guard let old_db_path, let db_path else {
throw Errors.cannot_find_db_path
}
let file_manager = FileManager.default
let old_db_files_exist = Self.db_files_exist(path: old_db_path)
let new_db_files_exist = Self.db_files_exist(path: db_path)
// Migration rules:
// 1. If DB files exist in the old path but not the new one, move files to the new path
// 2. If files do not exist anywhere, do nothing (let new DB be initialized)
// 3. If files exist in the new path, but not the old one, nothing needs to be done
// 4. If files exist on both, do nothing.
// Scenario 4 likely means that user has downgraded and re-upgraded.
// Although it might make sense to get the most recent DB, it might lead to data loss.
// If we leave both intact, it makes it easier to fix later, as no data loss would occur.
if old_db_files_exist && !new_db_files_exist {
Log.info("Migrating NostrDB to new file location…", for: .storage)
do {
try db_files.forEach { db_file in
let old_path = "\(old_db_path)/\(db_file)"
let new_path = "\(db_path)/\(db_file)"
try file_manager.moveItem(atPath: old_path, toPath: new_path)
}
Log.info("NostrDB files successfully migrated to the new location", for: .storage)
} catch {
throw Errors.db_file_migration_error
}
}
}
private static func db_files_exist(path: String) -> Bool {
return db_files.allSatisfy { FileManager.default.fileExists(atPath: "\(path)/\($0)") }
}
init(ndb: ndb_t) {
self.owns_db_file = true
self.ndb = ndb
}
@ -238,6 +311,11 @@ class Ndb {
return pks
}
}
enum Errors: Error {
case cannot_find_db_path
case db_file_migration_error
}
deinit {
ndb_destroy(ndb.ndb)

4
nostrdb/Test/NdbTests.swift
View File

@ -55,13 +55,13 @@ final class NdbTests: XCTestCase {
func test_ndb_init() {
do {
let ndb = Ndb(path: db_dir)!
let ndb = try! Ndb(path: db_dir)!
let ok = ndb.process_events(test_wire_events)
XCTAssertTrue(ok)
}
do {
let ndb = Ndb(path: db_dir)!
let ndb = try! Ndb(path: db_dir)!
let id = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")!
let txn = NdbTxn(ndb: ndb)
let note = ndb.lookup_note_with_txn(id: id, txn: txn)