Move rooms/messages over to localforage

README.md

@@ -54,12 +54,16 @@ If you like Coracle and want to support its development, you can donate sats via
 
 # Maintenance
 
+- [ ] Stop using until to paginate, we skip a ton of stuff. Or use until per relay?
+  - Or just stop doing loading and listen
+- [ ] If the latest message in a dm was the user, don't show notification
 - [ ] Normalize relay urls (lowercase, strip trailing slash)
 - [ ] Use nip 56 for reporting
   - https://github.com/nostr-protocol/nips/pull/205#issuecomment-1419234230
 - [ ] Change network tab to list relays the user is connected to
 - [ ] Sync mentions box and in-reply mentions
 - [ ] Channels
+  - [ ] Damus has chats divided into DMs and requests
   - [ ] Ability to leave/mute DM conversation
   - [ ] Add petnames for channels
   - [ ] Add notifications for chat messages

src/agent/data.js

@@ -1,7 +1,7 @@
 import Dexie, {liveQuery} from 'dexie'
 import {pick, isEmpty} from 'ramda'
 import {nip05} from 'nostr-tools'
-import {noop, ensurePlural, switcherFn} from 'hurdak/lib/hurdak'
+import {noop, createMap, ensurePlural, switcherFn} from 'hurdak/lib/hurdak'
 import {now} from 'src/util/misc'
 import {personKinds, Tags, roomAttrs, isRelay} from 'src/util/nostr'
 import database from 'src/agent/database'
@@ -19,8 +19,6 @@ export const db = new Dexie('agent/data/db')
 db.version(13).stores({
   relays: '++url, name',
   alerts: '++id, created_at',
-  messages: '++id, pubkey, recipient',
-  rooms: '++id, joined',
 })
 
 export const updatePeople = async updates => {
@@ -132,7 +130,7 @@ const processRoomEvents = async events => {
       continue
     }
 
-    const room = await db.table('rooms').get(roomId)
+    const room = await database.rooms.get(roomId)
 
     // Merge edits but don't let old ones override new ones
     if (room?.edited_at >= e.created_at) {
@@ -156,7 +154,7 @@ const processRoomEvents = async events => {
   }
 
   if (!isEmpty(updates)) {
-    await db.table('rooms').bulkPut(Object.values(updates))
+    await database.rooms.bulkPut(updates)
   }
 }
 
@@ -165,8 +163,9 @@ const processMessages = async events => {
     .filter(e => e.kind === 4)
     .map(e => ({...e, recipient: Tags.from(e).type("p").values().first()}))
 
+
   if (messages.length > 0) {
-    await db.table('messages').bulkPut(messages)
+    await database.messages.bulkPut(createMap('id', messages))
   }
 }
 

src/agent/database.ts

@@ -1,3 +1,4 @@
+import {debounce} from 'throttle-debounce'
 import {is, prop, without} from 'ramda'
 import {writable} from 'svelte/store'
 import {switcherFn, ensurePlural, first} from 'hurdak/lib/hurdak'
@@ -14,7 +15,8 @@ type Table = {
   name: string
   subscribe: (subscription: (value: any) => void) => (() => void)
   bulkPut: (data: object) => void
-  all: (where: object) => Promise<any>
+  bulkPatch: (data: object) => void
+  all: (where?: object) => Promise<any>
   get: (key: string) => any
 }
 
@@ -29,7 +31,11 @@ class Channel {
   onMessage: (e: MessageEvent) => void
   constructor({onMessage}) {
     this.id = Math.random().toString().slice(2)
-    this.onMessage = e => onMessage(e.data as Message)
+    this.onMessage = e => {
+      if (e.data.channel === this.id) {
+        onMessage(e.data as Message)
+      }
+    }
 
     worker.addEventListener('message', this.onMessage)
   }
@@ -70,9 +76,10 @@ const callLocalforage = async (storeName, method, ...args) => {
 const getItem = (storeName, ...args) => callLocalforage(storeName, 'getItem', ...args)
 const setItem = (storeName, ...args) => callLocalforage(storeName, 'setItem', ...args)
 const removeItem = (storeName, ...args) => callLocalforage(storeName, 'removeItem', ...args)
-const length = (storeName) => callLocalforage(storeName, 'length')
-const clear = (storeName) => callLocalforage(storeName, 'clear')
-const keys = (storeName) => callLocalforage(storeName, 'keys')
+
+const length = storeName => callLocalforage(storeName, 'length')
+const clear = storeName => callLocalforage(storeName, 'clear')
+const keys = storeName => callLocalforage(storeName, 'keys')
 
 const iterate = (storeName, where = {}) => ({
   [Symbol.asyncIterator]() {
@@ -90,6 +97,9 @@ const iterate = (storeName, where = {}) => ({
           promise.resolve()
           channel.close()
         },
+        default: () => {
+          throw new Error(`Invalid topic ${m.topic}`)
+        },
       }),
     })
 
@@ -147,7 +157,11 @@ const defineTable = (name: string): Table => {
     }
   }
 
-  const bulkPut = newData => {
+  const bulkPut = (newData: Record<string, object>): void => {
+    if (is(Array, newData)) {
+      throw new Error(`Updates must be an object, not an array`)
+    }
+
     setAndNotify({...data, ...newData})
 
     // Sync to storage, keeping updates in order
@@ -161,6 +175,19 @@ const defineTable = (name: string): Table => {
     }) as Promise<void>
   }
 
+  const bulkPatch = (updates: Record<string, object>): void => {
+    if (is(Array, updates)) {
+      throw new Error(`Updates must be an object, not an array`)
+    }
+
+    const newData = {}
+    for (const [k, v] of Object.entries(updates)) {
+      newData[k] = {...data[k], ...v}
+    }
+
+    bulkPut(newData)
+  }
+
   const all = (where = {}) => asyncIterableToArray(iterate(name, where), prop('v'))
   const one = (where = {}) => first(all(where))
   const get = k => data[k]
@@ -175,12 +202,14 @@ const defineTable = (name: string): Table => {
     setAndNotify(initialData)
   })()
 
-  registry[name] = {name, subscribe, bulkPut, all, one, get}
+  registry[name] = {name, subscribe, bulkPut, bulkPatch, all, one, get}
 
   return registry[name]
 }
 
 const people = defineTable('people')
+const rooms = defineTable('rooms')
+const messages = defineTable('messages')
 
 // Helper to allow us to listen to changes of any given table
 
@@ -218,10 +247,13 @@ const watch = (names, f) => {
     store.set(initialValue)
   }
 
+  // Debounce refresh so we don't get UI lag
+  const refresh = debounce(300, async () => store.set(await f(...tables)))
+
   // Listen for changes
-  listener.subscribe(async name => {
+  listener.subscribe(name => {
     if (names.includes(name)) {
-      store.set(await f(...tables))
+      refresh()
     }
   })
 
@@ -232,7 +264,9 @@ const watch = (names, f) => {
 
 const getPersonWithFallback = pubkey => people.get(pubkey) || {pubkey}
 
+const clearAll = () => Promise.all(Object.keys(registry).map(clear))
+
 export default {
   getItem, setItem, removeItem, length, clear, keys, iterate,
-  watch, getPersonWithFallback, people,
+  watch, getPersonWithFallback, clearAll, people, rooms, messages,
 }

src/app/messages.js

@@ -1,7 +1,7 @@
 import {pluck, reject} from 'ramda'
 import {get} from 'svelte/store'
 import {synced, now, timedelta, batch} from 'src/util/misc'
-import {listen as _listen, db, user} from 'src/agent'
+import {listen as _listen, database, user} from 'src/agent'
 import loaders from 'src/app/loaders'
 
 let listener
@@ -23,10 +23,7 @@ const listen = async (relays, pubkey) => {
       const $user = get(user)
 
       // Reload annotated messages, don't alert about messages to self
-      const messages = reject(
-        e => e.pubkey === e.recipient,
-        await db.table('messages').toArray()
-      )
+      const messages = reject(e => e.pubkey === e.recipient, await database.messages.all())
 
       if (messages.length > 0) {
         await loaders.loadPeople(relays, pluck('pubkey', messages))

src/routes/Chat.svelte

@@ -1,10 +1,10 @@
 <script>
-  import {without, assoc, uniq, sortBy} from 'ramda'
+  import {without, uniq, assoc, sortBy} from 'ramda'
   import {onMount} from "svelte"
   import {nip19} from 'nostr-tools'
   import {navigate} from "svelte-routing"
   import {fuzzy} from "src/util/misc"
-  import {getRelays, user, lq, database, listen, db} from 'src/agent'
+  import {getRelays, user, database, listen} from 'src/agent'
   import {modal, messages} from 'src/app'
   import loaders from 'src/app/loaders'
   import Room from "src/partials/Room.svelte"
@@ -18,9 +18,9 @@
 
   const {mostRecentByPubkey} = messages
 
-  const rooms = lq(async () => {
-    const rooms = await db.table('rooms').where('joined').equals(1).toArray()
-    const messages = await db.table('messages').toArray()
+  const rooms = database.watch(['rooms', 'messages'], async () => {
+    const rooms = await database.rooms.all({joined: 1})
+    const messages = await database.messages.all()
     const pubkeys = without([$user.pubkey], uniq(messages.flatMap(m => [m.pubkey, m.recipient])))
 
     await loaders.loadPeople(getRelays(), pubkeys)
@@ -30,8 +30,8 @@
       .concat(rooms.map(room => ({type: 'note', ...room})))
   })
 
-  const search = lq(async () => {
-    const rooms = await db.table('rooms').where('joined').equals(0).toArray()
+  const search = database.watch('rooms', async () => {
+    const rooms = await database.rooms.all({joined: 0})
 
     roomsCount = rooms.length
 
@@ -49,11 +49,11 @@
   }
 
   const joinRoom = id => {
-    db.table('rooms').where('id').equals(id).modify({joined: 1})
+    database.rooms.bulkPatch({id, joined: 1})
   }
 
   const leaveRoom = id => {
-    db.table('rooms').where('id').equals(id).modify({joined: 0})
+    database.rooms.bulkPatch({id, joined: 0})
   }
 
   onMount(() => {

src/routes/ChatRoom.svelte

@@ -3,7 +3,7 @@
   import {nip19} from 'nostr-tools'
   import {now, batch} from 'src/util/misc'
   import Channel from 'src/partials/Channel.svelte'
-  import {lq, getRelays, user, db, listen, load} from 'src/agent'
+  import {getRelays, user, database, listen, load} from 'src/agent'
   import {modal} from 'src/app'
   import loaders from 'src/app/loaders'
   import cmd from 'src/app/cmd'
@@ -11,7 +11,7 @@
   export let entity
 
   let {data: roomId} = nip19.decode(entity) as {data: string}
-  let room = lq(() => db.table('rooms').where('id').equals(roomId).first())
+  let room = database.watch('rooms', rooms => rooms.get(roomId))
 
   const getRoomRelays = $room => {
     let relays = getRelays()
@@ -24,9 +24,7 @@
   }
 
   const listenForMessages = async cb => {
-    // Make sure we have our room so we can calculate relays
-    const $room = await db.table('rooms').where('id').equals(roomId).first()
-    const relays = getRoomRelays($room)
+    const relays = getRoomRelays(database.rooms.get(roomId))
 
     return listen(
       relays,

src/routes/Logout.svelte

@@ -2,7 +2,7 @@
   import {fly} from 'svelte/transition'
   import Anchor from 'src/partials/Anchor.svelte'
   import Content from "src/partials/Content.svelte"
-  import {db} from 'src/agent'
+  import {db, database} from 'src/agent'
 
   let confirmed = false
 
@@ -11,6 +11,8 @@
 
     localStorage.clear()
 
+    await database.clearAll()
+
     try {
       await db.delete()
     } catch (e) {

src/routes/Messages.svelte

@@ -4,7 +4,7 @@
   import {personKinds} from 'src/util/nostr'
   import {batch, now} from 'src/util/misc'
   import Channel from 'src/partials/Channel.svelte'
-  import {database, getRelays, user, db, listen, keys} from 'src/agent'
+  import {database, getRelays, user, listen, keys} from 'src/agent'
   import {messages} from 'src/app'
   import {routes} from 'src/app/ui'
   import cmd from 'src/app/cmd'
@@ -36,15 +36,15 @@
     batch(300, async events => {
       // Reload from db since we annotate messages there
       const messageIds = pluck('id', events.filter(e => e.kind === 4))
-      const messages = await db.table('messages').where('id').anyOf(messageIds).toArray()
+      const messages = await database.messages.all({id: messageIds})
 
       cb(await decryptMessages(messages))
     })
   )
 
   const loadMessages = async ({until, limit}) => {
-    const fromThem = await db.table('messages').where('pubkey').equals(pubkey).toArray()
-    const toThem = await db.table('messages').where('recipient').equals(pubkey).toArray()
+    const fromThem = await database.messages.all({pubkey})
+    const toThem = await database.messages.all({recipient: pubkey})
     const events = fromThem.concat(toThem).filter(e => e.created_at < until)
     const messages = sortBy(e => -e.created_at, events).slice(0, limit)
 

src/routes/RelayList.svelte

@@ -105,8 +105,8 @@
       <i slot="before" class="fa-solid fa-search" />
     </Input>
     {/if}
-    {#each (search(q) || []).slice(0, 50) as relay, i (relay.url)}
-    <RelayCard {relay} {i} />
+    {#each (search(q) || []).slice(0, 50) as relay (relay.url)}
+    <RelayCard {relay} />
     {/each}
     <small class="text-center">
       Showing {Math.min(($knownRelays || []).length - relays.length, 50)}

src/views/ChatEdit.svelte

@@ -6,7 +6,7 @@
   import Content from "src/partials/Content.svelte"
   import Textarea from "src/partials/Textarea.svelte"
   import Button from "src/partials/Button.svelte"
-  import {getRelays, db} from 'src/agent'
+  import {getRelays, database} from 'src/agent'
   import {toast, modal} from "src/app"
   import cmd from "src/app/cmd"
 
@@ -34,11 +34,11 @@
     if (!room.name) {
       toast.show("error", "Please enter a name for your room.")
     } else {
-      room.id
+      const event = room.id
         ? await cmd.updateRoom(getRelays(), room)
         : await cmd.createRoom(getRelays(), room)
 
-      await db.table('rooms').where('id').equals(room.id).modify({joined: 1})
+      await database.rooms.bulkPatch({id: room.id || event.id, joined: 1})
 
       toast.show("info", `Your room has been ${room.id ? 'updated' : 'created'}!`)
 

src/workers/database.js

@@ -1,5 +1,5 @@
 import lf from 'localforage'
-import {complement, equals, isNil, pipe, prop, identity, allPass} from 'ramda'
+import {is, complement, equals, isNil, pipe, prop, identity, allPass} from 'ramda'
 import {switcherFn} from 'hurdak/lib/hurdak'
 
 const stores = {}
@@ -35,9 +35,11 @@ addEventListener('message', async ({data: {topic, payload, channel}}) => {
               modifier = complement
             }
 
-            if (operator === 'eq') {
+            if (operator === 'eq' && is(Array, value)) {
+              test = v => value.includes(v)
+            } else if (operator === 'eq') {
               test = equals(value)
-            } if (operator === 'nil') {
+            } else if (operator === 'nil') {
               test = isNil
             } else {
               throw new Error(`Invalid operator ${operator}`)