mirror of
https://github.com/coracle-social/coracle.git
synced 2024-09-19 11:43:35 +00:00
Add custom cache implementation for tables
This commit is contained in:
parent
fa2693d09e
commit
6dbaf7450a
@ -2,10 +2,12 @@
|
||||
|
||||
## 0.2.18
|
||||
|
||||
- [x] Re-write data storage layer to conserve memory using LRU cache
|
||||
- [x] Re-write data storage layer to conserve memory using a custom LRU cache
|
||||
- [x] Fix bugs with handling invalid keys
|
||||
- [x] Improve pubkey/anonymous login
|
||||
- [x] Generate placeholder profile images (@morkowski)
|
||||
- [x] Fix notifications to more complete and reliable
|
||||
- [x] Update license back to MIT. Enjoy!
|
||||
|
||||
## 0.2.17
|
||||
|
||||
|
@ -1,6 +1,5 @@
|
||||
# Current
|
||||
|
||||
- [ ] Update license
|
||||
- [ ] Test migration
|
||||
- [ ] Fix notifications
|
||||
- [ ] Add quotes to notifications
|
||||
|
@ -1,9 +1,9 @@
|
||||
import type {Writable} from "svelte/store"
|
||||
import LRUCache from "lru-cache"
|
||||
import {throttle} from "throttle-debounce"
|
||||
import {objOf, is, without} from "ramda"
|
||||
import {writable} from "svelte/store"
|
||||
import {isObject, mapValues, ensurePlural} from "hurdak/lib/hurdak"
|
||||
import Cache from "src/util/cache"
|
||||
import {log, error} from "src/util/logger"
|
||||
import {where} from "src/util/misc"
|
||||
|
||||
@ -70,10 +70,10 @@ export const getItem = k => lf("getItem", k)
|
||||
// ----------------------------------------------------------------------------
|
||||
// Database table abstraction, synced to worker storage
|
||||
|
||||
type CacheEntry = [string, {value: any}]
|
||||
type CacheEntry = [string, {value: any; lru: number}]
|
||||
|
||||
type TableOpts = {
|
||||
maxEntries?: number
|
||||
cache?: Cache
|
||||
initialize?: (table: Table) => Promise<Array<CacheEntry>>
|
||||
}
|
||||
|
||||
@ -83,14 +83,15 @@ export class Table {
|
||||
name: string
|
||||
pk: string
|
||||
opts: TableOpts
|
||||
cache: LRUCache<string, any>
|
||||
cache: Cache
|
||||
listeners: Array<(Table) => void>
|
||||
ready: Writable<boolean>
|
||||
interval: number
|
||||
constructor(name, pk, opts: TableOpts = {}) {
|
||||
this.name = name
|
||||
this.pk = pk
|
||||
this.opts = {maxEntries: 1000, initialize: t => this.dump(), ...opts}
|
||||
this.cache = new LRUCache({max: this.opts.maxEntries})
|
||||
this.opts = opts
|
||||
this.cache = this.opts.cache || new Cache()
|
||||
this.listeners = []
|
||||
this.ready = writable(false)
|
||||
|
||||
@ -100,13 +101,27 @@ export class Table {
|
||||
;(async () => {
|
||||
const t = Date.now()
|
||||
|
||||
this.cache.load((await this.opts.initialize(this)) || [])
|
||||
let data = (await getItem(this.name)) || []
|
||||
|
||||
// Backwards compat - we used to store objects rather than cache dump arrays
|
||||
if (isObject(data)) {
|
||||
data = Object.entries(mapValues(objOf("value"), data))
|
||||
}
|
||||
|
||||
// Initialize our cache and notify listeners
|
||||
this.cache.load(data)
|
||||
this._notify()
|
||||
|
||||
log(`Table ${name} ready in ${Date.now() - t}ms (${this.cache.size} records)`)
|
||||
log(`Table ${name} ready in ${Date.now() - t}ms (${this.cache.size()} records)`)
|
||||
|
||||
this.ready.set(true)
|
||||
})()
|
||||
|
||||
// Prune the cache periodically, with jitter to avoid doing all caches at once
|
||||
this.interval = window.setInterval(() => {
|
||||
this.cache.prune()
|
||||
this._persist()
|
||||
}, 30_000 + 10_000 * Math.random())
|
||||
}
|
||||
_persist = throttle(4_000, () => {
|
||||
setItem(this.name, this.cache.dump())
|
||||
@ -174,25 +189,10 @@ export class Table {
|
||||
async drop() {
|
||||
this.cache.clear()
|
||||
|
||||
return removeItem(this.name)
|
||||
}
|
||||
async dump() {
|
||||
let data = (await getItem(this.name)) || []
|
||||
|
||||
// Backwards compat - we used to store objects rather than cache dump arrays
|
||||
if (isObject(data)) {
|
||||
data = Object.entries(mapValues(objOf("value"), data))
|
||||
}
|
||||
|
||||
return data as Array<CacheEntry>
|
||||
await removeItem(this.name)
|
||||
}
|
||||
toArray() {
|
||||
const result = []
|
||||
for (const item of this.cache.values()) {
|
||||
result.push(item)
|
||||
}
|
||||
|
||||
return result
|
||||
return Array.from(this.cache.values())
|
||||
}
|
||||
all(spec = {}) {
|
||||
return this.toArray().filter(where(spec))
|
||||
|
@ -1,13 +1,23 @@
|
||||
import {pluck, all, identity} from "ramda"
|
||||
import {sortBy, pluck, all, identity} from "ramda"
|
||||
import {derived} from "svelte/store"
|
||||
import Cache from "src/util/cache"
|
||||
import {Table, listener, registry} from "src/agent/storage"
|
||||
|
||||
const sortByCreatedAt = sortBy(([k, x]) => -x.value.created_at)
|
||||
|
||||
// Temporarily put no upper bound on people for 0.2.18 migration
|
||||
export const people = new Table("people", "pubkey", {maxEntries: 100000})
|
||||
export const userEvents = new Table("userEvents", "id", {maxEntries: 100000})
|
||||
export const people = new Table("people", "pubkey", {
|
||||
// cache: new Cache({max: 5000}),
|
||||
// cache: new Cache({max: 20_000}),
|
||||
})
|
||||
|
||||
export const userEvents = new Table("userEvents", "id", {
|
||||
cache: new Cache({max: 5000, sort: sortByCreatedAt}),
|
||||
})
|
||||
|
||||
export const notifications = new Table("notifications", "id")
|
||||
export const contacts = new Table("contacts", "pubkey")
|
||||
export const rooms = new Table("rooms", "id")
|
||||
export const notifications = new Table("notifications", "id", {maxEntries: 100000})
|
||||
export const relays = new Table("relays", "url")
|
||||
export const routes = new Table("routes", "id")
|
||||
|
||||
|
@ -15,7 +15,11 @@
|
||||
let events = null
|
||||
|
||||
const prevChecked = $lastChecked.notifications || 0
|
||||
const notifications = watch("notifications", t => sortBy(e => -e.created_at, t.all()))
|
||||
const notifications = watch("notifications", t => {
|
||||
lastChecked.update(assoc("notifications", now()))
|
||||
|
||||
return sortBy(e => -e.created_at, t.all())
|
||||
})
|
||||
|
||||
// Group notifications so we're only showing the parent once per chunk
|
||||
$: events = $notifications
|
||||
@ -43,8 +47,6 @@
|
||||
onMount(() => {
|
||||
document.title = "Notifications"
|
||||
|
||||
lastChecked.update(assoc("notifications", now()))
|
||||
|
||||
return createScroller(async () => {
|
||||
limit += 50
|
||||
})
|
||||
|
80
src/util/cache.ts
Normal file
80
src/util/cache.ts
Normal file
@ -0,0 +1,80 @@
|
||||
import {sortBy, nth} from "ramda"
|
||||
|
||||
type CacheEntry = [string, {value: any; lru: number}]
|
||||
|
||||
type SortFn = (xs: Array<CacheEntry>) => Array<CacheEntry>
|
||||
|
||||
type CacheOptions = {
|
||||
max?: number
|
||||
sort?: SortFn
|
||||
}
|
||||
|
||||
const sortByLru = sortBy(([k, x]) => -x.lru)
|
||||
|
||||
export default class Cache {
|
||||
max: number
|
||||
sort: SortFn
|
||||
data: Map<string, any>
|
||||
lru: Map<string, number>
|
||||
constructor({max = 1000, sort = sortByLru}: CacheOptions = {}) {
|
||||
this.max = max
|
||||
this.sort = sort
|
||||
this.data = new Map()
|
||||
this.lru = new Map()
|
||||
}
|
||||
set(k, v) {
|
||||
this.data.set(k, v)
|
||||
this.lru.set(k, Date.now())
|
||||
}
|
||||
delete(k) {
|
||||
this.data.delete(k)
|
||||
this.lru.delete(k)
|
||||
}
|
||||
get(k) {
|
||||
return this.data.get(k)
|
||||
}
|
||||
size() {
|
||||
return this.data.size
|
||||
}
|
||||
entries() {
|
||||
return this.data.entries()
|
||||
}
|
||||
keys() {
|
||||
return this.data.keys()
|
||||
}
|
||||
values() {
|
||||
return this.data.values()
|
||||
}
|
||||
find(f) {
|
||||
for (const v of this.data.values()) {
|
||||
if (f(v)) {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
load(entries) {
|
||||
for (const [k, {value, lru}] of entries) {
|
||||
this.data.set(k, value)
|
||||
this.lru.set(k, lru)
|
||||
}
|
||||
}
|
||||
dump(): Array<CacheEntry> {
|
||||
return Array.from(this.data.keys()).map(k => [
|
||||
k,
|
||||
{value: this.data.get(k), lru: this.lru.get(k)},
|
||||
])
|
||||
}
|
||||
prune() {
|
||||
if (this.data.size <= this.max) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const k of this.sort(this.dump()).map(nth(0)).slice(this.max)) {
|
||||
this.delete(k)
|
||||
}
|
||||
}
|
||||
clear() {
|
||||
this.data.clear()
|
||||
this.lru.clear()
|
||||
}
|
||||
}
|
@ -68,7 +68,7 @@
|
||||
{/if}
|
||||
<p class="text-sm text-light">{formatTimestamp(timestamp)}</p>
|
||||
</div>
|
||||
<div class="ml-6 text-light">
|
||||
<div class="ml-6 break-all text-light">
|
||||
{ellipsize(note.content, 120)}
|
||||
</div>
|
||||
</button>
|
||||
|
Loading…
Reference in New Issue
Block a user