mirror of
https://github.com/coracle-social/coracle.git
synced 2024-09-19 19:46:42 +00:00
Add custom cache implementation for tables
This commit is contained in:
parent
fa2693d09e
commit
6dbaf7450a
@ -2,10 +2,12 @@
|
|||||||
|
|
||||||
## 0.2.18
|
## 0.2.18
|
||||||
|
|
||||||
- [x] Re-write data storage layer to conserve memory using LRU cache
|
- [x] Re-write data storage layer to conserve memory using a custom LRU cache
|
||||||
- [x] Fix bugs with handling invalid keys
|
- [x] Fix bugs with handling invalid keys
|
||||||
- [x] Improve pubkey/anonymous login
|
- [x] Improve pubkey/anonymous login
|
||||||
- [x] Generate placeholder profile images (@morkowski)
|
- [x] Generate placeholder profile images (@morkowski)
|
||||||
|
- [x] Fix notifications to more complete and reliable
|
||||||
|
- [x] Update license back to MIT. Enjoy!
|
||||||
|
|
||||||
## 0.2.17
|
## 0.2.17
|
||||||
|
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
# Current
|
# Current
|
||||||
|
|
||||||
- [ ] Update license
|
|
||||||
- [ ] Test migration
|
- [ ] Test migration
|
||||||
- [ ] Fix notifications
|
- [ ] Fix notifications
|
||||||
- [ ] Add quotes to notifications
|
- [ ] Add quotes to notifications
|
||||||
|
@ -1,9 +1,9 @@
|
|||||||
import type {Writable} from "svelte/store"
|
import type {Writable} from "svelte/store"
|
||||||
import LRUCache from "lru-cache"
|
|
||||||
import {throttle} from "throttle-debounce"
|
import {throttle} from "throttle-debounce"
|
||||||
import {objOf, is, without} from "ramda"
|
import {objOf, is, without} from "ramda"
|
||||||
import {writable} from "svelte/store"
|
import {writable} from "svelte/store"
|
||||||
import {isObject, mapValues, ensurePlural} from "hurdak/lib/hurdak"
|
import {isObject, mapValues, ensurePlural} from "hurdak/lib/hurdak"
|
||||||
|
import Cache from "src/util/cache"
|
||||||
import {log, error} from "src/util/logger"
|
import {log, error} from "src/util/logger"
|
||||||
import {where} from "src/util/misc"
|
import {where} from "src/util/misc"
|
||||||
|
|
||||||
@ -70,10 +70,10 @@ export const getItem = k => lf("getItem", k)
|
|||||||
// ----------------------------------------------------------------------------
|
// ----------------------------------------------------------------------------
|
||||||
// Database table abstraction, synced to worker storage
|
// Database table abstraction, synced to worker storage
|
||||||
|
|
||||||
type CacheEntry = [string, {value: any}]
|
type CacheEntry = [string, {value: any; lru: number}]
|
||||||
|
|
||||||
type TableOpts = {
|
type TableOpts = {
|
||||||
maxEntries?: number
|
cache?: Cache
|
||||||
initialize?: (table: Table) => Promise<Array<CacheEntry>>
|
initialize?: (table: Table) => Promise<Array<CacheEntry>>
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -83,14 +83,15 @@ export class Table {
|
|||||||
name: string
|
name: string
|
||||||
pk: string
|
pk: string
|
||||||
opts: TableOpts
|
opts: TableOpts
|
||||||
cache: LRUCache<string, any>
|
cache: Cache
|
||||||
listeners: Array<(Table) => void>
|
listeners: Array<(Table) => void>
|
||||||
ready: Writable<boolean>
|
ready: Writable<boolean>
|
||||||
|
interval: number
|
||||||
constructor(name, pk, opts: TableOpts = {}) {
|
constructor(name, pk, opts: TableOpts = {}) {
|
||||||
this.name = name
|
this.name = name
|
||||||
this.pk = pk
|
this.pk = pk
|
||||||
this.opts = {maxEntries: 1000, initialize: t => this.dump(), ...opts}
|
this.opts = opts
|
||||||
this.cache = new LRUCache({max: this.opts.maxEntries})
|
this.cache = this.opts.cache || new Cache()
|
||||||
this.listeners = []
|
this.listeners = []
|
||||||
this.ready = writable(false)
|
this.ready = writable(false)
|
||||||
|
|
||||||
@ -100,13 +101,27 @@ export class Table {
|
|||||||
;(async () => {
|
;(async () => {
|
||||||
const t = Date.now()
|
const t = Date.now()
|
||||||
|
|
||||||
this.cache.load((await this.opts.initialize(this)) || [])
|
let data = (await getItem(this.name)) || []
|
||||||
|
|
||||||
|
// Backwards compat - we used to store objects rather than cache dump arrays
|
||||||
|
if (isObject(data)) {
|
||||||
|
data = Object.entries(mapValues(objOf("value"), data))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize our cache and notify listeners
|
||||||
|
this.cache.load(data)
|
||||||
this._notify()
|
this._notify()
|
||||||
|
|
||||||
log(`Table ${name} ready in ${Date.now() - t}ms (${this.cache.size} records)`)
|
log(`Table ${name} ready in ${Date.now() - t}ms (${this.cache.size()} records)`)
|
||||||
|
|
||||||
this.ready.set(true)
|
this.ready.set(true)
|
||||||
})()
|
})()
|
||||||
|
|
||||||
|
// Prune the cache periodically, with jitter to avoid doing all caches at once
|
||||||
|
this.interval = window.setInterval(() => {
|
||||||
|
this.cache.prune()
|
||||||
|
this._persist()
|
||||||
|
}, 30_000 + 10_000 * Math.random())
|
||||||
}
|
}
|
||||||
_persist = throttle(4_000, () => {
|
_persist = throttle(4_000, () => {
|
||||||
setItem(this.name, this.cache.dump())
|
setItem(this.name, this.cache.dump())
|
||||||
@ -174,25 +189,10 @@ export class Table {
|
|||||||
async drop() {
|
async drop() {
|
||||||
this.cache.clear()
|
this.cache.clear()
|
||||||
|
|
||||||
return removeItem(this.name)
|
await removeItem(this.name)
|
||||||
}
|
|
||||||
async dump() {
|
|
||||||
let data = (await getItem(this.name)) || []
|
|
||||||
|
|
||||||
// Backwards compat - we used to store objects rather than cache dump arrays
|
|
||||||
if (isObject(data)) {
|
|
||||||
data = Object.entries(mapValues(objOf("value"), data))
|
|
||||||
}
|
|
||||||
|
|
||||||
return data as Array<CacheEntry>
|
|
||||||
}
|
}
|
||||||
toArray() {
|
toArray() {
|
||||||
const result = []
|
return Array.from(this.cache.values())
|
||||||
for (const item of this.cache.values()) {
|
|
||||||
result.push(item)
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
}
|
||||||
all(spec = {}) {
|
all(spec = {}) {
|
||||||
return this.toArray().filter(where(spec))
|
return this.toArray().filter(where(spec))
|
||||||
|
@ -1,13 +1,23 @@
|
|||||||
import {pluck, all, identity} from "ramda"
|
import {sortBy, pluck, all, identity} from "ramda"
|
||||||
import {derived} from "svelte/store"
|
import {derived} from "svelte/store"
|
||||||
|
import Cache from "src/util/cache"
|
||||||
import {Table, listener, registry} from "src/agent/storage"
|
import {Table, listener, registry} from "src/agent/storage"
|
||||||
|
|
||||||
|
const sortByCreatedAt = sortBy(([k, x]) => -x.value.created_at)
|
||||||
|
|
||||||
// Temporarily put no upper bound on people for 0.2.18 migration
|
// Temporarily put no upper bound on people for 0.2.18 migration
|
||||||
export const people = new Table("people", "pubkey", {maxEntries: 100000})
|
export const people = new Table("people", "pubkey", {
|
||||||
export const userEvents = new Table("userEvents", "id", {maxEntries: 100000})
|
// cache: new Cache({max: 5000}),
|
||||||
|
// cache: new Cache({max: 20_000}),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const userEvents = new Table("userEvents", "id", {
|
||||||
|
cache: new Cache({max: 5000, sort: sortByCreatedAt}),
|
||||||
|
})
|
||||||
|
|
||||||
|
export const notifications = new Table("notifications", "id")
|
||||||
export const contacts = new Table("contacts", "pubkey")
|
export const contacts = new Table("contacts", "pubkey")
|
||||||
export const rooms = new Table("rooms", "id")
|
export const rooms = new Table("rooms", "id")
|
||||||
export const notifications = new Table("notifications", "id", {maxEntries: 100000})
|
|
||||||
export const relays = new Table("relays", "url")
|
export const relays = new Table("relays", "url")
|
||||||
export const routes = new Table("routes", "id")
|
export const routes = new Table("routes", "id")
|
||||||
|
|
||||||
|
@ -15,7 +15,11 @@
|
|||||||
let events = null
|
let events = null
|
||||||
|
|
||||||
const prevChecked = $lastChecked.notifications || 0
|
const prevChecked = $lastChecked.notifications || 0
|
||||||
const notifications = watch("notifications", t => sortBy(e => -e.created_at, t.all()))
|
const notifications = watch("notifications", t => {
|
||||||
|
lastChecked.update(assoc("notifications", now()))
|
||||||
|
|
||||||
|
return sortBy(e => -e.created_at, t.all())
|
||||||
|
})
|
||||||
|
|
||||||
// Group notifications so we're only showing the parent once per chunk
|
// Group notifications so we're only showing the parent once per chunk
|
||||||
$: events = $notifications
|
$: events = $notifications
|
||||||
@ -43,8 +47,6 @@
|
|||||||
onMount(() => {
|
onMount(() => {
|
||||||
document.title = "Notifications"
|
document.title = "Notifications"
|
||||||
|
|
||||||
lastChecked.update(assoc("notifications", now()))
|
|
||||||
|
|
||||||
return createScroller(async () => {
|
return createScroller(async () => {
|
||||||
limit += 50
|
limit += 50
|
||||||
})
|
})
|
||||||
|
80
src/util/cache.ts
Normal file
80
src/util/cache.ts
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
import {sortBy, nth} from "ramda"
|
||||||
|
|
||||||
|
type CacheEntry = [string, {value: any; lru: number}]
|
||||||
|
|
||||||
|
type SortFn = (xs: Array<CacheEntry>) => Array<CacheEntry>
|
||||||
|
|
||||||
|
type CacheOptions = {
|
||||||
|
max?: number
|
||||||
|
sort?: SortFn
|
||||||
|
}
|
||||||
|
|
||||||
|
const sortByLru = sortBy(([k, x]) => -x.lru)
|
||||||
|
|
||||||
|
export default class Cache {
|
||||||
|
max: number
|
||||||
|
sort: SortFn
|
||||||
|
data: Map<string, any>
|
||||||
|
lru: Map<string, number>
|
||||||
|
constructor({max = 1000, sort = sortByLru}: CacheOptions = {}) {
|
||||||
|
this.max = max
|
||||||
|
this.sort = sort
|
||||||
|
this.data = new Map()
|
||||||
|
this.lru = new Map()
|
||||||
|
}
|
||||||
|
set(k, v) {
|
||||||
|
this.data.set(k, v)
|
||||||
|
this.lru.set(k, Date.now())
|
||||||
|
}
|
||||||
|
delete(k) {
|
||||||
|
this.data.delete(k)
|
||||||
|
this.lru.delete(k)
|
||||||
|
}
|
||||||
|
get(k) {
|
||||||
|
return this.data.get(k)
|
||||||
|
}
|
||||||
|
size() {
|
||||||
|
return this.data.size
|
||||||
|
}
|
||||||
|
entries() {
|
||||||
|
return this.data.entries()
|
||||||
|
}
|
||||||
|
keys() {
|
||||||
|
return this.data.keys()
|
||||||
|
}
|
||||||
|
values() {
|
||||||
|
return this.data.values()
|
||||||
|
}
|
||||||
|
find(f) {
|
||||||
|
for (const v of this.data.values()) {
|
||||||
|
if (f(v)) {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
load(entries) {
|
||||||
|
for (const [k, {value, lru}] of entries) {
|
||||||
|
this.data.set(k, value)
|
||||||
|
this.lru.set(k, lru)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dump(): Array<CacheEntry> {
|
||||||
|
return Array.from(this.data.keys()).map(k => [
|
||||||
|
k,
|
||||||
|
{value: this.data.get(k), lru: this.lru.get(k)},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
prune() {
|
||||||
|
if (this.data.size <= this.max) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const k of this.sort(this.dump()).map(nth(0)).slice(this.max)) {
|
||||||
|
this.delete(k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clear() {
|
||||||
|
this.data.clear()
|
||||||
|
this.lru.clear()
|
||||||
|
}
|
||||||
|
}
|
@ -68,7 +68,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
<p class="text-sm text-light">{formatTimestamp(timestamp)}</p>
|
<p class="text-sm text-light">{formatTimestamp(timestamp)}</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="ml-6 text-light">
|
<div class="ml-6 break-all text-light">
|
||||||
{ellipsize(note.content, 120)}
|
{ellipsize(note.content, 120)}
|
||||||
</div>
|
</div>
|
||||||
</button>
|
</button>
|
||||||
|
Loading…
Reference in New Issue
Block a user