florian/blossom-drive-webserver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Some code cleanup

Browse Source
This commit is contained in:
florian 2024-03-18 20:47:30 +01:00
parent d0991e3784
commit beda39b085
5 changed files with 66 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cdn.ts
View File

@ -5,14 +5,14 @@ import uniq from 'lodash/uniq.js';
const log = debug('web:discover:upstream');
export async function searchCdn(servers: string[], hash: string) {
export async function searchHashOnCDNServers(servers: string[], hash: string) {
const uniqueSevers = uniq(servers);
log('Looking for', hash);
for (const server of uniqueSevers) {
try {
log('Checking CDN server', server);
return await checkCDN(server, hash);
return await lookupHashOnCdnServer(server, hash);
} catch (e) {
log('Ingoring error', e);
} // Ignore errors during lookup
@ -20,7 +20,7 @@ export async function searchCdn(servers: string[], hash: string) {
log('No CDN with the content found.');
}
async function checkCDN(cdn: string, hash: string): Promise<IncomingMessage> {
async function lookupHashOnCdnServer(cdn: string, hash: string): Promise<IncomingMessage> {
return new Promise((resolve, reject) => {
const url = new URL(hash, cdn);
const backend = url.protocol === 'https:' ? followRedirects.https : followRedirects.http;

66
src/drive.ts
View File

@ -1,32 +1,39 @@
import NDK, { NDKEvent, NDKKind, NDKRelaySet } from '@nostr-dev-kit/ndk';
import debug from 'debug';
import NodeCache from 'node-cache';
import { Drive } from './types.js';
import { Drive, FileMeta } from './types.js';
import uniqBy from 'lodash/uniqBy.js';
import { DRIVE_CACHE_TTL, DRIVE_REFRESH_TIME, NOSTR_DEFAULT_RELAYS, SINGLE_SITE } from './env.js';
import {
DRIVE_CACHE_TTL,
DRIVE_REFRESH_INTERVAL,
DRIVE_REFRESH_TIMEOUT,
NOSTR_DEFAULT_RELAYS,
SINGLE_SITE,
} from './env.js';
import { nip19 } from 'nostr-tools';
import { AddressPointer } from 'nostr-tools/nip19';
const driveCache = new NodeCache({ stdTTL: DRIVE_CACHE_TTL });
import { prefixSlashIfMissing } from './utils.js';
const log = debug('web:drive:nostr');
// TODO use relays from naddr
export const DRIVE_KIND = 30563;
// Intialize Cache that stores Blossom Drive data
const driveCache = new NodeCache({ stdTTL: DRIVE_CACHE_TTL });
// Intialize NDK
const ndk = new NDK({
explicitRelayUrls: NOSTR_DEFAULT_RELAYS,
});
ndk.connect();
export const DRIVE_KIND = 30563;
const handleEvent = (event: NDKEvent | null): Drive | undefined => {
if (!event) return;
const kind = event.kind;
const pubkey = event.pubkey;
const driveIdentifier = event.tags.find((t) => t[0] === 'd')?.[1];
const driveKey = `${kind}-${pubkey}-${driveIdentifier}`;
log(driveKey);
const treeTags = event.tags.filter((t) => t[0] === 'x' || t[0] === 'folder');
const files = treeTags.map((t) => ({
@ -35,38 +42,44 @@ const handleEvent = (event: NDKEvent | null): Drive | undefined => {
size: parseInt(t[3], 10) || 0,
mimeType: t[4],
}));
// log(files);
const servers = event.tags.filter((t) => t[0] === 'r' && t[1]).map((t) => new URL('/', t[1]).toString()) || [];
// log(servers);
const drive = { files, servers };
// Store Drive Data in the cache
driveCache.set(driveKey, drive);
return drive;
};
// TODO Add incremental update with from date
const fetchAllDrives = async () => {
// TODO incremental update with from date
let sortedEvents: NDKEvent[] = [];
const sub = await ndk.subscribe(
{
kinds: [DRIVE_KIND as NDKKind],
},
{ closeOnEose: true },
);
sub.on('event', (event: NDKEvent) => {
const newEvents = sortedEvents
.concat([event])
.sort((a: NDKEvent, b: NDKEvent) => (b.created_at ?? 0) - (a.created_at ?? 0));
sortedEvents = uniqBy(newEvents, (e: NDKEvent) => e.tagId());
});
sub.start();
sub.on('close', () => {
for (const event of sortedEvents) {
handleEvent(event);
}
});
setTimeout(() => sub.stop(), 10000);
sub.start();
setTimeout(() => sub.stop(), DRIVE_REFRESH_TIMEOUT);
};
export const readDrive = async (
@ -82,29 +95,34 @@ export const readDrive = async (
}
log('Fetching drive event.');
const filter = {
kinds: [kind],
authors: [pubkey],
'#d': [driveIdentifier], // name.toLowerCase().replaceAll(/\s/g, "-") || nanoid(8);
'#d': [driveIdentifier],
};
log(filter);
const event = await ndk.fetchEvent(
filter,
{},
NDKRelaySet.fromRelayUrls(relays?.concat(NOSTR_DEFAULT_RELAYS) || NOSTR_DEFAULT_RELAYS, ndk),
);
log('fetch finsihed');
if (event) {
return handleEvent(event);
} else {
log('no drive event found.');
log(`No drive event found for ${driveIdentifier}`);
}
};
// TODO add lastchange date, to fetch changes
// Load all drives into cache and refresh periodically
export const findFile = (drive: Drive, filePathToSearch: string): FileMeta | undefined => {
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.find((f) => f.path == searchPath);
};
export const findFolderContents = (drive: Drive, filePathToSearch: string): FileMeta[] => {
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.filter((f) => f.path.startsWith(searchPath));
};
if (SINGLE_SITE) {
// Load the single drive into the cache
@ -115,6 +133,10 @@ if (SINGLE_SITE) {
}
} else {
// Load all drives into the cache
// TODO add lastchange date, to fetch changes
// Load all drives into cache and refresh periodically
fetchAllDrives();
setInterval(() => fetchAllDrives(), DRIVE_REFRESH_TIME);
setInterval(() => fetchAllDrives(), DRIVE_REFRESH_INTERVAL);
}

13
src/env.ts
View File

@ -4,11 +4,16 @@ export const SINGLE_SITE = process.env.SINGLE_SITE;
export const FOLDER_LISTING = (process.env.FOLDER_LISTING || 'true') === 'true';
export const DRIVE_PRELOAD = (process.env.DRIVE_PRELOAD || 'false') === 'true';
export const DRIVE_REFRESH_TIME = parseInt(process.env.DRIVE_REFRESH_TIME || '300', 10) * 1000; // 5min
export const DRIVE_REFRESH_INTERVAL = parseInt(process.env.DRIVE_REFRESH_INTERVAL || '300', 10) * 1000; // 5min
export const DRIVE_REFRESH_TIMEOUT = parseInt(process.env.DRIVE_REFRESH_TIMEOUT || '10', 10) * 1000; // 10s
export const DRIVE_CACHE_TTL = parseInt(process.env.DRIVE_CACHE_TTL || '3600', 10); // 1h
export const CDN_CACHE_DIR = process.env.CDN_CACHE_DIR || './cache';
export const CDN_CACHE_DIR = process.env.CDN_CACHE_DIR || './cache';
export const CDN_MAX_LOCAL_CACHE_SIZE = parseInt(process.env.CDN_MAX_LOCAL_CACHE_SIZE || '', 100000); // 100KB
export const CDN_ADDITIONAL_SERVERS = (process.env.CDN_ADDITIONAL_SERVERS || 'https://cdn.hzrd149.com,https://cdn.satellite.earth').split(',');
export const CDN_ADDITIONAL_SERVERS = (
process.env.CDN_ADDITIONAL_SERVERS || 'https://cdn.hzrd149.com,https://cdn.satellite.earth'
).split(',');
export const NOSTR_DEFAULT_RELAYS = (process.env.NOSTR_DEFAULT_RELAYS || 'wss://nostrue.com,wss://relay.damus.io,wss://nos.lol').split(',');
export const NOSTR_DEFAULT_RELAYS = (
process.env.NOSTR_DEFAULT_RELAYS || 'wss://nostrue.com,wss://relay.damus.io,wss://nos.lol'
).split(',');

32
src/index.ts
View File

@ -5,8 +5,8 @@ import Router from '@koa/router';
import { nip19 } from 'nostr-tools';
import { AddressPointer } from 'nostr-tools/nip19';
import { Drive, FileMeta } from './types.js';
import { readDrive } from './drive.js';
import { searchCdn } from './cdn.js';
import { findFile, findFolderContents, readDrive } from './drive.js';
import { searchHashOnCDNServers } from './cdn.js';
import { PassThrough } from 'stream';
import fs from 'node:fs';
import nodePath from 'path';
@ -21,21 +21,11 @@ import {
import { appendSlashIfMissing, prefixSlashIfMissing, removeLeadingSlashes } from './utils.js';
const log = debug('web');
const app = new Koa();
const router = new Router();
// const cacheAdapter = new NDKCacheAdapterDexie({ dbName: "ndk-cache" });
const findFile = (drive: Drive, filePathToSearch: string): FileMeta | undefined => {
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.find((f) => f.path == searchPath);
};
const findFolderContents = (drive: Drive, filePathToSearch: string): FileMeta[] => {
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.filter((f) => f.path.startsWith(searchPath));
};
// Set up cache dir
if (!fs.existsSync(CDN_CACHE_DIR)) {
fs.mkdirSync(CDN_CACHE_DIR, { recursive: true });
}
@ -61,15 +51,12 @@ router.get('/health', async (ctx, next) => {
ctx.status = 200;
});
const storeInCache = async (src: NodeJS.ReadableStream, cacheFile: string) => {
const storeInLocalFileCache = async (src: NodeJS.ReadableStream, cacheFile: string) => {
log('storing cache file:', cacheFile);
const cacheFileStream = fs.createWriteStream(cacheFile);
src.pipe(cacheFileStream);
//const hash = createHash("sha256");
//stream.pipe(hash);
return new Promise<void>((res) => {
cacheFileStream.on('close', async () => {
log('cache file stored:', cacheFile);
@ -97,12 +84,12 @@ const serveDriveFile = async (ctx: Koa.ParameterizedContext, drive: Drive, fileM
serveStream(ctx, mimeType, src);
} else {
// lookup media sevrers for user -> ndk (optional)
const cdnSource = await searchCdn([...drive.servers, ...CDN_ADDITIONAL_SERVERS], hash);
const cdnSource = await searchHashOnCDNServers([...drive.servers, ...CDN_ADDITIONAL_SERVERS], hash);
if (cdnSource) {
if (size < CDN_MAX_LOCAL_CACHE_SIZE) {
// if small file < 100KB, download and serve downloaded file
await storeInCache(cdnSource, cacheFile);
await storeInLocalFileCache(cdnSource, cacheFile);
serveStream(ctx, mimeType, fs.createReadStream(cacheFile));
} else {
// else ()>100kb) stream content from backend.
@ -119,14 +106,11 @@ const serveDriveFile = async (ctx: Koa.ParameterizedContext, drive: Drive, fileM
router.get('(.*)', async (ctx, next) => {
log(`serving: host:${ctx.hostname} path:${ctx.path}`);
// We don't serve favicon.ico
if (ctx.path == '/favicon.ico') {
return next();
}
// naddr1qvzqqqrhvvpzpd7x76g4e756vtlldg0syczdazxz83kxcmgm3a3v0nqswj0nql5pqqzhgetnwseqmkcq93
// test2
// 1708411351353.jpeg
// http://localhost:3010/naddr1qvzqqqrhvvpzpd7x76g4e756vtlldg0syczdazxz83kxcmgm3a3v0nqswj0nql5pqqzhgetnwseqmkcq93/test2/1708411351353.jpeg
const normalizedSplitPath = ctx.path.replace(/^\//, '').split('/');
const [naddr, ...path] = SINGLE_SITE ? [SINGLE_SITE, ...normalizedSplitPath] : normalizedSplitPath;

4
src/utils.ts
View File

@ -7,5 +7,5 @@ export const appendSlashIfMissing = (path: string): string => {
};
export const removeLeadingSlashes = (path: string): string => {
return path.replace(/^\/+/, '');
}
return path.replace(/^\/+/, '');
};