feat: improved index.html lookup

This commit is contained in:
florian 2024-03-18 19:58:40 +01:00
parent d0185042c4
commit 42808a9c77
6 changed files with 195 additions and 154 deletions

View File

@ -1,28 +1,29 @@
import debug from "debug";
import type { IncomingMessage } from "http";
import followRedirects from "follow-redirects";
import uniq from "lodash/uniq.js";
import debug from 'debug';
import type { IncomingMessage } from 'http';
import followRedirects from 'follow-redirects';
import uniq from 'lodash/uniq.js';
const log = debug("web:discover:upstream");
const log = debug('web:discover:upstream');
export async function searchCdn(servers: string[], hash: string) {
const uniqueSevers = uniq(servers);
log("Looking for", hash);
log('Looking for', hash);
for (const server of uniqueSevers) {
try {
log("Checking CDN server", server);
log('Checking CDN server', server);
return await checkCDN(server, hash);
} catch (e) {
log("Ingoring error", e);
log('Ingoring error', e);
} // Ignore errors during lookup
}
log("No CDN with the content found.");
log('No CDN with the content found.');
}
async function checkCDN(cdn: string, hash: string): Promise<IncomingMessage> {
return new Promise((resolve, reject) => {
const url = new URL(hash, cdn);
const backend = url.protocol === "https:" ? followRedirects.https : followRedirects.http;
const backend = url.protocol === 'https:' ? followRedirects.https : followRedirects.http;
var req = backend
.get(url.toString(), (res) => {
@ -32,11 +33,11 @@ async function checkCDN(cdn: string, hash: string): Promise<IncomingMessage> {
reject(new Error(res.statusMessage));
} else {
resolve(res);
log("Found", hash, "at", cdn);
log('Found', hash, 'at', cdn);
}
})
.on("error", function (error) {
log("Ignoring error", error);
.on('error', function (error) {
log('Ignoring error', error);
})
.end();
});

View File

@ -1,22 +1,27 @@
import NDK, { NDKEvent, NDKKind } from "@nostr-dev-kit/ndk";
import debug from "debug";
import NodeCache from "node-cache";
import { Drive } from "./types.js";
import uniqBy from "lodash/uniqBy.js";
import NDK, { NDKEvent, NDKKind, NDKRelaySet } from '@nostr-dev-kit/ndk';
import debug from 'debug';
import NodeCache from 'node-cache';
import { Drive } from './types.js';
import uniqBy from 'lodash/uniqBy.js';
import { SINGLE_SITE } from './env.js';
import { nip19 } from 'nostr-tools';
import { AddressPointer } from 'nostr-tools/nip19';
const driveCache = new NodeCache({ stdTTL: 60 * 60 }); // 5min for development
const log = debug("web:drive:nostr");
const log = debug('web:drive:nostr');
export const defaultRelays = [
'wss://nostrue.com',
'wss://relay.damus.io',
//"wss://nostr.wine",
'wss://nos.lol',
// "wss://nostr-pub.wellorder.net",
];
// TODO use relays from naddr
const ndk = new NDK({
explicitRelayUrls: [
"wss://nostrue.com",
"wss://relay.damus.io",
//"wss://nostr.wine",
"wss://nos.lol",
// "wss://nostr-pub.wellorder.net",
],
explicitRelayUrls: defaultRelays,
});
ndk.connect();
@ -27,11 +32,11 @@ const handleEvent = (event: NDKEvent | null): Drive | undefined => {
if (!event) return;
const kind = event.kind;
const pubkey = event.pubkey;
const driveIdentifier = event.tags.find((t) => t[0] === "d")?.[1];
const driveIdentifier = event.tags.find((t) => t[0] === 'd')?.[1];
const driveKey = `${kind}-${pubkey}-${driveIdentifier}`;
log(driveKey);
const treeTags = event.tags.filter((t) => t[0] === "x" || t[0] === "folder");
const treeTags = event.tags.filter((t) => t[0] === 'x' || t[0] === 'folder');
const files = treeTags.map((t) => ({
hash: t[1],
path: t[2],
@ -40,7 +45,7 @@ const handleEvent = (event: NDKEvent | null): Drive | undefined => {
}));
// log(files);
const servers = event.tags.filter((t) => t[0] === "r" && t[1]).map((t) => new URL("/", t[1]).toString()) || [];
const servers = event.tags.filter((t) => t[0] === 'r' && t[1]).map((t) => new URL('/', t[1]).toString()) || [];
// log(servers);
const drive = { files, servers };
@ -57,14 +62,14 @@ const fetchAllDrives = async () => {
},
{ closeOnEose: true },
);
sub.on("event", (event: NDKEvent) => {
sub.on('event', (event: NDKEvent) => {
const newEvents = sortedEvents
.concat([event])
.sort((a: NDKEvent, b: NDKEvent) => (b.created_at ?? 0) - (a.created_at ?? 0));
sortedEvents = uniqBy(newEvents, (e: NDKEvent) => e.tagId());
});
sub.start();
sub.on("close", () => {
sub.on('close', () => {
for (const event of sortedEvents) {
handleEvent(event);
}
@ -72,32 +77,52 @@ const fetchAllDrives = async () => {
setTimeout(() => sub.stop(), 10000);
};
export const readDrive = async (kind: number, pubkey: string, driveIdentifier: string): Promise<Drive | undefined> => {
export const readDrive = async (
kind: number,
pubkey: string,
driveIdentifier: string,
relays?: string[],
): Promise<Drive | undefined> => {
const driveKey = `${kind}-${pubkey}-${driveIdentifier}`;
if (driveCache.has(driveKey)) {
log("using cached drive data for " + driveKey);
log('using cached drive data for ' + driveKey);
return driveCache.get(driveKey);
}
log("Fetching drive event.");
log('Fetching drive event.');
const filter = {
kinds: [kind],
authors: [pubkey],
"#d": [driveIdentifier], // name.toLowerCase().replaceAll(/\s/g, "-") || nanoid(8);
'#d': [driveIdentifier], // name.toLowerCase().replaceAll(/\s/g, "-") || nanoid(8);
};
log(filter);
const event = await ndk.fetchEvent(filter);
log("fetch finsihed");
const event = await ndk.fetchEvent(
filter,
{},
NDKRelaySet.fromRelayUrls(relays?.concat(defaultRelays) || defaultRelays, ndk),
);
log('fetch finsihed');
if (event) {
return handleEvent(event);
} else {
log("no drive event found.");
log('no drive event found.');
}
};
// TODO add lastchange date, to fetch changes
// Load all drives into cache and refresh periodically
fetchAllDrives();
setInterval(() => fetchAllDrives(), 60000);
if (SINGLE_SITE) {
// Load the single drive into the cache
const { pubkey, kind, identifier, relays } = nip19.decode(SINGLE_SITE).data as AddressPointer;
const drive = await readDrive(kind, pubkey, identifier, relays);
if (!drive) {
console.error(`Drive ${SINGLE_SITE} not found.`);
}
} else {
// Load all drives into the cache
fetchAllDrives();
setInterval(() => fetchAllDrives(), 60000);
}

2
src/env.ts Normal file
View File

@ -0,0 +1,2 @@
export const SINGLE_SITE = process.env.SINGLE_SITE;
export const PORT = process.env.PORT || 3010;

View File

@ -1,33 +1,36 @@
import "websocket-polyfill";
import Koa, { HttpError } from "koa";
import debug from "debug";
import Router from "@koa/router";
import { nip19 } from "nostr-tools";
import { AddressPointer } from "nostr-tools/nip19";
import { Drive, FileMeta } from "./types.js";
import { readDrive } from "./drive.js";
import { searchCdn } from "./cdn.js";
import { PassThrough } from "stream";
import fs from "node:fs";
import nodePath from "path";
import 'websocket-polyfill';
import Koa, { HttpError } from 'koa';
import debug from 'debug';
import Router from '@koa/router';
import { nip19 } from 'nostr-tools';
import { AddressPointer } from 'nostr-tools/nip19';
import { Drive, FileMeta } from './types.js';
import { defaultRelays, readDrive } from './drive.js';
import { searchCdn } from './cdn.js';
import { PassThrough } from 'stream';
import fs from 'node:fs';
import nodePath from 'path';
import { PORT, SINGLE_SITE } from './env.js';
import { appendSlashIfMissing, prefixSlashIfMissing, removeLeadingSlashes } from './utils.js';
const log = debug("web");
const log = debug('web');
const app = new Koa();
const router = new Router();
// const cacheAdapter = new NDKCacheAdapterDexie({ dbName: "ndk-cache" });
const additionalServers = ["https://cdn.hzrd149.com", "https://cdn.satellite.earth"];
const additionalServers = ['https://cdn.hzrd149.com', 'https://cdn.satellite.earth'];
const findFile = (drive: Drive, filePathToSearch: string): FileMeta | undefined => {
return drive.files.find((f) => f.path == "/" + filePathToSearch);
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.find((f) => f.path == searchPath);
};
const findFolderContents = (drive: Drive, filePathToSearch: string): FileMeta[] => {
// TODO filter name
return drive.files.filter((f) => f.path.startsWith("/" + filePathToSearch));
const searchPath = prefixSlashIfMissing(filePathToSearch);
return drive.files.filter((f) => f.path.startsWith(searchPath));
};
const cacheDir = "./cache";
const cacheDir = './cache';
if (!fs.existsSync(cacheDir)) {
fs.mkdirSync(cacheDir, { recursive: true });
}
@ -40,21 +43,21 @@ app.use(async (ctx, next) => {
if (err instanceof HttpError) {
const status = (ctx.status = err.status || 500);
if (status >= 500) console.error(err.stack);
ctx.body = status > 500 ? { message: "Something went wrong" } : { message: err.message };
ctx.body = status > 500 ? { message: 'Something went wrong' } : { message: err.message };
} else {
console.log(err);
ctx.status = 500;
ctx.body = { message: "Something went wrong" };
ctx.body = { message: 'Something went wrong' };
}
}
});
router.get("/health", async (ctx, next) => {
router.get('/health', async (ctx, next) => {
ctx.status = 200;
});
const storeInCache = (src: NodeJS.ReadableStream, cacheFile: string) => {
log("storing cache file:", cacheFile);
const storeInCache = async (src: NodeJS.ReadableStream, cacheFile: string) => {
log('storing cache file:', cacheFile);
const cacheFileStream = fs.createWriteStream(cacheFile);
src.pipe(cacheFileStream);
@ -63,17 +66,55 @@ const storeInCache = (src: NodeJS.ReadableStream, cacheFile: string) => {
//stream.pipe(hash);
return new Promise<void>((res) => {
src.on("end", async () => {
log("cache file stored:", cacheFile);
cacheFileStream.on('close', async () => {
log('cache file stored:', cacheFile);
res();
});
});
};
router.get("(.*)", async (ctx, next) => {
const serveStream = (ctx: Koa.ParameterizedContext, mimeType: string, src: NodeJS.ReadableStream) => {
ctx.set({
'Content-Type': mimeType,
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
});
ctx.body = src;
};
const serveDriveFile = async (ctx: Koa.ParameterizedContext, drive: Drive, fileMeta: FileMeta) => {
const { hash, mimeType, size } = fileMeta;
const cacheFile = nodePath.join(cacheDir, hash);
if (fs.existsSync(cacheFile)) {
log(`returning cached data for ${hash}`);
const src = fs.createReadStream(cacheFile);
serveStream(ctx, mimeType, src);
} else {
// lookup media sevrers for user -> ndk (optional)
const cdnSource = await searchCdn([...drive.servers, ...additionalServers], hash);
if (cdnSource) {
if (size < 100000) {
// if small file < 100KB, download and serve downloaded file
await storeInCache(cdnSource, cacheFile);
serveStream(ctx, mimeType, fs.createReadStream(cacheFile));
} else {
// else ()>100kb) stream content from backend.
// TODO or maybe redirect????
log(`streaming content ${hash} ...`);
serveStream(ctx, mimeType, cdnSource);
}
} else {
log('no CDN server found for blob: ' + hash);
}
}
};
router.get('(.*)', async (ctx, next) => {
log(`serving: host:${ctx.hostname} path:${ctx.path}`);
if (ctx.path == "/favicon.ico") {
if (ctx.path == '/favicon.ico') {
return next();
}
@ -81,110 +122,72 @@ router.get("(.*)", async (ctx, next) => {
// test2
// 1708411351353.jpeg
// http://localhost:3010/naddr1qvzqqqrhvvpzpd7x76g4e756vtlldg0syczdazxz83kxcmgm3a3v0nqswj0nql5pqqzhgetnwseqmkcq93/test2/1708411351353.jpeg
const normalizedSplitPath = ctx.path.replace(/^\//, '').split('/');
const [naddr, ...path] = ctx.path.replace(/^\//, "").split("/");
if (!naddr || !naddr.startsWith("naddr")) {
const [naddr, ...path] = SINGLE_SITE ? [SINGLE_SITE, ...normalizedSplitPath] : normalizedSplitPath;
if (!naddr || !naddr.startsWith('naddr')) {
ctx.status = 400;
ctx.message = "A naddr needs to be specific in the url.";
ctx.message = 'A naddr needs to be specific in the url.';
return;
}
let searchPath = decodeURI(path.join("/"));
let searchPath = decodeURI(path.join('/'));
log(`naddr: ${naddr} path: ${searchPath}`);
/* TODO try lookup of index.html and fallback to list, if not found
if (searchPath.endsWith("/")) {
searchPath = searchPath + "index.html";
}
if (searchPath == "") {
searchPath = searchPath + "/index.html";
}*/
// decode naddr
const { pubkey, kind, identifier } = nip19.decode(naddr).data as AddressPointer;
const { pubkey, kind, identifier, relays } = nip19.decode(naddr).data as AddressPointer;
// fetch drive descriptor -> ndk
const drive = await readDrive(kind, pubkey, identifier);
const drive = await readDrive(kind, pubkey, identifier, relays);
// log(`drive: ${drive}`);
if (drive) {
// lookup file in file-list
const fileMeta = findFile(drive, searchPath);
// log(`fileMeta: ${fileMeta}`);
const candidates = [searchPath];
if (fileMeta) {
const { hash, mimeType, size } = fileMeta;
const cacheFile = nodePath.join(cacheDir, hash);
if (fs.existsSync(cacheFile)) {
log(`returning cached data for ${hash}`);
ctx.set({
"Content-Type": mimeType,
"Cache-Control": "no-cache",
Connection: "keep-alive",
});
const src = fs.createReadStream(cacheFile);
ctx.body = src;
} else {
// lookup media sevrers for user -> ndk (optional)
const cdnSource = await searchCdn([...drive.servers, ...additionalServers], hash);
//log(cdnSource);
if (cdnSource) {
if (size < 100000) {
// if small file < 100KB, download and serve downloaded file
await storeInCache(cdnSource, cacheFile);
ctx.set({
"Content-Type": mimeType,
"Cache-Control": "no-cache",
Connection: "keep-alive",
});
ctx.body = fs.createReadStream(cacheFile);
} else {
// else ()>100kb) stream content from backend.
// TODO or maybe redirect????
ctx.set({
"Content-Type": mimeType,
"Cache-Control": "no-cache",
Connection: "keep-alive",
});
ctx.body = cdnSource;
}
} else {
log("no CDN server found for blob: " + hash);
}
let found = false;
for (let i = 0; i < candidates.length; i++) {
const fileMeta = findFile(drive, candidates[i]);
if (fileMeta) {
await serveDriveFile(ctx, drive, fileMeta);
found = true;
break;
}
} else {
}
if (!found) {
const basePath = SINGLE_SITE ? '' : '/' + naddr;
if (findFile(drive, appendSlashIfMissing(searchPath) + 'index.html')) {
ctx.redirect(
appendSlashIfMissing(basePath) + removeLeadingSlashes(appendSlashIfMissing(searchPath)) + 'index.html',
);
return;
}
const folder = findFolderContents(drive, searchPath);
log("file not found in drive: " + searchPath);
log('file not found in drive: ' + searchPath);
ctx.status = 404;
ctx.set("Content-Type", "text/html");
const folderList = folder.map((f) => `<li><a href="${encodeURI("/" + naddr + f.path)}">${f.path}</a></li>`);
ctx.body = `<html><body><h2>Index of ${searchPath || "/"}</h2><ul>${folderList.join("")}</ul></body></html>`;
ctx.set('Content-Type', 'text/html');
const folderList = folder.map((f) => `<li><a href="${encodeURI(basePath + f.path)}">${f.path}</a></li>`);
ctx.body = `<html><body><h2>Index of ${searchPath || '/'}</h2><ul>${folderList.join('')}</ul></body></html>`;
}
}
// fetch file from mediaserver and return to the caller
});
app.use(router.routes()).use(router.allowedMethods());
const port = process.env.PORT || 3010;
log("Starting blossom-drive-webserver on port " + port);
app.listen(port);
log('Starting blossom-drive-webserver on port ' + PORT);
app.listen(PORT);
async function shutdown() {
log("Shutting down blossom-drive-webserver...");
log('Shutting down blossom-drive-webserver...');
process.exit(0);
}
// add this handler before emitting any events
process.on("uncaughtException", function (err) {
console.error("UNCAUGHT EXCEPTION - keeping process alive:", err);
process.on('uncaughtException', function (err) {
console.error('UNCAUGHT EXCEPTION - keeping process alive:', err);
});
process.addListener("SIGTERM", shutdown);
process.addListener("SIGINT", shutdown);
process.addListener('SIGTERM', shutdown);
process.addListener('SIGINT', shutdown);

View File

@ -1,12 +1,11 @@
export type FileMeta = {
hash: string;
path: string;
size: number;
mimeType: string;
};
export type Drive = {
servers: string[];
files: FileMeta[];
};
hash: string;
path: string;
size: number;
mimeType: string;
};
export type Drive = {
servers: string[];
files: FileMeta[];
};

11
src/utils.ts Normal file
View File

@ -0,0 +1,11 @@
export const prefixSlashIfMissing = (path: string): string => {
return path.startsWith('/') ? path : '/' + path;
};
export const appendSlashIfMissing = (path: string): string => {
return path.endsWith('/') ? path : path + '/';
};
export const removeLeadingSlashes = (path: string): string => {
return path.replace(/^\/+/, '');
}