feat: improved index.html lookup
This commit is contained in:
parent
d0185042c4
commit
42808a9c77
27
src/cdn.ts
27
src/cdn.ts
@ -1,28 +1,29 @@
|
||||
import debug from "debug";
|
||||
import type { IncomingMessage } from "http";
|
||||
import followRedirects from "follow-redirects";
|
||||
import uniq from "lodash/uniq.js";
|
||||
import debug from 'debug';
|
||||
import type { IncomingMessage } from 'http';
|
||||
import followRedirects from 'follow-redirects';
|
||||
import uniq from 'lodash/uniq.js';
|
||||
|
||||
const log = debug("web:discover:upstream");
|
||||
const log = debug('web:discover:upstream');
|
||||
|
||||
export async function searchCdn(servers: string[], hash: string) {
|
||||
const uniqueSevers = uniq(servers);
|
||||
log("Looking for", hash);
|
||||
|
||||
log('Looking for', hash);
|
||||
for (const server of uniqueSevers) {
|
||||
try {
|
||||
log("Checking CDN server", server);
|
||||
log('Checking CDN server', server);
|
||||
return await checkCDN(server, hash);
|
||||
} catch (e) {
|
||||
log("Ingoring error", e);
|
||||
log('Ingoring error', e);
|
||||
} // Ignore errors during lookup
|
||||
}
|
||||
log("No CDN with the content found.");
|
||||
log('No CDN with the content found.');
|
||||
}
|
||||
|
||||
async function checkCDN(cdn: string, hash: string): Promise<IncomingMessage> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(hash, cdn);
|
||||
const backend = url.protocol === "https:" ? followRedirects.https : followRedirects.http;
|
||||
const backend = url.protocol === 'https:' ? followRedirects.https : followRedirects.http;
|
||||
|
||||
var req = backend
|
||||
.get(url.toString(), (res) => {
|
||||
@ -32,11 +33,11 @@ async function checkCDN(cdn: string, hash: string): Promise<IncomingMessage> {
|
||||
reject(new Error(res.statusMessage));
|
||||
} else {
|
||||
resolve(res);
|
||||
log("Found", hash, "at", cdn);
|
||||
log('Found', hash, 'at', cdn);
|
||||
}
|
||||
})
|
||||
.on("error", function (error) {
|
||||
log("Ignoring error", error);
|
||||
.on('error', function (error) {
|
||||
log('Ignoring error', error);
|
||||
})
|
||||
.end();
|
||||
});
|
||||
|
79
src/drive.ts
79
src/drive.ts
@ -1,22 +1,27 @@
|
||||
import NDK, { NDKEvent, NDKKind } from "@nostr-dev-kit/ndk";
|
||||
import debug from "debug";
|
||||
import NodeCache from "node-cache";
|
||||
import { Drive } from "./types.js";
|
||||
import uniqBy from "lodash/uniqBy.js";
|
||||
import NDK, { NDKEvent, NDKKind, NDKRelaySet } from '@nostr-dev-kit/ndk';
|
||||
import debug from 'debug';
|
||||
import NodeCache from 'node-cache';
|
||||
import { Drive } from './types.js';
|
||||
import uniqBy from 'lodash/uniqBy.js';
|
||||
import { SINGLE_SITE } from './env.js';
|
||||
import { nip19 } from 'nostr-tools';
|
||||
import { AddressPointer } from 'nostr-tools/nip19';
|
||||
|
||||
const driveCache = new NodeCache({ stdTTL: 60 * 60 }); // 5min for development
|
||||
|
||||
const log = debug("web:drive:nostr");
|
||||
const log = debug('web:drive:nostr');
|
||||
|
||||
export const defaultRelays = [
|
||||
'wss://nostrue.com',
|
||||
'wss://relay.damus.io',
|
||||
//"wss://nostr.wine",
|
||||
'wss://nos.lol',
|
||||
// "wss://nostr-pub.wellorder.net",
|
||||
];
|
||||
|
||||
// TODO use relays from naddr
|
||||
const ndk = new NDK({
|
||||
explicitRelayUrls: [
|
||||
"wss://nostrue.com",
|
||||
"wss://relay.damus.io",
|
||||
//"wss://nostr.wine",
|
||||
"wss://nos.lol",
|
||||
// "wss://nostr-pub.wellorder.net",
|
||||
],
|
||||
explicitRelayUrls: defaultRelays,
|
||||
});
|
||||
|
||||
ndk.connect();
|
||||
@ -27,11 +32,11 @@ const handleEvent = (event: NDKEvent | null): Drive | undefined => {
|
||||
if (!event) return;
|
||||
const kind = event.kind;
|
||||
const pubkey = event.pubkey;
|
||||
const driveIdentifier = event.tags.find((t) => t[0] === "d")?.[1];
|
||||
const driveIdentifier = event.tags.find((t) => t[0] === 'd')?.[1];
|
||||
const driveKey = `${kind}-${pubkey}-${driveIdentifier}`;
|
||||
log(driveKey);
|
||||
|
||||
const treeTags = event.tags.filter((t) => t[0] === "x" || t[0] === "folder");
|
||||
const treeTags = event.tags.filter((t) => t[0] === 'x' || t[0] === 'folder');
|
||||
const files = treeTags.map((t) => ({
|
||||
hash: t[1],
|
||||
path: t[2],
|
||||
@ -40,7 +45,7 @@ const handleEvent = (event: NDKEvent | null): Drive | undefined => {
|
||||
}));
|
||||
// log(files);
|
||||
|
||||
const servers = event.tags.filter((t) => t[0] === "r" && t[1]).map((t) => new URL("/", t[1]).toString()) || [];
|
||||
const servers = event.tags.filter((t) => t[0] === 'r' && t[1]).map((t) => new URL('/', t[1]).toString()) || [];
|
||||
// log(servers);
|
||||
|
||||
const drive = { files, servers };
|
||||
@ -57,14 +62,14 @@ const fetchAllDrives = async () => {
|
||||
},
|
||||
{ closeOnEose: true },
|
||||
);
|
||||
sub.on("event", (event: NDKEvent) => {
|
||||
sub.on('event', (event: NDKEvent) => {
|
||||
const newEvents = sortedEvents
|
||||
.concat([event])
|
||||
.sort((a: NDKEvent, b: NDKEvent) => (b.created_at ?? 0) - (a.created_at ?? 0));
|
||||
sortedEvents = uniqBy(newEvents, (e: NDKEvent) => e.tagId());
|
||||
});
|
||||
sub.start();
|
||||
sub.on("close", () => {
|
||||
sub.on('close', () => {
|
||||
for (const event of sortedEvents) {
|
||||
handleEvent(event);
|
||||
}
|
||||
@ -72,32 +77,52 @@ const fetchAllDrives = async () => {
|
||||
setTimeout(() => sub.stop(), 10000);
|
||||
};
|
||||
|
||||
export const readDrive = async (kind: number, pubkey: string, driveIdentifier: string): Promise<Drive | undefined> => {
|
||||
export const readDrive = async (
|
||||
kind: number,
|
||||
pubkey: string,
|
||||
driveIdentifier: string,
|
||||
relays?: string[],
|
||||
): Promise<Drive | undefined> => {
|
||||
const driveKey = `${kind}-${pubkey}-${driveIdentifier}`;
|
||||
if (driveCache.has(driveKey)) {
|
||||
log("using cached drive data for " + driveKey);
|
||||
log('using cached drive data for ' + driveKey);
|
||||
return driveCache.get(driveKey);
|
||||
}
|
||||
|
||||
log("Fetching drive event.");
|
||||
log('Fetching drive event.');
|
||||
|
||||
const filter = {
|
||||
kinds: [kind],
|
||||
authors: [pubkey],
|
||||
"#d": [driveIdentifier], // name.toLowerCase().replaceAll(/\s/g, "-") || nanoid(8);
|
||||
'#d': [driveIdentifier], // name.toLowerCase().replaceAll(/\s/g, "-") || nanoid(8);
|
||||
};
|
||||
log(filter);
|
||||
const event = await ndk.fetchEvent(filter);
|
||||
log("fetch finsihed");
|
||||
const event = await ndk.fetchEvent(
|
||||
filter,
|
||||
{},
|
||||
NDKRelaySet.fromRelayUrls(relays?.concat(defaultRelays) || defaultRelays, ndk),
|
||||
);
|
||||
log('fetch finsihed');
|
||||
|
||||
if (event) {
|
||||
return handleEvent(event);
|
||||
} else {
|
||||
log("no drive event found.");
|
||||
log('no drive event found.');
|
||||
}
|
||||
};
|
||||
|
||||
// TODO add lastchange date, to fetch changes
|
||||
// Load all drives into cache and refresh periodically
|
||||
fetchAllDrives();
|
||||
setInterval(() => fetchAllDrives(), 60000);
|
||||
|
||||
if (SINGLE_SITE) {
|
||||
// Load the single drive into the cache
|
||||
const { pubkey, kind, identifier, relays } = nip19.decode(SINGLE_SITE).data as AddressPointer;
|
||||
const drive = await readDrive(kind, pubkey, identifier, relays);
|
||||
if (!drive) {
|
||||
console.error(`Drive ${SINGLE_SITE} not found.`);
|
||||
}
|
||||
} else {
|
||||
// Load all drives into the cache
|
||||
fetchAllDrives();
|
||||
setInterval(() => fetchAllDrives(), 60000);
|
||||
}
|
||||
|
2
src/env.ts
Normal file
2
src/env.ts
Normal file
@ -0,0 +1,2 @@
|
||||
export const SINGLE_SITE = process.env.SINGLE_SITE;
|
||||
export const PORT = process.env.PORT || 3010;
|
209
src/index.ts
209
src/index.ts
@ -1,33 +1,36 @@
|
||||
import "websocket-polyfill";
|
||||
import Koa, { HttpError } from "koa";
|
||||
import debug from "debug";
|
||||
import Router from "@koa/router";
|
||||
import { nip19 } from "nostr-tools";
|
||||
import { AddressPointer } from "nostr-tools/nip19";
|
||||
import { Drive, FileMeta } from "./types.js";
|
||||
import { readDrive } from "./drive.js";
|
||||
import { searchCdn } from "./cdn.js";
|
||||
import { PassThrough } from "stream";
|
||||
import fs from "node:fs";
|
||||
import nodePath from "path";
|
||||
import 'websocket-polyfill';
|
||||
import Koa, { HttpError } from 'koa';
|
||||
import debug from 'debug';
|
||||
import Router from '@koa/router';
|
||||
import { nip19 } from 'nostr-tools';
|
||||
import { AddressPointer } from 'nostr-tools/nip19';
|
||||
import { Drive, FileMeta } from './types.js';
|
||||
import { defaultRelays, readDrive } from './drive.js';
|
||||
import { searchCdn } from './cdn.js';
|
||||
import { PassThrough } from 'stream';
|
||||
import fs from 'node:fs';
|
||||
import nodePath from 'path';
|
||||
import { PORT, SINGLE_SITE } from './env.js';
|
||||
import { appendSlashIfMissing, prefixSlashIfMissing, removeLeadingSlashes } from './utils.js';
|
||||
|
||||
const log = debug("web");
|
||||
const log = debug('web');
|
||||
const app = new Koa();
|
||||
const router = new Router();
|
||||
|
||||
// const cacheAdapter = new NDKCacheAdapterDexie({ dbName: "ndk-cache" });
|
||||
const additionalServers = ["https://cdn.hzrd149.com", "https://cdn.satellite.earth"];
|
||||
const additionalServers = ['https://cdn.hzrd149.com', 'https://cdn.satellite.earth'];
|
||||
|
||||
const findFile = (drive: Drive, filePathToSearch: string): FileMeta | undefined => {
|
||||
return drive.files.find((f) => f.path == "/" + filePathToSearch);
|
||||
const searchPath = prefixSlashIfMissing(filePathToSearch);
|
||||
return drive.files.find((f) => f.path == searchPath);
|
||||
};
|
||||
|
||||
const findFolderContents = (drive: Drive, filePathToSearch: string): FileMeta[] => {
|
||||
// TODO filter name
|
||||
return drive.files.filter((f) => f.path.startsWith("/" + filePathToSearch));
|
||||
const searchPath = prefixSlashIfMissing(filePathToSearch);
|
||||
return drive.files.filter((f) => f.path.startsWith(searchPath));
|
||||
};
|
||||
|
||||
const cacheDir = "./cache";
|
||||
const cacheDir = './cache';
|
||||
if (!fs.existsSync(cacheDir)) {
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
}
|
||||
@ -40,21 +43,21 @@ app.use(async (ctx, next) => {
|
||||
if (err instanceof HttpError) {
|
||||
const status = (ctx.status = err.status || 500);
|
||||
if (status >= 500) console.error(err.stack);
|
||||
ctx.body = status > 500 ? { message: "Something went wrong" } : { message: err.message };
|
||||
ctx.body = status > 500 ? { message: 'Something went wrong' } : { message: err.message };
|
||||
} else {
|
||||
console.log(err);
|
||||
ctx.status = 500;
|
||||
ctx.body = { message: "Something went wrong" };
|
||||
ctx.body = { message: 'Something went wrong' };
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/health", async (ctx, next) => {
|
||||
router.get('/health', async (ctx, next) => {
|
||||
ctx.status = 200;
|
||||
});
|
||||
|
||||
const storeInCache = (src: NodeJS.ReadableStream, cacheFile: string) => {
|
||||
log("storing cache file:", cacheFile);
|
||||
const storeInCache = async (src: NodeJS.ReadableStream, cacheFile: string) => {
|
||||
log('storing cache file:', cacheFile);
|
||||
|
||||
const cacheFileStream = fs.createWriteStream(cacheFile);
|
||||
src.pipe(cacheFileStream);
|
||||
@ -63,17 +66,55 @@ const storeInCache = (src: NodeJS.ReadableStream, cacheFile: string) => {
|
||||
//stream.pipe(hash);
|
||||
|
||||
return new Promise<void>((res) => {
|
||||
src.on("end", async () => {
|
||||
log("cache file stored:", cacheFile);
|
||||
cacheFileStream.on('close', async () => {
|
||||
log('cache file stored:', cacheFile);
|
||||
res();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
router.get("(.*)", async (ctx, next) => {
|
||||
const serveStream = (ctx: Koa.ParameterizedContext, mimeType: string, src: NodeJS.ReadableStream) => {
|
||||
ctx.set({
|
||||
'Content-Type': mimeType,
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
});
|
||||
ctx.body = src;
|
||||
};
|
||||
|
||||
const serveDriveFile = async (ctx: Koa.ParameterizedContext, drive: Drive, fileMeta: FileMeta) => {
|
||||
const { hash, mimeType, size } = fileMeta;
|
||||
|
||||
const cacheFile = nodePath.join(cacheDir, hash);
|
||||
if (fs.existsSync(cacheFile)) {
|
||||
log(`returning cached data for ${hash}`);
|
||||
const src = fs.createReadStream(cacheFile);
|
||||
serveStream(ctx, mimeType, src);
|
||||
} else {
|
||||
// lookup media sevrers for user -> ndk (optional)
|
||||
const cdnSource = await searchCdn([...drive.servers, ...additionalServers], hash);
|
||||
|
||||
if (cdnSource) {
|
||||
if (size < 100000) {
|
||||
// if small file < 100KB, download and serve downloaded file
|
||||
await storeInCache(cdnSource, cacheFile);
|
||||
serveStream(ctx, mimeType, fs.createReadStream(cacheFile));
|
||||
} else {
|
||||
// else ()>100kb) stream content from backend.
|
||||
// TODO or maybe redirect????
|
||||
log(`streaming content ${hash} ...`);
|
||||
serveStream(ctx, mimeType, cdnSource);
|
||||
}
|
||||
} else {
|
||||
log('no CDN server found for blob: ' + hash);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
router.get('(.*)', async (ctx, next) => {
|
||||
log(`serving: host:${ctx.hostname} path:${ctx.path}`);
|
||||
|
||||
if (ctx.path == "/favicon.ico") {
|
||||
if (ctx.path == '/favicon.ico') {
|
||||
return next();
|
||||
}
|
||||
|
||||
@ -81,110 +122,72 @@ router.get("(.*)", async (ctx, next) => {
|
||||
// test2
|
||||
// 1708411351353.jpeg
|
||||
// http://localhost:3010/naddr1qvzqqqrhvvpzpd7x76g4e756vtlldg0syczdazxz83kxcmgm3a3v0nqswj0nql5pqqzhgetnwseqmkcq93/test2/1708411351353.jpeg
|
||||
const normalizedSplitPath = ctx.path.replace(/^\//, '').split('/');
|
||||
|
||||
const [naddr, ...path] = ctx.path.replace(/^\//, "").split("/");
|
||||
if (!naddr || !naddr.startsWith("naddr")) {
|
||||
const [naddr, ...path] = SINGLE_SITE ? [SINGLE_SITE, ...normalizedSplitPath] : normalizedSplitPath;
|
||||
if (!naddr || !naddr.startsWith('naddr')) {
|
||||
ctx.status = 400;
|
||||
ctx.message = "A naddr needs to be specific in the url.";
|
||||
ctx.message = 'A naddr needs to be specific in the url.';
|
||||
return;
|
||||
}
|
||||
|
||||
let searchPath = decodeURI(path.join("/"));
|
||||
let searchPath = decodeURI(path.join('/'));
|
||||
log(`naddr: ${naddr} path: ${searchPath}`);
|
||||
|
||||
/* TODO try lookup of index.html and fallback to list, if not found
|
||||
if (searchPath.endsWith("/")) {
|
||||
searchPath = searchPath + "index.html";
|
||||
}
|
||||
if (searchPath == "") {
|
||||
searchPath = searchPath + "/index.html";
|
||||
}*/
|
||||
|
||||
// decode naddr
|
||||
const { pubkey, kind, identifier } = nip19.decode(naddr).data as AddressPointer;
|
||||
const { pubkey, kind, identifier, relays } = nip19.decode(naddr).data as AddressPointer;
|
||||
|
||||
// fetch drive descriptor -> ndk
|
||||
const drive = await readDrive(kind, pubkey, identifier);
|
||||
const drive = await readDrive(kind, pubkey, identifier, relays);
|
||||
// log(`drive: ${drive}`);
|
||||
|
||||
if (drive) {
|
||||
// lookup file in file-list
|
||||
const fileMeta = findFile(drive, searchPath);
|
||||
// log(`fileMeta: ${fileMeta}`);
|
||||
const candidates = [searchPath];
|
||||
|
||||
if (fileMeta) {
|
||||
const { hash, mimeType, size } = fileMeta;
|
||||
|
||||
const cacheFile = nodePath.join(cacheDir, hash);
|
||||
if (fs.existsSync(cacheFile)) {
|
||||
log(`returning cached data for ${hash}`);
|
||||
ctx.set({
|
||||
"Content-Type": mimeType,
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
});
|
||||
const src = fs.createReadStream(cacheFile);
|
||||
ctx.body = src;
|
||||
} else {
|
||||
// lookup media sevrers for user -> ndk (optional)
|
||||
const cdnSource = await searchCdn([...drive.servers, ...additionalServers], hash);
|
||||
|
||||
//log(cdnSource);
|
||||
|
||||
if (cdnSource) {
|
||||
if (size < 100000) {
|
||||
// if small file < 100KB, download and serve downloaded file
|
||||
await storeInCache(cdnSource, cacheFile);
|
||||
ctx.set({
|
||||
"Content-Type": mimeType,
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
});
|
||||
ctx.body = fs.createReadStream(cacheFile);
|
||||
} else {
|
||||
// else ()>100kb) stream content from backend.
|
||||
// TODO or maybe redirect????
|
||||
ctx.set({
|
||||
"Content-Type": mimeType,
|
||||
"Cache-Control": "no-cache",
|
||||
Connection: "keep-alive",
|
||||
});
|
||||
ctx.body = cdnSource;
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
log("no CDN server found for blob: " + hash);
|
||||
}
|
||||
let found = false;
|
||||
for (let i = 0; i < candidates.length; i++) {
|
||||
const fileMeta = findFile(drive, candidates[i]);
|
||||
if (fileMeta) {
|
||||
await serveDriveFile(ctx, drive, fileMeta);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
const basePath = SINGLE_SITE ? '' : '/' + naddr;
|
||||
|
||||
if (findFile(drive, appendSlashIfMissing(searchPath) + 'index.html')) {
|
||||
ctx.redirect(
|
||||
appendSlashIfMissing(basePath) + removeLeadingSlashes(appendSlashIfMissing(searchPath)) + 'index.html',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const folder = findFolderContents(drive, searchPath);
|
||||
log("file not found in drive: " + searchPath);
|
||||
log('file not found in drive: ' + searchPath);
|
||||
ctx.status = 404;
|
||||
ctx.set("Content-Type", "text/html");
|
||||
const folderList = folder.map((f) => `<li><a href="${encodeURI("/" + naddr + f.path)}">${f.path}</a></li>`);
|
||||
ctx.body = `<html><body><h2>Index of ${searchPath || "/"}</h2><ul>${folderList.join("")}</ul></body></html>`;
|
||||
ctx.set('Content-Type', 'text/html');
|
||||
const folderList = folder.map((f) => `<li><a href="${encodeURI(basePath + f.path)}">${f.path}</a></li>`);
|
||||
ctx.body = `<html><body><h2>Index of ${searchPath || '/'}</h2><ul>${folderList.join('')}</ul></body></html>`;
|
||||
}
|
||||
}
|
||||
|
||||
// fetch file from mediaserver and return to the caller
|
||||
});
|
||||
|
||||
app.use(router.routes()).use(router.allowedMethods());
|
||||
|
||||
const port = process.env.PORT || 3010;
|
||||
log("Starting blossom-drive-webserver on port " + port);
|
||||
app.listen(port);
|
||||
log('Starting blossom-drive-webserver on port ' + PORT);
|
||||
app.listen(PORT);
|
||||
|
||||
async function shutdown() {
|
||||
log("Shutting down blossom-drive-webserver...");
|
||||
log('Shutting down blossom-drive-webserver...');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// add this handler before emitting any events
|
||||
process.on("uncaughtException", function (err) {
|
||||
console.error("UNCAUGHT EXCEPTION - keeping process alive:", err);
|
||||
process.on('uncaughtException', function (err) {
|
||||
console.error('UNCAUGHT EXCEPTION - keeping process alive:', err);
|
||||
});
|
||||
|
||||
process.addListener("SIGTERM", shutdown);
|
||||
process.addListener("SIGINT", shutdown);
|
||||
process.addListener('SIGTERM', shutdown);
|
||||
process.addListener('SIGINT', shutdown);
|
||||
|
21
src/types.ts
21
src/types.ts
@ -1,12 +1,11 @@
|
||||
|
||||
export type FileMeta = {
|
||||
hash: string;
|
||||
path: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
export type Drive = {
|
||||
servers: string[];
|
||||
files: FileMeta[];
|
||||
};
|
||||
hash: string;
|
||||
path: string;
|
||||
size: number;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
export type Drive = {
|
||||
servers: string[];
|
||||
files: FileMeta[];
|
||||
};
|
||||
|
11
src/utils.ts
Normal file
11
src/utils.ts
Normal file
@ -0,0 +1,11 @@
|
||||
export const prefixSlashIfMissing = (path: string): string => {
|
||||
return path.startsWith('/') ? path : '/' + path;
|
||||
};
|
||||
|
||||
export const appendSlashIfMissing = (path: string): string => {
|
||||
return path.endsWith('/') ? path : path + '/';
|
||||
};
|
||||
|
||||
export const removeLeadingSlashes = (path: string): string => {
|
||||
return path.replace(/^\/+/, '');
|
||||
}
|
Loading…
Reference in New Issue
Block a user