From c1c8486294fa39278402c910a2c5430e3d3d2286 Mon Sep 17 00:00:00 2001 From: florian <> Date: Sun, 23 Jun 2024 19:41:19 +0200 Subject: [PATCH] feat: Added redirect to cdn url, mirror feature --- src/index.ts | 247 ++++++++++++++++++++++++++++++++++--------- src/types.ts | 1 + wrangler.toml.sample | 5 + 3 files changed, 202 insertions(+), 51 deletions(-) diff --git a/src/index.ts b/src/index.ts index 7f7c5d3..297df66 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,49 @@ app.get('/', (c) => ) ); +/* +app.get('/migrate', async (c) => { + try { + const prefix = 'blob:'; + const kvList = await c.env.KV_BLOSSOM.list({ prefix, limit: 100 }); + let copyCount = 0; + + for (const entry of kvList.keys) { + const key = entry.name; + const targetHash = key.replace(prefix, ''); + const blobLookup = await c.env.BLOSSOM_BUCKET.head(targetHash); + const sourceId = await c.env.KV_BLOSSOM.get(key); + + if (sourceId) { + let deleteAfter = true; + if (!blobLookup) { + const blobContent = await c.env.BLOSSOM_BUCKET.get(sourceId); + if (blobContent) { + const contentBuffer = await blobContent.arrayBuffer(); + try { + await c.env.BLOSSOM_BUCKET.put(targetHash, contentBuffer); + copyCount++; + } catch (e) { + deleteAfter = false; + console.warn('Could not copy blob ' + sourceId); + } + } + } + + if (deleteAfter) { + await c.env.BLOSSOM_BUCKET.delete(sourceId); + await c.env.KV_BLOSSOM.delete(key); + } + } + } + + return c.json({ message: `${copyCount} Blobs processed successfull: ` + JSON.stringify(kvList.keys) }); + } catch (error: any) { + return c.json({ error: error.message }, 500); + } +}); +*/ + app.get('/list/:pubkey', async (c) => { const auth = getAuth(c.req.header('authorization') as string); await checkAuth(auth, 'list', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM); @@ -56,7 +99,110 @@ app.get('/list/:pubkey', async (c) => { return await c.json(listOfBlobs); }); -app.get('/list', cache({ cacheName: 'blossom', cacheControl: 'max-age=60' })); +app.get('/list', cache({ cacheName: 'blossom5', cacheControl: 'max-age=60' })); + +app.put('/mirror', async (c) => { + const auth = getAuth(c.req.header('authorization') as string); + await checkAuth(auth, 'upload', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM); + + const { url } = await c.req.json<{ url: string }>(); + if (!url) { + throw new HTTPException(400, { message: 'No source url in body.' }); + } + + if (!auth.objectHash) { + throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' }); + } + + const { pubkey } = auth.event; + let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash); + + if (!existing) { + console.log('not existing, trying to download ' + url); + + try { + // Fetch the file from the URL + const response = await fetch(url); + if (!response.ok) { + throw new HTTPException(500, { message: `Failed to fetch the file: ${response.statusText}` }); + } + + // Ensure the Content-Length header is present + const contentLength = response.headers.get('Content-Length'); + if (!contentLength) { + throw new Error('Content-Length header is required to determine the file size'); + } + + // Create a writable stream to R2 + const { writable, readable } = new FixedLengthStream(parseInt(contentLength, 10)); + const writer = writable.getWriter(); + + // Create a DigestStream for SHA-256 hashing + const digestStream = new crypto.DigestStream('SHA-256'); + const hashWriter = digestStream.getWriter(); + + // Stream the response directly to R2 and hash + const uploadPromise = c.env.BLOSSOM_BUCKET.put(auth.objectHash, readable, { + httpMetadata: response.headers, + customMetadata: { + pubkey, + }, + }); + + const reader = response.body?.getReader(); + if (!reader) { + throw new HTTPException(500, { message: 'Failed to get reader from response body' }); + } + + const pump = async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + // Write data to both R2 writer and hash writer + await writer.write(value); + await hashWriter.write(value); + } + await writer.close(); + await hashWriter.close(); + }; + + const [_, storedObject] = await Promise.all([pump(), uploadPromise, digestStream]); + + // Get the hash as a hex string + const hashBuffer = await digestStream.digest; + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join(''); + + if (hashHex != auth.objectHash) { + await c.env.BLOSSOM_BUCKET.delete(auth.objectHash); + throw new HTTPException(500, { message: `File content does not match the authenticated sha256 hash.` }); + } + + existing = storedObject; + } catch (error: any) { + throw new HTTPException(500, { message: `Error while mirroring url: ${error.message}` }); + } + } + + if (!existing) { + throw new HTTPException(500, { message: `Mirroring of url failed.` }); + } + + // Record the blob descriptors for the list operation + const blobData: BlobData = { + size: existing.size, + type: existing.httpMetadata?.contentType, + created: dayjs(existing.uploaded).unix(), + }; + await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData)); + + c.status(200); + return c.json({ + ...blobData, + url: c.env.PUBLIC_URL + '/' + existing.key, + sha256: existing.key, + }); +}); app.put('/upload', async (c) => { const auth = getAuth(c.req.header('authorization') as string); @@ -69,48 +215,53 @@ app.put('/upload', async (c) => { // TODO other npub upload an existing file // TODO same npub uploads existing file - const { pubkey } = auth.event; - const blobId = crypto.randomUUID(); - const digestStream = new crypto.DigestStream('SHA-256'); - const [stream1, stream2] = c.req.raw.body.tee(); - - const [_, storedObject] = await Promise.all([ - stream1.pipeTo(digestStream as WritableStream), - c.env.BLOSSOM_BUCKET.put(blobId, stream2 as ReadableStream, { - httpMetadata: c.req.raw.headers, - customMetadata: { - pubkey, - }, - }), - ]); - - const hash = Array.from(new Uint8Array(await digestStream.digest)) - .map((b) => b.toString(16).padStart(2, '0')) - .join(''); - - if (auth.objectHash) { - // currently not all clients support x tag in auth - if (auth.objectHash.toLocaleLowerCase() !== hash.toLocaleLowerCase()) { - c.env.BLOSSOM_BUCKET.delete(blobId); // delete the blob if hash does not match - throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' }); - } + if (!auth.objectHash) { + throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' }); } - await c.env.KV_BLOSSOM.put('blob:' + hash, blobId); // R2 IDs for each blob hash + const { pubkey } = auth.event; + let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash); + + if (!existing) { + const digestStream = new crypto.DigestStream('SHA-256'); + const [stream1, stream2] = c.req.raw.body.tee(); + + const [_, storedObject] = await Promise.all([ + stream1.pipeTo(digestStream as WritableStream), + c.env.BLOSSOM_BUCKET.put(auth.objectHash, stream2 as ReadableStream, { + httpMetadata: c.req.raw.headers, + customMetadata: { + pubkey, + }, + }), + ]); + + const storedHash = Array.from(new Uint8Array(await digestStream.digest)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); + + if (auth.objectHash.toLocaleLowerCase() !== storedHash.toLocaleLowerCase()) { + // Undo upload when the hash doesn't match + c.env.BLOSSOM_BUCKET.delete(auth.objectHash); + throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' }); + } + + existing = storedObject; + } // Record the blob descriptors for the list operation const blobData: BlobData = { - size: storedObject.size, - type: storedObject.httpMetadata?.contentType, - created: dayjs(storedObject.uploaded).unix(), + size: existing.size, + type: existing.httpMetadata?.contentType, + created: dayjs(existing.uploaded).unix(), }; - await c.env.KV_BLOSSOM.put(pubkey + ':' + hash, JSON.stringify(blobData)); + await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData)); c.status(200); return c.json({ ...blobData, - url: c.env.PUBLIC_URL + '/' + hash, - sha256: hash, + url: c.env.PUBLIC_URL + '/' + existing.key, + sha256: existing.key, }); }); @@ -133,17 +284,9 @@ app.delete('*', async (c) => { throw new HTTPException(403, { message: 'Not allowed to delete the blob.' }); } - const blobKey = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 }); - if (!blobKey) { - throw new HTTPException(404, { message: `Blob with the hash ${hash} not found.` }); - } - try { - await Promise.allSettled([ - c.env.BLOSSOM_BUCKET.delete(blobKey), - c.env.KV_BLOSSOM.delete('blob:' + hash), - c.env.KV_BLOSSOM.delete(pubkey + ':' + hash), - ]); + // TODO only delete the blob when no other npub is using it + await Promise.allSettled([c.env.BLOSSOM_BUCKET.delete(hash), c.env.KV_BLOSSOM.delete(pubkey + ':' + hash)]); return c.body(null, 204); } catch (e) { throw new HTTPException(404, { message: 'Blob not found.' }); @@ -151,15 +294,17 @@ app.delete('*', async (c) => { }); const getCache = cache({ - cacheName: 'blossom4', + cacheName: 'blossom5', cacheControl: 'max-age=604800', vary: ['Access-Control-Request-Headers', 'Accept-Encoding', 'Content-Range'], }); app.use(async (c, next: Next) => { - if (c.req.method == 'GET') return getCache(c, next); + // Only use cache for get requests and + // and when we don't have a CDN_PUBLIC_URL to redirect to + if (c.req.method == 'GET' && !c.env.CDN_PUBLIC_URL) return getCache(c, next); - // HEAD, PUT, POST + // HEAD, PUT, POST or when CDN_PUBLIC_URL is defined return next(); }); @@ -168,13 +313,13 @@ app.get('*', async (c) => { if (!hash) { throw new HTTPException(400, { message: 'Invalid path, hash missing' }); } - const key = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 }); - if (!key) { - throw new HTTPException(404, { message: 'Blob hash key not found.' }); + + if (c.env.CDN_PUBLIC_URL) { + return c.redirect(`${c.env.CDN_PUBLIC_URL}/${hash}`); } if (c.req.method == 'HEAD') { - const blob = await c.env.BLOSSOM_BUCKET.head(key); + const blob = await c.env.BLOSSOM_BUCKET.head(hash); if (!blob) { throw new HTTPException(404, { message: 'Blob hash not found.' }); } @@ -186,7 +331,7 @@ app.get('*', async (c) => { return c.body(null, 204); } else { - const blob = await c.env.BLOSSOM_BUCKET.get(key, { + const blob = await c.env.BLOSSOM_BUCKET.get(hash, { range: c.req.raw.headers, }); if (!blob) { diff --git a/src/types.ts b/src/types.ts index f0b4e18..e00f268 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,6 +3,7 @@ export type Bindings = { KV_BLOSSOM: KVNamespace; PUBLIC_URL: string; ALLOWED_NPUBS: string; + CDN_PUBLIC_URL?: string; }; export type BlobData = { diff --git a/wrangler.toml.sample b/wrangler.toml.sample index cffa19c..dd275bc 100644 --- a/wrangler.toml.sample +++ b/wrangler.toml.sample @@ -7,6 +7,11 @@ compatibility_date = "2023-12-01" # # The public URL under which the service is available, i.e. usually a custom domain PUBLIC_URL = "https://server.domain.name" + +# When a public CDN url is set, the blobs are served directly through this URL +# with a 302 redirect. RECOMMENDED. +CDN_PUBLIC_URL = "https://cdn.media-server.slidestr.net" + # # Comma separated list of nupbs of allowed users. No blanks spaces. Leave empty for public access. ALLOWED_NPUBS = ""