feat: Added redirect to cdn url, mirror feature
This commit is contained in:
florian
parent
dedea80914
commit
c1c8486294
247
src/index.ts
247
src/index.ts
@ -23,6 +23,49 @@ app.get('/', (c) =>
|
||||
)
|
||||
);
|
||||
|
||||
/*
|
||||
app.get('/migrate', async (c) => {
|
||||
try {
|
||||
const prefix = 'blob:';
|
||||
const kvList = await c.env.KV_BLOSSOM.list({ prefix, limit: 100 });
|
||||
let copyCount = 0;
|
||||
|
||||
for (const entry of kvList.keys) {
|
||||
const key = entry.name;
|
||||
const targetHash = key.replace(prefix, '');
|
||||
const blobLookup = await c.env.BLOSSOM_BUCKET.head(targetHash);
|
||||
const sourceId = await c.env.KV_BLOSSOM.get(key);
|
||||
|
||||
if (sourceId) {
|
||||
let deleteAfter = true;
|
||||
if (!blobLookup) {
|
||||
const blobContent = await c.env.BLOSSOM_BUCKET.get(sourceId);
|
||||
if (blobContent) {
|
||||
const contentBuffer = await blobContent.arrayBuffer();
|
||||
try {
|
||||
await c.env.BLOSSOM_BUCKET.put(targetHash, contentBuffer);
|
||||
copyCount++;
|
||||
} catch (e) {
|
||||
deleteAfter = false;
|
||||
console.warn('Could not copy blob ' + sourceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteAfter) {
|
||||
await c.env.BLOSSOM_BUCKET.delete(sourceId);
|
||||
await c.env.KV_BLOSSOM.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return c.json({ message: `${copyCount} Blobs processed successfull: ` + JSON.stringify(kvList.keys) });
|
||||
} catch (error: any) {
|
||||
return c.json({ error: error.message }, 500);
|
||||
}
|
||||
});
|
||||
*/
|
||||
|
||||
app.get('/list/:pubkey', async (c) => {
|
||||
const auth = getAuth(c.req.header('authorization') as string);
|
||||
await checkAuth(auth, 'list', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM);
|
||||
@ -56,7 +99,110 @@ app.get('/list/:pubkey', async (c) => {
|
||||
return await c.json(listOfBlobs);
|
||||
});
|
||||
|
||||
app.get('/list', cache({ cacheName: 'blossom', cacheControl: 'max-age=60' }));
|
||||
app.get('/list', cache({ cacheName: 'blossom5', cacheControl: 'max-age=60' }));
|
||||
|
||||
app.put('/mirror', async (c) => {
|
||||
const auth = getAuth(c.req.header('authorization') as string);
|
||||
await checkAuth(auth, 'upload', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM);
|
||||
|
||||
const { url } = await c.req.json<{ url: string }>();
|
||||
if (!url) {
|
||||
throw new HTTPException(400, { message: 'No source url in body.' });
|
||||
}
|
||||
|
||||
if (!auth.objectHash) {
|
||||
throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' });
|
||||
}
|
||||
|
||||
const { pubkey } = auth.event;
|
||||
let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash);
|
||||
|
||||
if (!existing) {
|
||||
console.log('not existing, trying to download ' + url);
|
||||
|
||||
try {
|
||||
// Fetch the file from the URL
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new HTTPException(500, { message: `Failed to fetch the file: ${response.statusText}` });
|
||||
}
|
||||
|
||||
// Ensure the Content-Length header is present
|
||||
const contentLength = response.headers.get('Content-Length');
|
||||
if (!contentLength) {
|
||||
throw new Error('Content-Length header is required to determine the file size');
|
||||
}
|
||||
|
||||
// Create a writable stream to R2
|
||||
const { writable, readable } = new FixedLengthStream(parseInt(contentLength, 10));
|
||||
const writer = writable.getWriter();
|
||||
|
||||
// Create a DigestStream for SHA-256 hashing
|
||||
const digestStream = new crypto.DigestStream('SHA-256');
|
||||
const hashWriter = digestStream.getWriter();
|
||||
|
||||
// Stream the response directly to R2 and hash
|
||||
const uploadPromise = c.env.BLOSSOM_BUCKET.put(auth.objectHash, readable, {
|
||||
httpMetadata: response.headers,
|
||||
customMetadata: {
|
||||
pubkey,
|
||||
},
|
||||
});
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new HTTPException(500, { message: 'Failed to get reader from response body' });
|
||||
}
|
||||
|
||||
const pump = async () => {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
// Write data to both R2 writer and hash writer
|
||||
await writer.write(value);
|
||||
await hashWriter.write(value);
|
||||
}
|
||||
await writer.close();
|
||||
await hashWriter.close();
|
||||
};
|
||||
|
||||
const [_, storedObject] = await Promise.all([pump(), uploadPromise, digestStream]);
|
||||
|
||||
// Get the hash as a hex string
|
||||
const hashBuffer = await digestStream.digest;
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
|
||||
|
||||
if (hashHex != auth.objectHash) {
|
||||
await c.env.BLOSSOM_BUCKET.delete(auth.objectHash);
|
||||
throw new HTTPException(500, { message: `File content does not match the authenticated sha256 hash.` });
|
||||
}
|
||||
|
||||
existing = storedObject;
|
||||
} catch (error: any) {
|
||||
throw new HTTPException(500, { message: `Error while mirroring url: ${error.message}` });
|
||||
}
|
||||
}
|
||||
|
||||
if (!existing) {
|
||||
throw new HTTPException(500, { message: `Mirroring of url failed.` });
|
||||
}
|
||||
|
||||
// Record the blob descriptors for the list operation
|
||||
const blobData: BlobData = {
|
||||
size: existing.size,
|
||||
type: existing.httpMetadata?.contentType,
|
||||
created: dayjs(existing.uploaded).unix(),
|
||||
};
|
||||
await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData));
|
||||
|
||||
c.status(200);
|
||||
return c.json({
|
||||
...blobData,
|
||||
url: c.env.PUBLIC_URL + '/' + existing.key,
|
||||
sha256: existing.key,
|
||||
});
|
||||
});
|
||||
|
||||
app.put('/upload', async (c) => {
|
||||
const auth = getAuth(c.req.header('authorization') as string);
|
||||
@ -69,48 +215,53 @@ app.put('/upload', async (c) => {
|
||||
// TODO other npub upload an existing file
|
||||
// TODO same npub uploads existing file
|
||||
|
||||
const { pubkey } = auth.event;
|
||||
const blobId = crypto.randomUUID();
|
||||
const digestStream = new crypto.DigestStream('SHA-256');
|
||||
const [stream1, stream2] = c.req.raw.body.tee();
|
||||
|
||||
const [_, storedObject] = await Promise.all([
|
||||
stream1.pipeTo(digestStream as WritableStream<any>),
|
||||
c.env.BLOSSOM_BUCKET.put(blobId, stream2 as ReadableStream<any>, {
|
||||
httpMetadata: c.req.raw.headers,
|
||||
customMetadata: {
|
||||
pubkey,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const hash = Array.from(new Uint8Array(await digestStream.digest))
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
|
||||
if (auth.objectHash) {
|
||||
// currently not all clients support x tag in auth
|
||||
if (auth.objectHash.toLocaleLowerCase() !== hash.toLocaleLowerCase()) {
|
||||
c.env.BLOSSOM_BUCKET.delete(blobId); // delete the blob if hash does not match
|
||||
throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' });
|
||||
}
|
||||
if (!auth.objectHash) {
|
||||
throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' });
|
||||
}
|
||||
|
||||
await c.env.KV_BLOSSOM.put('blob:' + hash, blobId); // R2 IDs for each blob hash
|
||||
const { pubkey } = auth.event;
|
||||
let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash);
|
||||
|
||||
if (!existing) {
|
||||
const digestStream = new crypto.DigestStream('SHA-256');
|
||||
const [stream1, stream2] = c.req.raw.body.tee();
|
||||
|
||||
const [_, storedObject] = await Promise.all([
|
||||
stream1.pipeTo(digestStream as WritableStream<any>),
|
||||
c.env.BLOSSOM_BUCKET.put(auth.objectHash, stream2 as ReadableStream<any>, {
|
||||
httpMetadata: c.req.raw.headers,
|
||||
customMetadata: {
|
||||
pubkey,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const storedHash = Array.from(new Uint8Array(await digestStream.digest))
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
|
||||
if (auth.objectHash.toLocaleLowerCase() !== storedHash.toLocaleLowerCase()) {
|
||||
// Undo upload when the hash doesn't match
|
||||
c.env.BLOSSOM_BUCKET.delete(auth.objectHash);
|
||||
throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' });
|
||||
}
|
||||
|
||||
existing = storedObject;
|
||||
}
|
||||
|
||||
// Record the blob descriptors for the list operation
|
||||
const blobData: BlobData = {
|
||||
size: storedObject.size,
|
||||
type: storedObject.httpMetadata?.contentType,
|
||||
created: dayjs(storedObject.uploaded).unix(),
|
||||
size: existing.size,
|
||||
type: existing.httpMetadata?.contentType,
|
||||
created: dayjs(existing.uploaded).unix(),
|
||||
};
|
||||
await c.env.KV_BLOSSOM.put(pubkey + ':' + hash, JSON.stringify(blobData));
|
||||
await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData));
|
||||
|
||||
c.status(200);
|
||||
return c.json({
|
||||
...blobData,
|
||||
url: c.env.PUBLIC_URL + '/' + hash,
|
||||
sha256: hash,
|
||||
url: c.env.PUBLIC_URL + '/' + existing.key,
|
||||
sha256: existing.key,
|
||||
});
|
||||
});
|
||||
|
||||
@ -133,17 +284,9 @@ app.delete('*', async (c) => {
|
||||
throw new HTTPException(403, { message: 'Not allowed to delete the blob.' });
|
||||
}
|
||||
|
||||
const blobKey = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 });
|
||||
if (!blobKey) {
|
||||
throw new HTTPException(404, { message: `Blob with the hash ${hash} not found.` });
|
||||
}
|
||||
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
c.env.BLOSSOM_BUCKET.delete(blobKey),
|
||||
c.env.KV_BLOSSOM.delete('blob:' + hash),
|
||||
c.env.KV_BLOSSOM.delete(pubkey + ':' + hash),
|
||||
]);
|
||||
// TODO only delete the blob when no other npub is using it
|
||||
await Promise.allSettled([c.env.BLOSSOM_BUCKET.delete(hash), c.env.KV_BLOSSOM.delete(pubkey + ':' + hash)]);
|
||||
return c.body(null, 204);
|
||||
} catch (e) {
|
||||
throw new HTTPException(404, { message: 'Blob not found.' });
|
||||
@ -151,15 +294,17 @@ app.delete('*', async (c) => {
|
||||
});
|
||||
|
||||
const getCache = cache({
|
||||
cacheName: 'blossom4',
|
||||
cacheName: 'blossom5',
|
||||
cacheControl: 'max-age=604800',
|
||||
vary: ['Access-Control-Request-Headers', 'Accept-Encoding', 'Content-Range'],
|
||||
});
|
||||
|
||||
app.use(async (c, next: Next) => {
|
||||
if (c.req.method == 'GET') return getCache(c, next);
|
||||
// Only use cache for get requests and
|
||||
// and when we don't have a CDN_PUBLIC_URL to redirect to
|
||||
if (c.req.method == 'GET' && !c.env.CDN_PUBLIC_URL) return getCache(c, next);
|
||||
|
||||
// HEAD, PUT, POST
|
||||
// HEAD, PUT, POST or when CDN_PUBLIC_URL is defined
|
||||
return next();
|
||||
});
|
||||
|
||||
@ -168,13 +313,13 @@ app.get('*', async (c) => {
|
||||
if (!hash) {
|
||||
throw new HTTPException(400, { message: 'Invalid path, hash missing' });
|
||||
}
|
||||
const key = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 });
|
||||
if (!key) {
|
||||
throw new HTTPException(404, { message: 'Blob hash key not found.' });
|
||||
|
||||
if (c.env.CDN_PUBLIC_URL) {
|
||||
return c.redirect(`${c.env.CDN_PUBLIC_URL}/${hash}`);
|
||||
}
|
||||
|
||||
if (c.req.method == 'HEAD') {
|
||||
const blob = await c.env.BLOSSOM_BUCKET.head(key);
|
||||
const blob = await c.env.BLOSSOM_BUCKET.head(hash);
|
||||
if (!blob) {
|
||||
throw new HTTPException(404, { message: 'Blob hash not found.' });
|
||||
}
|
||||
@ -186,7 +331,7 @@ app.get('*', async (c) => {
|
||||
|
||||
return c.body(null, 204);
|
||||
} else {
|
||||
const blob = await c.env.BLOSSOM_BUCKET.get(key, {
|
||||
const blob = await c.env.BLOSSOM_BUCKET.get(hash, {
|
||||
range: c.req.raw.headers,
|
||||
});
|
||||
if (!blob) {
|
||||
|
||||
@ -3,6 +3,7 @@ export type Bindings = {
|
||||
KV_BLOSSOM: KVNamespace;
|
||||
PUBLIC_URL: string;
|
||||
ALLOWED_NPUBS: string;
|
||||
CDN_PUBLIC_URL?: string;
|
||||
};
|
||||
|
||||
export type BlobData = {
|
||||
|
||||
@ -7,6 +7,11 @@ compatibility_date = "2023-12-01"
|
||||
#
|
||||
# The public URL under which the service is available, i.e. usually a custom domain
|
||||
PUBLIC_URL = "https://server.domain.name"
|
||||
|
||||
# When a public CDN url is set, the blobs are served directly through this URL
|
||||
# with a 302 redirect. RECOMMENDED.
|
||||
CDN_PUBLIC_URL = "https://cdn.media-server.slidestr.net"
|
||||
|
||||
#
|
||||
# Comma separated list of nupbs of allowed users. No blanks spaces. Leave empty for public access.
|
||||
ALLOWED_NPUBS = "<NPUB to give access>"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user