feat: Added redirect to cdn url, mirror feature
This commit is contained in:
parent
dedea80914
commit
c1c8486294
247
src/index.ts
247
src/index.ts
@ -23,6 +23,49 @@ app.get('/', (c) =>
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/*
|
||||||
|
app.get('/migrate', async (c) => {
|
||||||
|
try {
|
||||||
|
const prefix = 'blob:';
|
||||||
|
const kvList = await c.env.KV_BLOSSOM.list({ prefix, limit: 100 });
|
||||||
|
let copyCount = 0;
|
||||||
|
|
||||||
|
for (const entry of kvList.keys) {
|
||||||
|
const key = entry.name;
|
||||||
|
const targetHash = key.replace(prefix, '');
|
||||||
|
const blobLookup = await c.env.BLOSSOM_BUCKET.head(targetHash);
|
||||||
|
const sourceId = await c.env.KV_BLOSSOM.get(key);
|
||||||
|
|
||||||
|
if (sourceId) {
|
||||||
|
let deleteAfter = true;
|
||||||
|
if (!blobLookup) {
|
||||||
|
const blobContent = await c.env.BLOSSOM_BUCKET.get(sourceId);
|
||||||
|
if (blobContent) {
|
||||||
|
const contentBuffer = await blobContent.arrayBuffer();
|
||||||
|
try {
|
||||||
|
await c.env.BLOSSOM_BUCKET.put(targetHash, contentBuffer);
|
||||||
|
copyCount++;
|
||||||
|
} catch (e) {
|
||||||
|
deleteAfter = false;
|
||||||
|
console.warn('Could not copy blob ' + sourceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deleteAfter) {
|
||||||
|
await c.env.BLOSSOM_BUCKET.delete(sourceId);
|
||||||
|
await c.env.KV_BLOSSOM.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.json({ message: `${copyCount} Blobs processed successfull: ` + JSON.stringify(kvList.keys) });
|
||||||
|
} catch (error: any) {
|
||||||
|
return c.json({ error: error.message }, 500);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
*/
|
||||||
|
|
||||||
app.get('/list/:pubkey', async (c) => {
|
app.get('/list/:pubkey', async (c) => {
|
||||||
const auth = getAuth(c.req.header('authorization') as string);
|
const auth = getAuth(c.req.header('authorization') as string);
|
||||||
await checkAuth(auth, 'list', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM);
|
await checkAuth(auth, 'list', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM);
|
||||||
@ -56,7 +99,110 @@ app.get('/list/:pubkey', async (c) => {
|
|||||||
return await c.json(listOfBlobs);
|
return await c.json(listOfBlobs);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/list', cache({ cacheName: 'blossom', cacheControl: 'max-age=60' }));
|
app.get('/list', cache({ cacheName: 'blossom5', cacheControl: 'max-age=60' }));
|
||||||
|
|
||||||
|
app.put('/mirror', async (c) => {
|
||||||
|
const auth = getAuth(c.req.header('authorization') as string);
|
||||||
|
await checkAuth(auth, 'upload', c.env.ALLOWED_NPUBS, c.env.KV_BLOSSOM);
|
||||||
|
|
||||||
|
const { url } = await c.req.json<{ url: string }>();
|
||||||
|
if (!url) {
|
||||||
|
throw new HTTPException(400, { message: 'No source url in body.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!auth.objectHash) {
|
||||||
|
throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const { pubkey } = auth.event;
|
||||||
|
let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash);
|
||||||
|
|
||||||
|
if (!existing) {
|
||||||
|
console.log('not existing, trying to download ' + url);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Fetch the file from the URL
|
||||||
|
const response = await fetch(url);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new HTTPException(500, { message: `Failed to fetch the file: ${response.statusText}` });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the Content-Length header is present
|
||||||
|
const contentLength = response.headers.get('Content-Length');
|
||||||
|
if (!contentLength) {
|
||||||
|
throw new Error('Content-Length header is required to determine the file size');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a writable stream to R2
|
||||||
|
const { writable, readable } = new FixedLengthStream(parseInt(contentLength, 10));
|
||||||
|
const writer = writable.getWriter();
|
||||||
|
|
||||||
|
// Create a DigestStream for SHA-256 hashing
|
||||||
|
const digestStream = new crypto.DigestStream('SHA-256');
|
||||||
|
const hashWriter = digestStream.getWriter();
|
||||||
|
|
||||||
|
// Stream the response directly to R2 and hash
|
||||||
|
const uploadPromise = c.env.BLOSSOM_BUCKET.put(auth.objectHash, readable, {
|
||||||
|
httpMetadata: response.headers,
|
||||||
|
customMetadata: {
|
||||||
|
pubkey,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const reader = response.body?.getReader();
|
||||||
|
if (!reader) {
|
||||||
|
throw new HTTPException(500, { message: 'Failed to get reader from response body' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const pump = async () => {
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
// Write data to both R2 writer and hash writer
|
||||||
|
await writer.write(value);
|
||||||
|
await hashWriter.write(value);
|
||||||
|
}
|
||||||
|
await writer.close();
|
||||||
|
await hashWriter.close();
|
||||||
|
};
|
||||||
|
|
||||||
|
const [_, storedObject] = await Promise.all([pump(), uploadPromise, digestStream]);
|
||||||
|
|
||||||
|
// Get the hash as a hex string
|
||||||
|
const hashBuffer = await digestStream.digest;
|
||||||
|
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||||
|
const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join('');
|
||||||
|
|
||||||
|
if (hashHex != auth.objectHash) {
|
||||||
|
await c.env.BLOSSOM_BUCKET.delete(auth.objectHash);
|
||||||
|
throw new HTTPException(500, { message: `File content does not match the authenticated sha256 hash.` });
|
||||||
|
}
|
||||||
|
|
||||||
|
existing = storedObject;
|
||||||
|
} catch (error: any) {
|
||||||
|
throw new HTTPException(500, { message: `Error while mirroring url: ${error.message}` });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!existing) {
|
||||||
|
throw new HTTPException(500, { message: `Mirroring of url failed.` });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record the blob descriptors for the list operation
|
||||||
|
const blobData: BlobData = {
|
||||||
|
size: existing.size,
|
||||||
|
type: existing.httpMetadata?.contentType,
|
||||||
|
created: dayjs(existing.uploaded).unix(),
|
||||||
|
};
|
||||||
|
await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData));
|
||||||
|
|
||||||
|
c.status(200);
|
||||||
|
return c.json({
|
||||||
|
...blobData,
|
||||||
|
url: c.env.PUBLIC_URL + '/' + existing.key,
|
||||||
|
sha256: existing.key,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
app.put('/upload', async (c) => {
|
app.put('/upload', async (c) => {
|
||||||
const auth = getAuth(c.req.header('authorization') as string);
|
const auth = getAuth(c.req.header('authorization') as string);
|
||||||
@ -69,48 +215,53 @@ app.put('/upload', async (c) => {
|
|||||||
// TODO other npub upload an existing file
|
// TODO other npub upload an existing file
|
||||||
// TODO same npub uploads existing file
|
// TODO same npub uploads existing file
|
||||||
|
|
||||||
const { pubkey } = auth.event;
|
if (!auth.objectHash) {
|
||||||
const blobId = crypto.randomUUID();
|
throw new HTTPException(400, { message: 'Required "x" tag object hash is missing in auth event.' });
|
||||||
const digestStream = new crypto.DigestStream('SHA-256');
|
|
||||||
const [stream1, stream2] = c.req.raw.body.tee();
|
|
||||||
|
|
||||||
const [_, storedObject] = await Promise.all([
|
|
||||||
stream1.pipeTo(digestStream as WritableStream<any>),
|
|
||||||
c.env.BLOSSOM_BUCKET.put(blobId, stream2 as ReadableStream<any>, {
|
|
||||||
httpMetadata: c.req.raw.headers,
|
|
||||||
customMetadata: {
|
|
||||||
pubkey,
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const hash = Array.from(new Uint8Array(await digestStream.digest))
|
|
||||||
.map((b) => b.toString(16).padStart(2, '0'))
|
|
||||||
.join('');
|
|
||||||
|
|
||||||
if (auth.objectHash) {
|
|
||||||
// currently not all clients support x tag in auth
|
|
||||||
if (auth.objectHash.toLocaleLowerCase() !== hash.toLocaleLowerCase()) {
|
|
||||||
c.env.BLOSSOM_BUCKET.delete(blobId); // delete the blob if hash does not match
|
|
||||||
throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await c.env.KV_BLOSSOM.put('blob:' + hash, blobId); // R2 IDs for each blob hash
|
const { pubkey } = auth.event;
|
||||||
|
let existing = await c.env.BLOSSOM_BUCKET.head(auth.objectHash);
|
||||||
|
|
||||||
|
if (!existing) {
|
||||||
|
const digestStream = new crypto.DigestStream('SHA-256');
|
||||||
|
const [stream1, stream2] = c.req.raw.body.tee();
|
||||||
|
|
||||||
|
const [_, storedObject] = await Promise.all([
|
||||||
|
stream1.pipeTo(digestStream as WritableStream<any>),
|
||||||
|
c.env.BLOSSOM_BUCKET.put(auth.objectHash, stream2 as ReadableStream<any>, {
|
||||||
|
httpMetadata: c.req.raw.headers,
|
||||||
|
customMetadata: {
|
||||||
|
pubkey,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const storedHash = Array.from(new Uint8Array(await digestStream.digest))
|
||||||
|
.map((b) => b.toString(16).padStart(2, '0'))
|
||||||
|
.join('');
|
||||||
|
|
||||||
|
if (auth.objectHash.toLocaleLowerCase() !== storedHash.toLocaleLowerCase()) {
|
||||||
|
// Undo upload when the hash doesn't match
|
||||||
|
c.env.BLOSSOM_BUCKET.delete(auth.objectHash);
|
||||||
|
throw new HTTPException(400, { message: 'Object hash does not match the uploaded blob.' });
|
||||||
|
}
|
||||||
|
|
||||||
|
existing = storedObject;
|
||||||
|
}
|
||||||
|
|
||||||
// Record the blob descriptors for the list operation
|
// Record the blob descriptors for the list operation
|
||||||
const blobData: BlobData = {
|
const blobData: BlobData = {
|
||||||
size: storedObject.size,
|
size: existing.size,
|
||||||
type: storedObject.httpMetadata?.contentType,
|
type: existing.httpMetadata?.contentType,
|
||||||
created: dayjs(storedObject.uploaded).unix(),
|
created: dayjs(existing.uploaded).unix(),
|
||||||
};
|
};
|
||||||
await c.env.KV_BLOSSOM.put(pubkey + ':' + hash, JSON.stringify(blobData));
|
await c.env.KV_BLOSSOM.put(pubkey + ':' + existing.key, JSON.stringify(blobData));
|
||||||
|
|
||||||
c.status(200);
|
c.status(200);
|
||||||
return c.json({
|
return c.json({
|
||||||
...blobData,
|
...blobData,
|
||||||
url: c.env.PUBLIC_URL + '/' + hash,
|
url: c.env.PUBLIC_URL + '/' + existing.key,
|
||||||
sha256: hash,
|
sha256: existing.key,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -133,17 +284,9 @@ app.delete('*', async (c) => {
|
|||||||
throw new HTTPException(403, { message: 'Not allowed to delete the blob.' });
|
throw new HTTPException(403, { message: 'Not allowed to delete the blob.' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const blobKey = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 });
|
|
||||||
if (!blobKey) {
|
|
||||||
throw new HTTPException(404, { message: `Blob with the hash ${hash} not found.` });
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await Promise.allSettled([
|
// TODO only delete the blob when no other npub is using it
|
||||||
c.env.BLOSSOM_BUCKET.delete(blobKey),
|
await Promise.allSettled([c.env.BLOSSOM_BUCKET.delete(hash), c.env.KV_BLOSSOM.delete(pubkey + ':' + hash)]);
|
||||||
c.env.KV_BLOSSOM.delete('blob:' + hash),
|
|
||||||
c.env.KV_BLOSSOM.delete(pubkey + ':' + hash),
|
|
||||||
]);
|
|
||||||
return c.body(null, 204);
|
return c.body(null, 204);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
throw new HTTPException(404, { message: 'Blob not found.' });
|
throw new HTTPException(404, { message: 'Blob not found.' });
|
||||||
@ -151,15 +294,17 @@ app.delete('*', async (c) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const getCache = cache({
|
const getCache = cache({
|
||||||
cacheName: 'blossom4',
|
cacheName: 'blossom5',
|
||||||
cacheControl: 'max-age=604800',
|
cacheControl: 'max-age=604800',
|
||||||
vary: ['Access-Control-Request-Headers', 'Accept-Encoding', 'Content-Range'],
|
vary: ['Access-Control-Request-Headers', 'Accept-Encoding', 'Content-Range'],
|
||||||
});
|
});
|
||||||
|
|
||||||
app.use(async (c, next: Next) => {
|
app.use(async (c, next: Next) => {
|
||||||
if (c.req.method == 'GET') return getCache(c, next);
|
// Only use cache for get requests and
|
||||||
|
// and when we don't have a CDN_PUBLIC_URL to redirect to
|
||||||
|
if (c.req.method == 'GET' && !c.env.CDN_PUBLIC_URL) return getCache(c, next);
|
||||||
|
|
||||||
// HEAD, PUT, POST
|
// HEAD, PUT, POST or when CDN_PUBLIC_URL is defined
|
||||||
return next();
|
return next();
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -168,13 +313,13 @@ app.get('*', async (c) => {
|
|||||||
if (!hash) {
|
if (!hash) {
|
||||||
throw new HTTPException(400, { message: 'Invalid path, hash missing' });
|
throw new HTTPException(400, { message: 'Invalid path, hash missing' });
|
||||||
}
|
}
|
||||||
const key = await c.env.KV_BLOSSOM.get('blob:' + hash, { cacheTtl: 3600 });
|
|
||||||
if (!key) {
|
if (c.env.CDN_PUBLIC_URL) {
|
||||||
throw new HTTPException(404, { message: 'Blob hash key not found.' });
|
return c.redirect(`${c.env.CDN_PUBLIC_URL}/${hash}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (c.req.method == 'HEAD') {
|
if (c.req.method == 'HEAD') {
|
||||||
const blob = await c.env.BLOSSOM_BUCKET.head(key);
|
const blob = await c.env.BLOSSOM_BUCKET.head(hash);
|
||||||
if (!blob) {
|
if (!blob) {
|
||||||
throw new HTTPException(404, { message: 'Blob hash not found.' });
|
throw new HTTPException(404, { message: 'Blob hash not found.' });
|
||||||
}
|
}
|
||||||
@ -186,7 +331,7 @@ app.get('*', async (c) => {
|
|||||||
|
|
||||||
return c.body(null, 204);
|
return c.body(null, 204);
|
||||||
} else {
|
} else {
|
||||||
const blob = await c.env.BLOSSOM_BUCKET.get(key, {
|
const blob = await c.env.BLOSSOM_BUCKET.get(hash, {
|
||||||
range: c.req.raw.headers,
|
range: c.req.raw.headers,
|
||||||
});
|
});
|
||||||
if (!blob) {
|
if (!blob) {
|
||||||
|
@ -3,6 +3,7 @@ export type Bindings = {
|
|||||||
KV_BLOSSOM: KVNamespace;
|
KV_BLOSSOM: KVNamespace;
|
||||||
PUBLIC_URL: string;
|
PUBLIC_URL: string;
|
||||||
ALLOWED_NPUBS: string;
|
ALLOWED_NPUBS: string;
|
||||||
|
CDN_PUBLIC_URL?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type BlobData = {
|
export type BlobData = {
|
||||||
|
@ -7,6 +7,11 @@ compatibility_date = "2023-12-01"
|
|||||||
#
|
#
|
||||||
# The public URL under which the service is available, i.e. usually a custom domain
|
# The public URL under which the service is available, i.e. usually a custom domain
|
||||||
PUBLIC_URL = "https://server.domain.name"
|
PUBLIC_URL = "https://server.domain.name"
|
||||||
|
|
||||||
|
# When a public CDN url is set, the blobs are served directly through this URL
|
||||||
|
# with a 302 redirect. RECOMMENDED.
|
||||||
|
CDN_PUBLIC_URL = "https://cdn.media-server.slidestr.net"
|
||||||
|
|
||||||
#
|
#
|
||||||
# Comma separated list of nupbs of allowed users. No blanks spaces. Leave empty for public access.
|
# Comma separated list of nupbs of allowed users. No blanks spaces. Leave empty for public access.
|
||||||
ALLOWED_NPUBS = "<NPUB to give access>"
|
ALLOWED_NPUBS = "<NPUB to give access>"
|
||||||
|
Loading…
Reference in New Issue
Block a user