diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ab2bc76 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +Dockerfile +node_modules +.env.sample \ No newline at end of file diff --git a/.env.sample b/.env.sample index 1f22409..225e4ab 100644 --- a/.env.sample +++ b/.env.sample @@ -10,9 +10,5 @@ S3_ACCESS_KEY_ID=xx S3_ACCESS_KEY_SECRET=xxx S3_BUCKET_NAME=bucketname -# The public where the bucket is accessible from. This is used to generate a link -# to the stream but is for informational purposes only. -PUBLIC_BASE_URL=https://yourdomain.com - # The maximum number of files to keep in the TLS directory. This is used to MAX_TS_FILES_TO_KEEP=20 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9e31798 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM srs:latest + +# Install Node.js +RUN apt-get -y update && apt-get install -y curl +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && apt-get install -y nodejs + +# Install the S3 upload tool +RUN mkdir /usr/local/srs/upload +RUN chmod 755 /usr/local/srs/upload +COPY . /usr/local/srs/upload +WORKDIR /usr/local/srs/upload +RUN npm install + +# Setup the startup script +WORKDIR /usr/local/srs +COPY entrypoint.sh /usr/local/srs/entrypoint.sh +RUN chmod 755 entrypoint.sh + +# Copy SRS config +COPY conf conf + +CMD ["bash", "-c", "./entrypoint.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..13baa5c --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2023 +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README b/README new file mode 100644 index 0000000..7f6f455 --- /dev/null +++ b/README @@ -0,0 +1,31 @@ + +# Overview + +This project allows upload of HLS based streaming data from the SRS (Simple Realtime Server, https://ossrs.io/) to an S3 based storage. The purpose is to publish a stream in HLS format to a cloud based data store to leverage CDN distribution. + +This project implements a NodeJs based webserver that provides a web hook that can be registered with SRS's `on_hls` webhook. Whenever a new video segment is created, this web hook is called and the implementation in this project uploads the `.ts` video segment as well as the `.m3u8` playlist information +to the storage bucket. + +To keep the bucket usage limited to a small amount of data, segments before a certain time frame (e.g. 60s) are automatically deleted from the bucket. + +# Configuration + +Create a `.env` file based on `.env.sample` with the S3 credentials + +rtmp://localhost/live + +any stream key + +# Usage + +``` +docker build -t srs-s3 . +docker run -p 1935:1935 -it --rm srs-s3 +``` + + + +# Known Limitation + +* Currently only streams with 1 camera and 1 format are supported. +* This upload/sync job needs to run on the same machine as SRS, since data is read from the local hard disk. This is the reason it currently runs in the same docker container. diff --git a/conf/mysrs.conf b/conf/mysrs.conf new file mode 100644 index 0000000..cdd3ea5 --- /dev/null +++ b/conf/mysrs.conf @@ -0,0 +1,54 @@ +# main config for srs. +# @see full.conf for detail config. + +listen 1935; +max_connections 100; +#srs_log_tank file; +#srs_log_file ./objs/srs.log; +#daemon off; +http_api { + enabled on; + listen 1985; +} +http_server { + enabled off; + listen 8080; + dir ./objs/nginx/html; +} +rtc_server { + enabled on; + listen 8000; # UDP port + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#config-candidate + candidate $CANDIDATE; +} +vhost __defaultVhost__ { + + hls { + enabled on; + hls_fragment 2; + hls_window 30; + #hls_td_ratio 1.2; # 1.5 is needed, 1.2 did not work + hls_ts_file [stream]/[stream]-[seq].ts; + hls_m3u8_file [stream]/[stream].m3u8; + } + http_remux { + enabled off; + mount [vhost]/[app]/[stream].flv; + } + rtc { + enabled off; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtmp-to-rtc + rtmp_to_rtc off; + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp + rtc_to_rtmp off; + } + + play{ + gop_cache_max_frames 2500; + } + http_hooks { + enabled on; + on_hls http://127.0.0.1:3000/api/v1/hls; + } + +} diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..abca989 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,7 @@ +# Start S3 upload tool +cd /usr/local/srs/upload +npm start & + +# Start SRS +cd /usr/local/srs +./objs/srs -c conf/mysrs.conf diff --git a/package-lock.json b/package-lock.json index df7136d..77e115b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,9 @@ }, "devDependencies": { "@types/express": "^4.17.17", - "ts-node": "^10.9.1" + "@types/node": "^20.3.3", + "ts-node": "^10.9.1", + "tslib": "^2.6.0" } }, "node_modules/@aws-crypto/crc32": { diff --git a/package.json b/package.json index ad22e5d..cfc73c1 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ }, "devDependencies": { "@types/express": "^4.17.17", - "ts-node": "^10.9.1" + "@types/node": "^20.3.3", + "ts-node": "^10.9.1", + "tslib": "^2.6.0" } } diff --git a/src/env.ts b/src/env.ts index fe291fd..4851ffe 100644 --- a/src/env.ts +++ b/src/env.ts @@ -4,6 +4,5 @@ export const S3_ENDPOINT = process.env.S3_ENDPOINT; export const S3_ACCESS_KEY_ID = process.env.S3_ACCESS_KEY_ID; export const S3_ACCESS_KEY_SECRET = process.env.S3_ACCESS_KEY_SECRET; export const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME || 'bucket'; -export const PUBLIC_BASE_URL = process.env.PUBLIC_BASE_URL || 'https://your.public.domain'; export const MAX_TS_FILES_TO_KEEP = parseInt(process.env.MAX_TS_FILES_TO_KEEP || '20', 10); export const PORT = process.env.PORT || 3000; diff --git a/src/index.ts b/src/index.ts index 605b73b..4f5df21 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,21 +18,19 @@ const processOnHlsEvent = async (hlsEvent: HLSUpdateEvent) => { s3Client, `${hlsEvent.cwd}/${hlsEvent.file}`, S3_BUCKET_NAME, - `${hlsEvent.stream}/${hlsEvent.url}`, + `${hlsEvent.url}`, 'video/mp2t' ); // Upload m3u8 file - const m3u8S3Name = `${hlsEvent.stream}/${hlsEvent.m3u8_url}`; + const m3u8S3Name = `${hlsEvent.m3u8_url}`; await uploadFile(s3Client, `${hlsEvent.cwd}/${hlsEvent.m3u8}`, S3_BUCKET_NAME, m3u8S3Name, 'application/x-mpegURL'); - // console.log(`- Stream available at: ${PUBLIC_BASE_URL}/${m3u8S3Name}`); - // Delete OLD ts files const current = hlsEvent.seq_no; - if (current > MAX_TS_FILES_TO_KEEP) { + if (current >= MAX_TS_FILES_TO_KEEP) { // >= because we need to delete file 0.ts const fileToDelete = hlsEvent.url.replace(/[0-9]+\.ts$/, `${current - MAX_TS_FILES_TO_KEEP}.ts`); - await deleteFile(s3Client, S3_BUCKET_NAME, `${hlsEvent.stream}/${fileToDelete}`); + await deleteFile(s3Client, S3_BUCKET_NAME, `${fileToDelete}`); } };