1
0
mirror of git://jb55.com/damus synced 2024-09-28 16:00:43 +00:00

Cache videos

This commit implements a simple but functional video cache.

It works by providing a method called `maybe_cached_url`, where a video
URL can be passed in, and this method will either return the URL of a
cached version of this video if available, or the original URL if not. It also
downloads new video URLs on the background into the cache folder for use
next time.

Functional testing
-------------------

PASS

Device: iPhone 15 simulator
iOS: 17.4
Damus: Approximately this commit
Setup:
- Debug connection
- Expiry time locally changed to 5 minutes

Steps:
1. Basic functionality
  1. Go to a profile with lots of videos
  2. Scroll down
  3. Filter logs to only logs that start with "Loading video with URL"
  4. Check that most videos are being loaded from external URLs. PASS
  5. Now restart the app and go to that same profile
  6. Scroll down and watch logs. Videos should now be loaded with an internal file URL. PASS
2. Automatic cache refresh after expiry
  1. Go to the video-heavy profile, make note of the external URL.
  2. Go to a different screen and then come back to that video. Make sure the file was loaded from cache. PASS
  3. Now go to a different screen and wait 5 minutes.
  4. Come back to the same video. It should be loaded from the external URL. PASS
3. "Clear cache" button functionality
  1. Go to the video-heavy profile, make note of the external URL.
  2. Go to a different screen and then come back to that video. Make sure the file was loaded from cache. PASS
  3. Now quit the app (to ensure file is not in use when trying to delete it)
  4. Clear cache in settings
  5. Go back to the same video. It should now be loaded from the external URL. PASS

Performance testing
-----------------------

Device: iPhone 13 mini
iOS: 17.3.1
Damus: This commit
Baseline: 87de88861adb3b41d73998452e7c876ab5ee06bf
Setup:
- Debug connection
- Expiry time locally changed to 5 minutes
- Running on Profile mode, with XCode Instruments

Steps:
1. Start recording network activity with XCode Instruments
2. Go to a video-heavy profile (e.g. Julian Figueroa)
3. Scroll down to a specific video (Make sure to scroll through at least 5 videos)
4. Stop recording and measure the "Bytes In" from "Network connections"
5. Repeat this for all test configurations

Results:
- Baseline (No caching support): 26.74 MiB
- This commit (First run, cleared cache): 40.52 MiB
- This commit (Second run, cache filled with videos): 8.13 MiB

Automated test coverage
------------------------

PASS

Device: iPhone 15 simulator
iOS: 17.4
Damus: This commit
Coverage:
- Ran new automated tests multiple times. PASS 3/3 times
- Ran all other automated tests. PASS

Signed-off-by: Daniel D’Aquino <daniel@daquino.me>
Link: 20240411004129.84436-3-daniel@daquino.me
Signed-off-by: William Casarin <jb55@jb55.com>
This commit is contained in:
Daniel D’Aquino 2024-04-11 00:41:54 +00:00 committed by William Casarin
parent c2918aaf16
commit 26d2627a1c
6 changed files with 242 additions and 1 deletions

View File

@ -490,6 +490,7 @@
D76556D62B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D76556D52B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift */; };
D76874F32AE3632B00FB0F68 /* ProfileZapLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D76874F22AE3632B00FB0F68 /* ProfileZapLinkView.swift */; };
D77BFA0B2AE3051200621634 /* ProfileActionSheetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D77BFA0A2AE3051200621634 /* ProfileActionSheetView.swift */; };
D7831AF82BBE11E2005DA780 /* VideoCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */; };
D783A63F2AD4E53D00658DDA /* SuggestedHashtagsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D783A63E2AD4E53D00658DDA /* SuggestedHashtagsView.swift */; };
D78525252A7B2EA4002FA637 /* NoteContentViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */; };
D7870BC12AC4750B0080BA88 /* MentionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7870BC02AC4750B0080BA88 /* MentionView.swift */; };
@ -520,6 +521,7 @@
D7ADD3DE2B53854300F104C4 /* DamusPurpleURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3DD2B53854300F104C4 /* DamusPurpleURL.swift */; };
D7ADD3E02B538D4200F104C4 /* DamusPurpleURLSheetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3DF2B538D4200F104C4 /* DamusPurpleURLSheetView.swift */; };
D7ADD3E22B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3E12B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift */; };
D7C28E3B2BBB4D0000EE459F /* VideoCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */; };
D7C6787E2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7C6787D2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift */; };
D7CB5D3B2B112FBB00AD4105 /* NotificationFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D70A3B162B02DCE5008BD568 /* NotificationFormatter.swift */; };
D7CB5D3C2B1130C600AD4105 /* LocalNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CDA128B29EB19C40006FA5A /* LocalNotification.swift */; };
@ -1402,6 +1404,7 @@
D76556D52B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleWelcomeView.swift; sourceTree = "<group>"; };
D76874F22AE3632B00FB0F68 /* ProfileZapLinkView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileZapLinkView.swift; sourceTree = "<group>"; };
D77BFA0A2AE3051200621634 /* ProfileActionSheetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileActionSheetView.swift; sourceTree = "<group>"; };
D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoCacheTests.swift; sourceTree = "<group>"; };
D783A63E2AD4E53D00658DDA /* SuggestedHashtagsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestedHashtagsView.swift; sourceTree = "<group>"; };
D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NoteContentViewTests.swift; sourceTree = "<group>"; };
D7870BC02AC4750B0080BA88 /* MentionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MentionView.swift; sourceTree = "<group>"; };
@ -1417,6 +1420,7 @@
D7ADD3DD2B53854300F104C4 /* DamusPurpleURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleURL.swift; sourceTree = "<group>"; };
D7ADD3DF2B538D4200F104C4 /* DamusPurpleURLSheetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleURLSheetView.swift; sourceTree = "<group>"; };
D7ADD3E12B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleVerifyNpubView.swift; sourceTree = "<group>"; };
D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoCache.swift; sourceTree = "<group>"; };
D7C6787D2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NIP98AuthenticatedRequest.swift; sourceTree = "<group>"; };
D7CB5D3D2B116DAD00AD4105 /* NotificationsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NotificationsManager.swift; sourceTree = "<group>"; };
D7CB5D442B116FE800AD4105 /* Contacts+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Contacts+.swift"; sourceTree = "<group>"; };
@ -1650,6 +1654,7 @@
D74AAFC12B153395006CF0F4 /* HeadlessDamusState.swift */,
B5C60C1F2B530D5100C5ECA7 /* MuteItem.swift */,
B533694D2B66D791008A805E /* MutelistManager.swift */,
D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */,
);
path = Models;
sourceTree = "<group>";
@ -2553,6 +2558,7 @@
E06336A92B75832100A88E6B /* ImageMetadataTest.swift */,
D7CBD1D52B8D509800BFD889 /* DamusPurpleImpendingExpirationTests.swift */,
D72927AC2BAB515C00F93E90 /* RelayURLTests.swift */,
D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */,
);
path = damusTests;
sourceTree = "<group>";
@ -3288,6 +3294,7 @@
4C1253522A76C6130004F4B8 /* ComposeNotify.swift in Sources */,
4C7D09662A0AE62100943473 /* AlbyButton.swift in Sources */,
D7100C582B76FC8400C59298 /* MarketingContentView.swift in Sources */,
D7C28E3B2BBB4D0000EE459F /* VideoCache.swift in Sources */,
4CAAD8AD298851D000060CEA /* AccountDeletion.swift in Sources */,
4CFF8F6329CC9AD7008DB934 /* ImageContextMenuModifier.swift in Sources */,
4C54AA0A29A55429003E4487 /* EventGroup.swift in Sources */,
@ -3544,6 +3551,7 @@
75AD872B2AA23A460085EF2C /* Block+Tests.swift in Sources */,
E0E024112B7C19C20075735D /* TranslationTests.swift in Sources */,
F944F56E29EA9CCC0067B3BF /* DamusParseContentTests.swift in Sources */,
D7831AF82BBE11E2005DA780 /* VideoCacheTests.swift in Sources */,
3A5E47C72A4A76C800C0D090 /* TrieTests.swift in Sources */,
B501062D2B363036003874F5 /* AuthIntegrationTests.swift in Sources */,
4CB883AE2976FA9300DC99E7 /* FormatTests.swift in Sources */,

View File

@ -516,6 +516,7 @@ struct ContentView: View {
print("txn: 📙 DAMUS BACKGROUNDED")
Task { @MainActor in
damus_state.ndb.close()
VideoCache.standard?.periodic_purge()
}
break
case .inactive:

View File

@ -0,0 +1,114 @@
//
// VideoCache.swift
// damus
//
// Created by Daniel D'Aquino on 2024-04-01.
//
import Foundation
import CryptoKit
// Default expiry time of only 1 day to prevent using too much storage
fileprivate let DEFAULT_EXPIRY_TIME: TimeInterval = 60*60*24
// Default cache directory is in the system-provided caches directory, so that the operating system can delete files when it needs storage space
// (https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html)
fileprivate let DEFAULT_CACHE_DIRECTORY_PATH: URL? = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("video_cache")
struct VideoCache {
private let cache_url: URL
private let expiry_time: TimeInterval
static let standard: VideoCache? = try? VideoCache()
init?(cache_url: URL? = nil, expiry_time: TimeInterval = DEFAULT_EXPIRY_TIME) throws {
guard let cache_url_to_apply = cache_url ?? DEFAULT_CACHE_DIRECTORY_PATH else { return nil }
self.cache_url = cache_url_to_apply
self.expiry_time = expiry_time
// Create the cache directory if it doesn't exist
do {
try FileManager.default.createDirectory(at: self.cache_url, withIntermediateDirectories: true, attributes: nil)
} catch {
Log.error("Could not create cache directory: %s", for: .storage, error.localizedDescription)
throw error
}
}
/// Checks for a cached video and returns its URL if available, otherwise downloads and caches the video.
func maybe_cached_url_for(video_url: URL) throws -> URL {
let cached_url = url_to_cached_url(url: video_url)
if FileManager.default.fileExists(atPath: cached_url.path) {
// Check if the cached video has expired
let file_attributes = try FileManager.default.attributesOfItem(atPath: cached_url.path)
if let modification_date = file_attributes[.modificationDate] as? Date, Date().timeIntervalSince(modification_date) <= expiry_time {
// Video is not expired
return cached_url
} else {
Task {
// Video is expired, delete and re-download on the background
try FileManager.default.removeItem(at: cached_url)
return try await download_and_cache_video(from: video_url)
}
return video_url
}
} else {
Task {
// Video is not cached, download and cache on the background
return try await download_and_cache_video(from: video_url)
}
return video_url
}
}
/// Downloads video content using URLSession and caches it to disk.
private func download_and_cache_video(from url: URL) async throws -> URL {
let (data, response) = try await URLSession.shared.data(from: url)
guard let http_response = response as? HTTPURLResponse,
200..<300 ~= http_response.statusCode else {
throw URLError(.badServerResponse)
}
let destination_url = url_to_cached_url(url: url)
try data.write(to: destination_url)
return destination_url
}
func url_to_cached_url(url: URL) -> URL {
let hashed_url = hash_url(url)
let file_extension = url.pathExtension
return self.cache_url.appendingPathComponent(hashed_url + "." + file_extension)
}
/// Deletes all cached videos older than the expiry time.
func periodic_purge(completion: ((Error?) -> Void)? = nil) {
DispatchQueue.global(qos: .background).async {
Log.info("Starting periodic video cache purge", for: .storage)
let file_manager = FileManager.default
do {
let cached_files = try file_manager.contentsOfDirectory(at: self.cache_url, includingPropertiesForKeys: [.contentModificationDateKey], options: .skipsHiddenFiles)
for file in cached_files {
let attributes = try file.resourceValues(forKeys: [.contentModificationDateKey])
if let modification_date = attributes.contentModificationDate, Date().timeIntervalSince(modification_date) > self.expiry_time {
try file_manager.removeItem(at: file)
}
}
DispatchQueue.main.async {
completion?(nil)
}
} catch {
DispatchQueue.main.async {
completion?(error)
}
}
}
}
/// Hashes the URL using SHA-256
private func hash_url(_ url: URL) -> String {
let data = Data(url.absoluteString.utf8)
let hashed_data = SHA256.hash(data: data)
return hashed_data.compactMap { String(format: "%02x", $0) }.joined()
}
}

View File

@ -33,7 +33,7 @@ struct DamusVideoPlayer: View {
else {
mute = nil
}
_model = StateObject(wrappedValue: DamusVideoPlayerViewModel(url: url, video_size: video_size, controller: controller, mute: mute))
_model = StateObject(wrappedValue: DamusVideoPlayerViewModel.cached_video_model(url: url, video_size: video_size, controller: controller, mute: mute))
self.visibility_tracking_method = visibility_tracking_method
self.style = style
}

View File

@ -57,6 +57,12 @@ final class DamusVideoPlayerViewModel: ObservableObject {
}
}
static func cached_video_model(url: URL, video_size: Binding<CGSize?>, controller: VideoController, mute: Bool? = nil) -> Self {
let maybe_cached_url = (try? VideoCache.standard?.maybe_cached_url_for(video_url: url)) ?? url
Log.info("Loading video with URL: %s",for: .render, maybe_cached_url.absoluteString)
return Self.init(url: maybe_cached_url, video_size: video_size, controller: controller, mute: mute)
}
init(url: URL, video_size: Binding<CGSize?>, controller: VideoController, mute: Bool? = nil) {
self.url = url
player_item = AVPlayerItem(url: url)

View File

@ -0,0 +1,112 @@
//
// VideoCacheTests.swift
// damusTests
//
// Created by Daniel DAquino on 2024-04-03.
//
import Foundation
import XCTest
@testable import damus
// TODO: Reduce test dependency on external factors such as external URLs.
let TEST_VIDEO_URL = "http://cdn.jb55.com/s/zaps-build.mp4"
let LONG_TEST_EXPIRY_TIME: TimeInterval = 60 * 60 * 24 // A long expiry time for a video (in seconds).
let SHORT_TEST_EXPIRY_TIME: TimeInterval = 15 // A short expiry time for a video (in seconds). Must be as short as possible but large enough to allow some test operations to occur
let CACHE_SAVE_TIME_TIMEOUT: TimeInterval = 8 // How long the test will wait for the cache to save a file (in seconds)
let EXPIRY_TIME_MARGIN: TimeInterval = 3 // The extra time we will wait after expected expiry, to avoid test timing issues. (in seconds)
final class VideoCacheTests: XCTestCase {
func testCachedURLForExistingVideo() throws {
// Create a temporary directory for the cache
let test_cache_directory = FileManager.default.temporaryDirectory.appendingPathComponent("test_video_cache")
// Create a test video file
let original_video_url = URL(string: TEST_VIDEO_URL)!
FileManager.default.createFile(atPath: original_video_url.path, contents: Data(), attributes: nil)
// Create a VideoCache instance with the temporary cache directory
let test_expiry_time: TimeInterval = 10
let video_cache = try VideoCache(cache_url: test_cache_directory, expiry_time: test_expiry_time)!
// Call the maybe_cached_url_for method with the test video URL
let expected_cache_url = video_cache.url_to_cached_url(url: original_video_url)
let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
// Assert that the returned URL is the same as the original
XCTAssertEqual(maybe_cached_url, original_video_url, "Returned URL should be the same as the original video URL on the first time we download it")
// Check that next time we get this video, we get the cached URL.
let cached_url_expectation = XCTestExpectation(description: "On second time we get a video, the cached URL should be returned")
let start_time = Date()
while Date().timeIntervalSince(start_time) < CACHE_SAVE_TIME_TIMEOUT {
let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
if maybe_cached_url == expected_cache_url {
cached_url_expectation.fulfill()
break
}
sleep(1)
}
wait(for: [cached_url_expectation], timeout: CACHE_SAVE_TIME_TIMEOUT)
// Now wait for the remaining time until the expiry time + a margin
let remaining_time = test_expiry_time + EXPIRY_TIME_MARGIN - Date().timeIntervalSince(start_time)
// Wait for the expiry time to pass
sleep(UInt32(max(remaining_time, 0)))
// Call the periodic_purge method to purge expired video items
video_cache.periodic_purge()
// Call the maybe_cached_url_for method again
let maybe_cached_url_after_expiry = try video_cache.maybe_cached_url_for(video_url: original_video_url)
// Assert that the returned URL is the same as the original video URL, since the cache should have expired.
XCTAssertEqual(maybe_cached_url_after_expiry, original_video_url, "Video cache should expire after expiry time")
// Clean up the temporary files and directory
try FileManager.default.removeItem(at: test_cache_directory)
}
func testClearCache() throws {
// Create a temporary directory for the cache
let test_cache_directory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first!.appendingPathComponent("test_video_cache")
try FileManager.default.createDirectory(at: test_cache_directory, withIntermediateDirectories: true, attributes: nil)
// Create a test video file
let original_video_url = URL(string: TEST_VIDEO_URL)!
FileManager.default.createFile(atPath: original_video_url.path, contents: Data(), attributes: nil)
// Create a VideoCache instance with the temporary cache directory and a longer expiry time
let expiry_time: TimeInterval = LONG_TEST_EXPIRY_TIME
let video_cache = try VideoCache(cache_url: test_cache_directory, expiry_time: expiry_time)!
// Request the cached URL for the test video to create the cached file
let expected_cache_url = video_cache.url_to_cached_url(url: original_video_url)
let _ = try video_cache.maybe_cached_url_for(video_url: original_video_url)
// Check that next time we get this video, we get the cached URL.
let cached_url_expectation = XCTestExpectation(description: "On second time we get a video, the cached URL should be returned")
let start_time = Date()
while Date().timeIntervalSince(start_time) < CACHE_SAVE_TIME_TIMEOUT {
let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
if maybe_cached_url == expected_cache_url {
cached_url_expectation.fulfill()
break
}
sleep(1)
}
wait(for: [cached_url_expectation], timeout: CACHE_SAVE_TIME_TIMEOUT)
// Call the periodic_purge method
DamusCacheManager.shared.clear_cache(damus_state: test_damus_state, completion: {
// Assert that fetching the cached URL after clearing cache will
let maybe_cached_url_after_purge = try? video_cache.maybe_cached_url_for(video_url: original_video_url)
XCTAssertEqual(maybe_cached_url_after_purge, original_video_url)
// Clean up the temporary directory
try? FileManager.default.removeItem(at: test_cache_directory)
})
}
}