From f57e46c7793820ca599710f9e2014a4718dad2c3 Mon Sep 17 00:00:00 2001 From: Kieran Date: Thu, 29 Nov 2018 16:44:06 +0800 Subject: [PATCH] fix stats, add redis slave reads --- src/php/abuse.php | 4 ++-- src/php/api.php | 3 ++- src/php/config.php | 4 ++-- src/php/filestore.php | 4 ++-- src/php/staticredis.php | 30 +++++++++++++++++++++++++++--- src/php/stats.php | 22 +++++++++++++--------- src/php/sync.php | 2 +- src/php/tracking.php | 5 +++-- src/php/upload.php | 18 ++---------------- 9 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/php/abuse.php b/src/php/abuse.php index f16264a..a208ee4 100644 --- a/src/php/abuse.php +++ b/src/php/abuse.php @@ -1,7 +1,7 @@ hGet($key, $id); @@ -38,7 +38,7 @@ } public function ResetRateLimits($id) { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::WriteOp(); $key = REDIS_PREFIX . "uvc:" . USER_IP; $redis->hSet($key, $id, 0); } diff --git a/src/php/api.php b/src/php/api.php index 79e0906..2c0fc81 100644 --- a/src/php/api.php +++ b/src/php/api.php @@ -29,7 +29,8 @@ "max_upload_size" => Config::$Instance->max_upload_size, "basic_stats" => Stats::Get(), "upload_host" => Upload::GetUploadHost(), - "geoip_info" => geoip_database_info() + "geoip_info" => geoip_database_info(), + "host" => gethostname() ); break; } diff --git a/src/php/config.php b/src/php/config.php index 9120b31..e20b7ee 100644 --- a/src/php/config.php +++ b/src/php/config.php @@ -3,12 +3,12 @@ public static $Instance; public static function GetConfig($config_name) { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); return $redis->hGet(REDIS_PREFIX . 'config', $config_name); } public static function MGetConfig($config_name) { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); return (object)$redis->hMGet(REDIS_PREFIX . 'config', $config_name); } diff --git a/src/php/filestore.php b/src/php/filestore.php index c97cf80..3ac7898 100644 --- a/src/php/filestore.php +++ b/src/php/filestore.php @@ -9,7 +9,7 @@ } public function SetFileStats($info) : void { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::WriteOp(); $file_key = REDIS_PREFIX . $info->FileId; $redis->hMSet($file_key, array( @@ -19,7 +19,7 @@ } public function GetFileStats($id) : object { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); $file_key = REDIS_PREFIX . $id; $public_file_info = $redis->hMGet($file_key, array('views', 'lastview')); diff --git a/src/php/staticredis.php b/src/php/staticredis.php index 8de05fe..00b1bb3 100644 --- a/src/php/staticredis.php +++ b/src/php/staticredis.php @@ -2,10 +2,34 @@ class StaticRedis { public static $Instance = NULL; + public static $MasterInstance = NULL; - public static function Connect(){ - self::$Instance = new Redis(); - return self::$Instance->pconnect(REDIS_CONFIG); + public static function ReadOp() : object { + return self::$Instance; } + + public static function WriteOp() : object { + if(self::$MasterInstance != NULL){ + return self::$MasterInstance; + } else { + return self::$Instance; + } + } + + public static function Connect() : bool { + self::$Instance = new Redis(); + $con = self::$Instance->pconnect(REDIS_CONFIG); + if($con){ + $rep = self::$Instance->info("REPLICATION"); + if($rep["role"] == "slave"){ + self::$MasterInstance = new Redis(); + $mcon = self::$MasterInstance->pconnect($rep["master_host"], $rep["master_port"]); + return $con && $mcon; + } + } + return $con; + } + + } ?> \ No newline at end of file diff --git a/src/php/stats.php b/src/php/stats.php index 3388a94..8c50f37 100644 --- a/src/php/stats.php +++ b/src/php/stats.php @@ -4,17 +4,21 @@ public $Size; public $Transfer_24h; - private static $AllTransferStatsKey = REDIS_PREFIX . "stats-transfer-all"; + private static $AllTransferStatsKey = REDIS_PREFIX . "stats-transfer-all:"; private static $GeneralStatsKey = REDIS_PREFIX . "stats-general"; public static function Get() : Stats { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); //calculate 24hr transfer stats - $tx_24h_array = $redis->zRange(self::$AllTransferStatsKey, 0, 24, true); //stats are 1hr interval $tx_24h = 0; - foreach($tx_24h_array as $tx_key => $tx_bytes) { - $tx_24h += $tx_bytes; + $now = time(); + for($x = 0; $x < 24; $x += 1) { + $stat_key = date("YmdH", $now - (60 * 60 * $x)); + $val = $redis->get(self::$AllTransferStatsKey . $stat_key); + if($val != False){ + $tx_24h += intval($val); + } } //get general stats @@ -29,18 +33,18 @@ } public static function TrackTransfer($id, $size) : void { - //$redis = StaticRedis::$Instance; self::AddAllTransfer($size); } public static function AddAllTransfer($size) : void { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::WriteOp(); $stat_member = date("YmdH"); - $redis->zIncrBy(self::$AllTransferStatsKey, $size, $stat_member); + $redis->incrBy(self::$AllTransferStatsKey . $stat_member, $size); + $redis->setTimeout(self::$AllTransferStatsKey . $stat_member, 2592000); //store 30 days only } public static function Collect($fs) : void { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::WriteOp(); $files = $fs->ListFiles(); $total_size = 0; diff --git a/src/php/sync.php b/src/php/sync.php index 1affcc9..34f9815 100644 --- a/src/php/sync.php +++ b/src/php/sync.php @@ -11,7 +11,7 @@ $fs = new FileStore(Config::$Instance->upload_folder); if(!$fs->FileExists($id)) { //resolve the hostnames to ips - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); $sync_hosts = $redis->sMembers(REDIS_PREFIX . 'sync-hosts'); $sync_hosts_ips = array(); diff --git a/src/php/tracking.php b/src/php/tracking.php index ba6ca34..b725640 100644 --- a/src/php/tracking.php +++ b/src/php/tracking.php @@ -1,7 +1,7 @@ GetFileSize($id); @@ -52,7 +52,8 @@ "urlref" => isset($_SERVER["HTTP_REFERER"]) ? $_SERVER["HTTP_REFERER"] : "" )); - StaticRedis::$Instance->publish('ga-page-view-matomo', $msg); + //this should be sent to the slave node if we are connected on a slave + StaticRedis::ReadOp()->publish('ga-page-view-matomo', $msg); } } ?> \ No newline at end of file diff --git a/src/php/upload.php b/src/php/upload.php index cb63104..fc807aa 100644 --- a/src/php/upload.php +++ b/src/php/upload.php @@ -53,7 +53,7 @@ } function SyncFileUpload($id) : array { - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); $sync_hosts = $redis->sMembers(REDIS_PREFIX . 'sync-hosts'); if($sync_hosts !== False) { $fs = new FileStore(Config::$Instance->upload_folder); @@ -64,20 +64,6 @@ } return $status_codes; - /* - $sync_threads = array(); - foreach($sync_hosts as $host) { - $new_thread = new SyncThread($id, $fs->GetAbsoluteFilePath($id), $host); - $new_thread->start(); - $sync_threads[] = $new_thread; - } - - foreach($sync_threads as $thread) { - while($thread->isRunning()) { - usleep(100); - } - $thread->join(); - }*/ } return array(); @@ -98,7 +84,7 @@ $cont = "EU"; } - $redis = StaticRedis::$Instance; + $redis = StaticRedis::ReadOp(); $map = $redis->hGetAll(REDIS_PREFIX . "upload-region-mapping"); if($map !== False && isset($map[$cont])) { return $map[$cont];