Compare commits

...

2 Commits

Author SHA1 Message Date
e2d6d84439 fix: improve patch host code
All checks were successful
continuous-integration/drone/push Build is passing
2025-03-28 13:38:49 +00:00
ea6499558d chore: add grafana dashboard 2025-03-28 12:58:15 +00:00
3 changed files with 757 additions and 42 deletions

700
grafana.json Normal file
View File

@ -0,0 +1,700 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"links": [],
"panels": [
{
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": false,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "cpu"
},
"properties": [
{
"id": "custom.width",
"value": 62
}
]
},
{
"matcher": {
"id": "byName",
"options": "gb_memory"
},
"properties": [
{
"id": "custom.width",
"value": 88
}
]
},
{
"matcher": {
"id": "byName",
"options": "gb_ssd"
},
"properties": [
{
"id": "custom.width",
"value": 81
}
]
},
{
"matcher": {
"id": "byName",
"options": "gb_hdd"
},
"properties": [
{
"id": "custom.width",
"value": 75
}
]
},
{
"matcher": {
"id": "byName",
"options": "load_factor"
},
"properties": [
{
"id": "custom.width",
"value": 93
}
]
},
{
"matcher": {
"id": "byName",
"options": "sold_cpu"
},
"properties": [
{
"id": "custom.width",
"value": 87
}
]
},
{
"matcher": {
"id": "byName",
"options": "sold_gb_memory"
},
"properties": [
{
"id": "custom.width",
"value": 133
}
]
},
{
"matcher": {
"id": "byName",
"options": "sold_gb_ssd_disk"
},
"properties": [
{
"id": "custom.width",
"value": 141
}
]
},
{
"matcher": {
"id": "byName",
"options": "name"
},
"properties": [
{
"id": "custom.width",
"value": 205
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "11.5.2",
"targets": [
{
"dataset": "lnvps",
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \nh.name,\nh.cpu,\nh.memory / 1024 / 1024 / 1024 gb_memory,\n(select sum(size) from vm_host_disk hd where hd.host_id = h.id and hd.enabled = 1 and hd.kind = 1) / 1024 / 1024 / 1024 gb_ssd,\n(select sum(size) from vm_host_disk hd where hd.host_id = h.id and hd.enabled = 1 and hd.kind = 0) / 1024 / 1024 / 1024 gb_hdd,\n(select sum(case when v.template_id is null then (select cpu from vm_custom_template vct where vct.id = v.custom_template_id) else (select cpu from vm_template vt where vt.id = v.template_id) end) from vm v where v.host_id = h.id and expires > current_timestamp()) sold_cpu,\n(select sum(case when v.template_id is null then (select memory from vm_custom_template vct where vct.id = v.custom_template_id) else (select memory from vm_template vt where vt.id = v.template_id) end) from vm v where v.host_id = h.id and expires > current_timestamp()) / 1024 / 1024 / 1024 sold_gb_memory,\n(select sum(case when v.template_id is null then (select disk_size from vm_custom_template vct where vct.id = v.custom_template_id and vct.disk_type = 1) else (select disk_size from vm_template vt where vt.id = v.template_id and vt.disk_type = 1) end) from vm v where v.host_id = h.id and expires > current_timestamp()) / 1024 / 1024 / 1024 sold_gb_ssd_disk,\n(select sum(case when v.template_id is null then (select disk_size from vm_custom_template vct where vct.id = v.custom_template_id and vct.disk_type = 0) else (select disk_size from vm_template vt where vt.id = v.template_id and vt.disk_type = 0) end) from vm v where v.host_id = h.id and expires > current_timestamp()) / 1024 / 1024 / 1024 sold_gb_hdd_disk,\n(select sum(case when v.template_id is null then 0 else (select (case when cp.currency = 'BTC' then (cp.amount/1e9) else cp.amount end) from vm_template vt,vm_cost_plan cp where vt.id = v.template_id and vt.cost_plan_id = cp.id) end) from vm v where v.host_id = h.id and deleted = 0) income\nfrom vm_host h",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Host Allocation",
"type": "table"
},
{
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 13,
"w": 12,
"x": 0,
"y": 5
},
"id": 2,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "11.5.2",
"targets": [
{
"dataset": "lnvps",
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select\nv.id,\ndatediff(current_timestamp(), v.created) age,\ndatediff(v.expires, current_timestamp()) days_to_expire,\n(select sum((case when currency = 'BTC' then amount / 1e11 else amount end) * rate) from lnvps.vm_payment where vm_id = v.id and is_paid = 1) total_payments\nfrom vm v\nwhere v.deleted = 0\norder by 3 asc",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Renewals",
"type": "table"
},
{
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 49
}
]
},
{
"matcher": {
"id": "byName",
"options": "ref_code"
},
"properties": [
{
"id": "custom.width",
"value": 91
}
]
},
{
"matcher": {
"id": "byName",
"options": "created"
},
"properties": [
{
"id": "custom.width"
}
]
},
{
"matcher": {
"id": "byName",
"options": "amount"
},
"properties": [
{
"id": "custom.width",
"value": 71
}
]
},
{
"matcher": {
"id": "byName",
"options": "currency"
},
"properties": [
{
"id": "custom.width",
"value": 78
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"id": 4,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "11.5.2",
"targets": [
{
"dataset": "lnvps",
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select v.id, \nv.ref_code, v.created, \n(case when vp.currency = 'BTC' then vp.amount / 1000 else vp.amount / 100 end) amount,\nvp.currency,\n(case when vp.currency = 'BTC' then vp.amount / 1000 else vp.amount / 100 end) * 0.33 comission\nfrom vm v, vm_payment vp\nwhere v.ref_code is not null\nand v.id = vp.vm_id\nand vp.is_paid = 1\norder by vp.created desc",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "RefCodes",
"type": "table"
},
{
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 13
},
"id": 5,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "11.5.2",
"targets": [
{
"dataset": "lnvps",
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select vm_id, created, \n(case when currency = 'BTC' then (amount / 1e3) else amount / 100 end) amount, \n(case when currency = 'BTC' then (amount / 1e11) * rate else amount * rate end) amount_eur,\ncurrency,\n(case when payment_method = 0 then 'LN' else 'Revolut' end) method\nfrom vm_payment\nwhere is_paid = 1\norder by created desc\nlimit 20",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Payments",
"type": "table"
},
{
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "free"
},
"properties": [
{
"id": "unit",
"value": "percentunit"
}
]
},
{
"matcher": {
"id": "byName",
"options": "region"
},
"properties": [
{
"id": "custom.width",
"value": 70
}
]
},
{
"matcher": {
"id": "byName",
"options": "used"
},
"properties": [
{
"id": "custom.width",
"value": 59
}
]
},
{
"matcher": {
"id": "byName",
"options": "size"
},
"properties": [
{
"id": "custom.width",
"value": 70
}
]
},
{
"matcher": {
"id": "byName",
"options": "size"
},
"properties": [
{
"id": "unit",
"value": "sishort"
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 18
},
"id": 1,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "11.5.2",
"targets": [
{
"dataset": "lnvps",
"datasource": {
"type": "mysql",
"uid": "cegjfe9u9181sf"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "select i.cidr, i.region, i.used, i.size, (1-i.used/i.size) as free\nfrom (\nselect r.cidr, \n(select count(id) from lnvps.vm_ip_assignment where ip_range_id = r.id and deleted = 0) used,\nhr.name as region,\npow(2, (case when r.cidr like '%:%' then 128 else 32 end)-substring_index(r.cidr, '/', -1)) as size\nfrom ip_range r, vm_host_region hr\nwhere r.region_id = hr.id) i",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "IP Ranges",
"type": "table"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 40,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "LNVPS",
"uid": "begjfxfrjwu80e",
"version": 23,
"weekStart": ""
}

View File

@ -6,7 +6,7 @@ use crate::json_api::JsonApi;
use crate::settings::{QemuConfig, SshConfig};
use crate::ssh_client::SshClient;
use crate::status::{VmRunningState, VmState};
use anyhow::{anyhow, bail, ensure, Result};
use anyhow::{anyhow, bail, ensure, Context, Result};
use chrono::Utc;
use futures::StreamExt;
use ipnetwork::IpNetwork;
@ -500,14 +500,25 @@ impl VmHostClient for ProxmoxClient {
if let Some(n) = nodes.iter().find(|n| n.name == self.node) {
let storages = self.list_storage(&n.name).await?;
let info = VmHostInfo {
cpu: n.max_cpu.unwrap_or(0),
memory: n.max_mem.unwrap_or(0),
cpu: n.max_cpu
.context("Missing cpu count, please make sure you have Sys.Audit permission")?,
memory: n.max_mem
.context("Missing memory size, please make sure you have Sys.Audit permission")?,
disks: storages
.into_iter()
.map(|s| VmHostDiskInfo {
name: s.storage,
size: s.total.unwrap_or(0),
used: s.used.unwrap_or(0),
.filter_map(|s| {
let size = s.total
.context("Missing disk size, please make sure you have Datastore.Audit permission")
.ok()?;
let used = s.used
.context("Missing used disk, please make sure you have Datastore.Audit permission")
.ok()?;
Some(VmHostDiskInfo {
name: s.storage,
size,
used,
})
})
.collect(),
};

View File

@ -2,13 +2,13 @@ use crate::host::get_host_client;
use crate::provisioner::LNVpsProvisioner;
use crate::settings::{ProvisionerConfig, Settings, SmtpConfig};
use crate::status::{VmRunningState, VmState, VmStateCache};
use anyhow::Result;
use anyhow::{bail, Result};
use chrono::{DateTime, Datelike, Days, Utc};
use lettre::message::{MessageBuilder, MultiPart};
use lettre::transport::smtp::authentication::Credentials;
use lettre::AsyncTransport;
use lettre::{AsyncSmtpTransport, Tokio1Executor};
use lnvps_db::{LNVpsDb, Vm};
use lnvps_db::{LNVpsDb, Vm, VmHost};
use log::{debug, error, info, warn};
use nostr::{EventBuilder, PublicKey, ToBech32};
use nostr_sdk::Client;
@ -280,45 +280,49 @@ impl Worker {
Ok(())
}
async fn patch_host(&self, host: &mut VmHost) -> Result<()> {
let client = match get_host_client(host, &self.settings.provisioner_config) {
Ok(h) => h,
Err(e) => bail!("Failed to get host client: {} {}", host.name, e),
};
let info = client.get_info().await?;
let needs_update = info.cpu != host.cpu || info.memory != host.memory;
if needs_update {
host.cpu = info.cpu;
host.memory = info.memory;
self.db.update_host(host).await?;
info!(
"Updated host {}: cpu={}, memory={}",
host.name, host.cpu, host.memory
);
}
let mut host_disks = self.db.list_host_disks(host.id).await?;
for disk in &info.disks {
if let Some(mut hd) = host_disks.iter_mut().find(|d| d.name == disk.name) {
if hd.size != disk.size {
hd.size = disk.size;
self.db.update_host_disk(hd).await?;
info!(
"Updated host disk {}: size={},type={},interface={}",
hd.name, hd.size, hd.kind, hd.interface
);
}
} else {
warn!("Un-mapped host disk {}", disk.name);
}
}
Ok(())
}
async fn try_job(&mut self, job: &WorkJob) -> Result<()> {
match job {
WorkJob::PatchHosts => {
let mut hosts = self.db.list_hosts().await?;
for mut host in &mut hosts {
info!("Patching host {}", host.name);
let client = match get_host_client(host, &self.settings.provisioner_config) {
Ok(h) => h,
Err(e) => {
warn!("Failed to get host client: {} {}", host.name, e);
continue;
}
};
let info = client.get_info().await?;
let needs_update = info.cpu != host.cpu || info.memory != host.memory;
if needs_update {
host.cpu = info.cpu;
host.memory = info.memory;
self.db.update_host(host).await?;
info!(
"Updated host {}: cpu={}, memory={}",
host.name, host.cpu, host.memory
);
}
let mut host_disks = self.db.list_host_disks(host.id).await?;
for disk in &info.disks {
if let Some(mut hd) = host_disks.iter_mut().find(|d| d.name == disk.name) {
if hd.size != disk.size {
hd.size = disk.size;
self.db.update_host_disk(hd).await?;
info!(
"Updated host disk {}: size={},type={},interface={}",
hd.name, hd.size, hd.kind, hd.interface
);
}
} else {
warn!("Un-mapped host disk {}", disk.name);
}
if let Err(e) = self.patch_host(&mut host).await {
error!("Failed to patch host {}: {}", host.name, e);
}
}
}