用ECharts繪製Prometheus圖表實現類似Grafana的自定義Dashboard

  大家一般都是用Grafana自定義Dashboard來監控Prometheus數據的，作者這次嘗試用ECharts來繪製Prometheus數據圖表，一方面可以減少依賴，另一方面可以將監控界面靈活的集成進應用系統。至於如何在被監測機器上安裝NodeExporter以及如何部署Prometheus作者就不描述了，園子裡有很多文章介紹。

一、數據查詢及轉換

  Prometheus提供了Http Api來執行promql查詢，但需要將返回的數據格式轉換為ECharts的格式，好在EChars的xAxis.type可以設置為'time'類型，與Prometheus返回的格式接近。作者寫了個簡單的服務來執行查詢及轉換數據，詳見以下代碼：

public class MetricService
{
 private static readonly HttpClient http = new HttpClient()
 {
 //請修改指向Prometheus地址
 BaseAddress = new Uri("http://10.211.55.2:9090/api/v1/"),
 Timeout = TimeSpan.FromSeconds(2)
 };
 public async Task<object> GetCpuUsages(string node, DateTime start, DateTime end)
 {
 var promql = $"100-irate(node_cpu{{instance='{node}:9100',mode='idle'}}[5m])*100";
 return await QueryRange(promql, start, end, 20, 2);
 }
 public async Task<object> GetMemUsages(string node, DateTime start, DateTime end)
 {
 var promql = $"(1-(node_memory_MemAvailable{{instance='{node}:9100'}}/(node_memory_MemTotal{{instance='{node}:9100'}})))*100";
 return await QueryRange(promql, start, end, 20, 2);
 }
 public async Task<object> GetNetTraffic(string node, DateTime start, DateTime end)
 {
 var downql = $"irate(node_network_receive_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
 var ls = await QueryRange(downql, start, end, 15/*4*/, 0);
 var upql = $"irate(node_network_transmit_bytes{{instance='{node}:9100',device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}}[5m])";
 ls.Add(await QueryRange(upql, start, end, 15/*4*/, 0));
 return ls;
 }
 public async Task<object> GetDiskIO(string node, DateTime start, DateTime end)
 {
 var readql = $"irate(node_disk_bytes_read{{instance='{node}:9100'}}[1m])";
 var ls = await QueryRange(readql, start, end, 15/*10*/, 0);
 var writeql = $"irate(node_disk_bytes_written{{instance='{node}:9100'}}[1m])";
 ls.Add(await QueryRange(writeql, start, end, 15/*10*/, 0));
 return ls;
 } 

 #region ====Parse PromQL====
 private static async Task<list>> QueryRange(string promql, DateTime start, DateTime end, int step, int round)
 {
 if (start >= end) throw new ArgumentOutOfRangeException();
 var ts1 = (int)(start.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
 var ts2 = (int)(end.ToUniversalTime() - DateTime.UnixEpoch).TotalSeconds;
 var res = await http.GetAsync($"query_range?query={promql}&start={ts1}&end={ts2}&step={step}s");
 var stream = await res.Content.ReadAsStreamAsync();
 using (var sr = new System.IO.StreamReader(stream))
 using (var jr = new JsonTextReader(sr))
 {
 return ParseToSeries(jr, round);
 }
 }
 private static List<object> ParseToSeries(JsonTextReader jr, int round)
 {
 if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "status")
 throw new Exception();
 var status = jr.ReadAsString();
 if (status != "success") throw new Exception();
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "data")
 throw new Exception();
 if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "resultType")
 throw new Exception();
 var resultType = jr.ReadAsString();
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "result")
 throw new Exception();
 return ReadResultArray(jr, round);
 //No need read others
 }
 private static List<object> ReadResultArray(JsonTextReader jr, int round)
 {
 if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();
 var ls = new List<object>();
 do
 {
 if (!jr.Read()) throw new Exception();
 if (jr.TokenType == JsonToken.EndArray) break;
 if (jr.TokenType != JsonToken.StartObject) throw new Exception();
 ls.Add(ReadResultItem(jr, round));
 } while (true);
 return ls;
 }
 private static List<double> ReadResultItem(JsonTextReader jr, int round)
 {
 //已讀取StartObject標記
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "metric")
 throw new Exception(); 

 ReadMetric(jr);
 if (!jr.Read() || jr.TokenType != JsonToken.PropertyName || (string)jr.Value != "values")
 throw new Exception();
 var values = ReadValues(jr, round);
 if (!jr.Read() || jr.TokenType != JsonToken.EndObject) throw new Exception();
 return values;
 }
 private static void ReadMetric(JsonTextReader jr)
 {
 if (!jr.Read() || jr.TokenType != JsonToken.StartObject) throw new Exception();
 do
 {
 //PropertyName or EndObject
 if (!jr.Read()) throw new Exception();
 if (jr.TokenType == JsonToken.EndObject) return;
 //PropertyValue
 jr.Read();
 } while (true);
 }
 private static List<double> ReadValues(JsonTextReader jr, int round)
 {
 if (!jr.Read() || jr.TokenType != JsonToken.StartArray) throw new Exception();
 var ls = new List<double>();
 do
 {
 if (!jr.Read()) throw new Exception();
 if (jr.TokenType == JsonToken.EndArray) break;
 if (jr.TokenType != JsonToken.StartArray) throw new Exception();
 var ts = jr.ReadAsDouble().Value * 1000; //PromQL時間*1000
 var value = Math.Round(double.Parse(jr.ReadAsString()), round, MidpointRounding.ToEven); //PromQL值為字符串
 ls.Add(new double[] { ts, value });
 if (!jr.Read() || jr.TokenType != JsonToken.EndArray) throw new Exception();
 } while (true);
 return ls;
 }
 #endregion
}
/<double>/<double>/<double>/<object>/<object>/<object>/<list>/<object>/<object>/<object>/<object>

Tip: promql的寫法可參考grafana網站相關Dashboard。

二、單指標Vue組件

  作者使用Vue-ECharts作為ECharts的包裝，以CPU使用率Vue組件為例：

<v-chart>
/<v-chart>

@Component
export default class CpuUsages extends Vue {
 /** 目標實例IP */
 @Prop({ type: String, default: '10.211.55.3' }) node
 /** 開始時間 */
 @Prop({ type: Date, default: () => { var now = new Date(); return new Date(now.getFullYear(), now.getMonth(), now.getDate()) } }) start
 /** 結束時間 */
 @Prop({ type: Date, default: () => { return new Date() } }) end
 chartOptions = {
 title: { text: 'Cpu Usages', x: 'center' },
 tooltip: { trigger: 'axis' },
 xAxis: { type: 'time' },
 yAxis: { min: 0, max: 100 },
 series: []
 }
 refresh() {
 sys.Services.MetricService.GetCpuUsages(this.node, this.start, this.end).then(res => {
 this.chartOptions.series.splice(0)
 for (var i = 0; i < res.length; ++i) {
 var seria = { type: 'line', name: 'cpu' + i, data: res[i], showSymbol: false }
 this.chartOptions.series.push(seria)
 }
 }).catch(err => {
 this.$message(err)
 })
 }
 mounted() {
 this.refresh()
 }
}

三、組合多個組件形成Dashboard

  根據需要可以靈活組合多個指標組件，形成相應的Dashboard界面（如下圖所示）。

四、小結

  感謝Vue、ECharts、Vue-ECharts、Prometheus等項目，使得開發並集成監控Dashboard如此簡單。另碼文不易，碼技術文更不易，所以請您多多推薦。