fix timescaledb inserting and grafana dashboard accordingly

chaoyangnz · Jun 11, 2021 · dc74bdd · dc74bdd
1 parent 72c45d1
commit dc74bdd
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 117 deletions.
diff --git a/ops/docker-compose.yml b/ops/docker-compose.yml
@@ -16,6 +16,8 @@ services:
     image: grafana/grafana:latest
     ports:
       - "3000:3000"
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
     volumes:
       - ./grafana/provisioning:/etc/grafana/provisioning
     links:

diff --git a/ops/grafana/provisioning/dashboards/loadrush.json b/ops/grafana/provisioning/dashboards/loadrush.json
@@ -23,12 +23,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "timescaledb",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -56,7 +50,7 @@
         "alertThreshold": false
       },
       "percentage": false,
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -167,8 +161,9 @@
       "description": "",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "mappings": [],
+          "max": 100,
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -198,17 +193,18 @@
           "values": false
         },
         "showThresholdLabels": false,
-        "showThresholdMarkers": true
+        "showThresholdMarkers": true,
+        "text": {}
       },
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "targets": [
         {
           "format": "time_series",
           "group": [],
           "metricColumn": "none",
           "queryType": "randomWalk",
           "rawQuery": true,
-          "rawSql": "SELECT time, \ncase \n  when (sum(success) + sum(failure) + sum(error_capture)) != 0 then sum(success) * 100 / (sum(success) + sum(failure) + sum(error_capture)) \n  else null\nend as \"success\",\ncase \n  when (sum(success) + sum(failure) + sum(error_capture)) = 0 then null \n  else sum(failure) * 100 / (sum(success) + sum(failure) + sum(error_capture)) \nend as \"failure\"\nfrom\n(\n  SELECT\n    time_bucket('1 seconds', time) as time, \n    case when success = 'true' then 1 else 0 end as success, \n    case when failure = 'true' then 1 else 0 end as failure, \n    case when error_capture = 'true' then 1 else 0 end as error_capture\n  FROM \n   response\n  WHERE\n    $__timeFilter(\"time\")\n) as response\nGROUP BY time\nORDER BY time",
+          "rawSql": "SELECT time, \ncase \n  when (sum(success) + sum(failure) + sum(error_capture)) != 0 then sum(success) * 100 / (sum(success) + sum(failure) + sum(error_capture)) \n  else null\nend as \"success\",\ncase \n  when (sum(success) + sum(failure) + sum(error_capture)) = 0 then null \n  else sum(failure) * 100 / (sum(success) + sum(failure) + sum(error_capture)) \nend as \"failure\"\nfrom\n(\n  SELECT\n    time_bucket('1 seconds', response.time) as time, \n    case when response_success.success = 'true' then 1 else 0 end as success, \n    case when response_failure.failure = 'true' then 1 else 0 end as failure, \n    case when response.error_capture = 'true' then 1 else 0 end as error_capture\n  FROM \n   response\n  LEFT join\n   response_success\n  on (response.trace = response_success.trace)\n  LEFT join\n   response_failure\n  on (response.trace = response_failure.trace) \n  WHERE\n    $__timeFilter(response.time)\n) as response\nGROUP BY time\nORDER BY time",
           "refId": "A",
           "select": [
             [
@@ -241,12 +237,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "timescaledb",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -273,7 +263,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -359,12 +349,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "timescaledb",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -391,7 +375,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -407,7 +391,7 @@
           "metricColumn": "none",
           "queryType": "randomWalk",
           "rawQuery": true,
-          "rawSql": "SELECT time, count(success) as response, sum(success) as \"😀\", sum(failure) as \"😩\", sum(error_capture) as \"😱\" from\n(\n  SELECT\n    time_bucket('1 s', time) as time, \n    case when success = 'true' then 1 else 0 end as success, \n    case when failure = 'true' then 1 else 0 end as failure, \n    case when error_capture = 'true' then 1 else 0 end as error_capture\n  FROM \n   response\n  WHERE\n    $__timeFilter(\"time\")\n) as response\nGROUP BY time\nORDER BY time",
+          "rawSql": "SELECT time, count(success) as response, sum(success) as \"😀\", sum(failure) as \"😩\", sum(error_capture) as \"😱\" from\n(\n  SELECT\n    time_bucket('1 s', response.time) as time, \n    case when response_success.success = 'true' then 1 else 0 end as success, \n    case when response_failure.failure = 'true' then 1 else 0 end as failure, \n    case when response.error_capture = 'true' then 1 else 0 end as error_capture\n  FROM \n   response\n  LEFT JOIN\n   response_success\n  ON (response.trace = response_success.trace) \n  LEFT JOIN\n   response_failure\n  ON (response.trace = response_failure.trace) \n  WHERE\n    $__timeFilter(response.time)\n) as response\nGROUP BY time\nORDER BY time",
           "refId": "A",
           "select": [
             [
@@ -478,12 +462,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "timescaledb",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -510,7 +488,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -596,12 +574,6 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "timescaledb",
-      "fieldConfig": {
-        "defaults": {
-          "custom": {}
-        },
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -628,7 +600,7 @@
         "alertThreshold": false
       },
       "percentage": false,
-      "pluginVersion": "7.3.5",
+      "pluginVersion": "8.0.0",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -735,7 +707,7 @@
     }
   ],
   "refresh": "5s",
-  "schemaVersion": 26,
+  "schemaVersion": 30,
   "style": "dark",
   "tags": [],
   "templating": {
@@ -749,5 +721,5 @@
   "timezone": "",
   "title": "Loadrush Dashboard",
   "uid": "loadrush",
-  "version": 1
+  "version": 2
 }
diff --git a/ops/timescaledb/loadrush.sql b/ops/timescaledb/loadrush.sql
@@ -1,12 +1,12 @@
 CREATE TABLE virtual_user (
  dataset       VARCHAR         NOT NULL,
- time          TIMESTAMPTZ     NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
  active        INTEGER         NOT NULL
 );
 
 CREATE TABLE request (
  dataset       VARCHAR         NOT NULL,
- time          TIMESTAMPTZ     NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
  trace         VARCHAR         NOT NULL,
  scenario      VARCHAR         NOT NULL,
  action        VARCHAR         NOT NULL,
@@ -17,7 +17,7 @@ CREATE TABLE request (
 
 CREATE TABLE response (
  dataset       VARCHAR         NOT NULL,
- time          TIMESTAMPTZ     NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
  trace         VARCHAR         NOT NULL,
  method        VARCHAR         NOT NULL,
  url           VARCHAR         NOT NULL,
@@ -31,16 +31,30 @@ CREATE TABLE response (
  timing_total  INTEGER         NOT NULL,
  status_code   INTEGER         NOT NULL,
  success       BOOLEAN,
- time_success  TIMESTAMPTZ,
+ time_success  TIMESTAMP(3) WITH TIME ZONE,
  failure       BOOLEAN,
- time_failure  TIMESTAMPTZ,
+ time_failure  TIMESTAMP(3) WITH TIME ZONE,
  error_capture       BOOLEAN,
- time_error_capture  TIMESTAMPTZ
+ time_error_capture  TIMESTAMP(3) WITH TIME ZONE
+);
+
+CREATE TABLE response_success (
+ dataset       VARCHAR         NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
+ trace         VARCHAR         NOT NULL,
+ success       BOOLEAN
+);
+
+CREATE TABLE response_failure (
+ dataset       VARCHAR         NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
+ trace         VARCHAR         NOT NULL,
+ failure       BOOLEAN
 );
 
 CREATE TABLE error (
  dataset       VARCHAR         NOT NULL,
- time          TIMESTAMPTZ     NOT NULL,
+ time          TIMESTAMP(3) WITH TIME ZONE     NOT NULL,
  trace         VARCHAR         NOT NULL,
  virtual_user  INTEGER         NOT NULL,
  method        VARCHAR         NOT NULL,
@@ -53,4 +67,6 @@ CREATE TABLE error (
 SELECT create_hypertable('virtual_user', 'time');
 SELECT create_hypertable('request', 'time');
 SELECT create_hypertable('response', 'time');
+SELECT create_hypertable('response_success', 'time');
+SELECT create_hypertable('response_failure', 'time');
 SELECT create_hypertable('error', 'time');
diff --git a/src/loadrush/actions/http.ts b/src/loadrush/actions/http.ts
@@ -59,12 +59,13 @@ export type AfterResponseCallback = (
   context: Context,
 ) => Promise<void>;
 
+const logger = new Logger('loadrush:action:http');
+
 export function request(requestSpec: RequestSpec): Action {
   return {
     type: ActionType.STEP,
     title: `${requestSpec.method} ${requestSpec.url}`,
     run: async (context: ActionContext) => {
-      const logger = new Logger('loadrush:action:http');
       const spec = cloneDeep(requestSpec);
 
       const method = spec.method;

diff --git a/src/loadrush/config.ts b/src/loadrush/config.ts
@@ -10,28 +10,26 @@ import {
 import dotenv from 'dotenv';
 
 export interface Config {
-  loadrush: {
-    vuPoolSize: number;
-    duration: number;
-    baseUrl: string;
-    meter: 'timescaledb' | 'influxdb';
-    http: {
-      maxSockets: number;
-      activeSocketTimeout: number;
-      freeSocketTimeout: number;
-    };
-    timescaledb?: {
-      host: string;
-      port: number;
-    };
-    influxdb?: {
-      org: string;
-      bucket: string;
-      token: string;
-      api: string;
-      tags: string;
-      verboseMetrics: boolean;
-    };
+  vuPoolSize: number;
+  duration: number;
+  baseUrl: string;
+  meter: 'timescaledb' | 'influxdb';
+  http: {
+    maxSockets: number;
+    activeSocketTimeout: number;
+    freeSocketTimeout: number;
+  };
+  timescaledb?: {
+    host: string;
+    port: number;
+  };
+  influxdb?: {
+    org: string;
+    bucket: string;
+    token: string;
+    api: string;
+    tags: string;
+    verboseMetrics: boolean;
   };
 }
 
@@ -75,22 +73,21 @@ export function initConfig() {
   dotenv.config();
   config = resolve<Config>(
     {
-      loadrush: {
-        vuPoolSize: 10_000,
-        duration: 600,
-        baseUrl: '',
-        meter: 'timescaledb',
-        http: {
-          maxSockets: 1000_000,
-          activeSocketTimeout: 60_000,
-          freeSocketTimeout: 30_000,
-        },
-        timescaledb: {
-          host: 'localhost',
-          port: 5432,
-        },
+      vuPoolSize: 10_000,
+      duration: 600,
+      baseUrl: '',
+      meter: 'timescaledb',
+      http: {
+        maxSockets: 1000_000,
+        activeSocketTimeout: 60_000,
+        freeSocketTimeout: 30_000,
+      },
+      timescaledb: {
+        host: 'localhost',
+        port: 5432,
       },
     },
     process.env,
+    'LOADRUSH',
   );
 }
diff --git a/src/loadrush/http.ts b/src/loadrush/http.ts
@@ -51,9 +51,9 @@ export class DefaultHttp implements Http {
   constructor() {
     const agentOptions: HttpOptions = {
       keepAlive: true,
-      maxSockets: config.loadrush.http.maxSockets,
-      timeout: config.loadrush.http.activeSocketTimeout,
-      freeSocketTimeout: config.loadrush.http.freeSocketTimeout,
+      maxSockets: config.http.maxSockets,
+      timeout: config.http.activeSocketTimeout,
+      freeSocketTimeout: config.http.freeSocketTimeout,
     };
     this.instance = got.extend({
       mutableDefaults: true,